1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 let worktree = state.worktree_root().join("under-review");
1033 if let Some(parent) = worktree.parent() {
1034 tokio::fs::create_dir_all(parent).await.ok();
1035 }
1036 let path = worktree.to_string_lossy().to_string();
1037 git::git(repo, &["worktree", "add", &path, branch])
1038 .await
1039 .with_context(|| {
1040 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1041 })?;
1042
1043 let setup = match ensure_setup_config(&mut state, repo).await {
1044 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1045 Err(e) => Err(e),
1046 };
1047 if let Err(e) = setup {
1048 git::worktree_remove_clean(repo, &worktree).await.ok();
1049 return Err(e);
1050 }
1051 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1052 .await
1053 .unwrap_or(0);
1054 if commits == 0 {
1055 git::worktree_remove_clean(repo, &worktree).await.ok();
1056 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1057 }
1058 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1059 .await
1060 .map(|f| f.len())
1061 .unwrap_or(0);
1062 if files == 0
1063 && let (Ok(head_tree), Ok(base_tree)) = (
1064 git::tree_of(&worktree, "HEAD").await,
1065 git::tree_of(&worktree, &base_commit).await,
1066 )
1067 && head_tree == base_tree
1068 {
1069 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1070 git::worktree_remove_clean(repo, &worktree).await.ok();
1071 bail!(
1072 "`{branch}` at {} has a tree identical to base {}; this usually means \
1073 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1074 against `{}/{branch}`) rather than an empty change",
1075 short(&head),
1076 short(&base_commit),
1077 state.config.merge.remote
1078 );
1079 }
1080 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1081 .await
1082 .unwrap_or_default();
1083
1084 state.candidates.push(Candidate {
1085 index: 0,
1086 label: 'A',
1087 agent: EXISTING_BRANCH.to_owned(),
1090 branch: branch.to_owned(),
1091 worktree,
1092 summary: String::new(),
1093 stat,
1094 files,
1095 commits,
1096 empty: false,
1097 failed: None,
1098 verified_noop: None,
1099 duration_ms: 0,
1100 folded: false,
1101 });
1102 state.tally = Some(Tally {
1103 first_choice: BTreeMap::from([('A', 0)]),
1104 borda: BTreeMap::new(),
1105 winner: 'A',
1106 rankings: 0,
1107 unanimous_initial: false,
1108 deliberated: false,
1109 changed_votes: 0,
1110 unanimous_final: false,
1111 tie_break: None,
1112 judges: 0,
1116 present: 0,
1117 quorum: 0,
1118 met_quorum: true,
1119 uncontested: Some("review-only run: nothing competed".to_owned()),
1120 });
1121 state.status = RunStatus::Reviewing;
1122 state.prep_complete = Some(true);
1123 state.event(
1124 "start",
1125 format!(
1126 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1127 state.id
1128 ),
1129 );
1130 state.save()?;
1131 Ok(Self {
1132 state,
1133 roles,
1134 sem: Arc::new(Semaphore::new(max_parallel)),
1135 pause: Pause::new(),
1136 interrupt: Pause::new(),
1137 })
1138 }
1139
1140 pub fn resume(id: &str) -> Result<Self> {
1142 let state = RunState::load(id)?;
1143 crate::pr_head::ensure_owner(&state)?;
1144 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1145 bail!(
1146 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1147 );
1148 }
1149 if let Some(to) = &state.released_to {
1150 bail!(
1151 "run {} cannot be resumed: its worktree was released to run {}",
1152 state.short(),
1153 crate::run::short_of(to)
1154 );
1155 }
1156 let roles = state.config.resolve_roles()?;
1157 if let Some(id) = &state.config.graph.selected_implementer
1158 && state.candidates.iter().any(|c| c.agent != *id)
1159 {
1160 bail!(
1161 "replacement implementer pin does not match the saved candidates; task must stay held"
1162 );
1163 }
1164 let max_parallel = state.config.graph.max_parallel.max(1);
1165 Ok(Self {
1166 state,
1167 roles,
1168 sem: Arc::new(Semaphore::new(max_parallel)),
1169 pause: Pause::new(),
1170 interrupt: Pause::new(),
1171 })
1172 }
1173
1174 pub async fn execute(&mut self) -> Result<()> {
1181 if let Some(selection) = &self.state.pr_recovery
1182 && (selection.source_proof.is_some() || selection.source_run.is_some())
1183 {
1184 Box::pin(selection.verify_source(&self.state.repo)).await?;
1185 }
1186 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1189 || self.state.land_approval.is_some()
1190 || self.state.merge_decision.is_some()
1191 || !self.state.merge_repairs.is_empty()
1192 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1193 {
1194 let claim = FixClaim::acquire(&self.state.dir())?;
1195 let disk = RunState::load(&self.state.id)?;
1196 let checkpoint = |s: &RunState| {
1197 (
1198 s.merge_repairs.len(),
1199 s.merge_repairs
1200 .last()
1201 .map(|r| (r.question.clone(), r.phase)),
1202 s.land_approval
1203 .as_ref()
1204 .map(|a| (a.question.clone(), a.head.clone())),
1205 )
1206 };
1207 if checkpoint(&disk) != checkpoint(&self.state) {
1208 bail!(
1209 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1210 );
1211 }
1212 Some(claim)
1213 } else {
1214 None
1215 };
1216 let mut execution = self.claim_driver()?;
1217 let runner = &mut *execution.runner;
1218 let wait = runner.repository_wait();
1219 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1222 let result = match result {
1223 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1224 record_repository_block(&mut runner.state, &e);
1225 runner.park_here()?;
1226 Ok(())
1227 }
1228 other => other,
1229 };
1230 execution.driver.finish(&mut runner.state)?;
1231 let ended = if result.is_err() {
1232 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1233 } else {
1234 crate::notices::run_ended(&runner.state)
1235 };
1236 if let Some(notice) = ended {
1237 crate::notices::raise(notice);
1238 }
1239 result
1240 }
1241
1242 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1243 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1244 }
1245
1246 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1247 let wait = self.repository_wait();
1248 coordinate_operation(&mut self.state, cwd, &wait).await
1249 }
1250
1251 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1253 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1254 Ok(owner) => owner,
1255 Err(error) => {
1256 let detail = format!("worktree maintenance or ownership update: {error:#}");
1257 self.state.resource_block = Some(crate::run::ResourceBlock {
1261 node: "start".into(),
1262 kind: "worktree maintenance".into(),
1263 detail: detail.clone(),
1264 at: Timestamp::now(),
1265 });
1266 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1267 }
1268 };
1269 let disk = RunState::load(&self.state.id)?;
1270 crate::pr_head::ensure_owner(&disk)?;
1271 if disk.liveness(false) == crate::run::Liveness::Live {
1272 bail!("run already has a live driver; wait for its execution to end");
1273 }
1274 if let Some(to) = &disk.released_to {
1275 bail!(
1276 "run {} cannot continue: its worktree was released to run {}",
1277 self.state.short(),
1278 crate::run::short_of(to)
1279 );
1280 }
1281 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1282 let mut execution = DriverExecution {
1283 runner: self,
1284 driver,
1285 };
1286 let _owner = owner;
1289 let state = &mut execution.runner.state;
1290 state.parked = false;
1295 state.resource_wait = None;
1296 state.resource_block = None;
1297 state.clear_active();
1304 let pid = std::process::id();
1319 state.driver_pid = Some(pid);
1320 state.driver_started_at = crate::proc::process_started_at(pid);
1321 state.driver_exited = false;
1322 state.begin_pass();
1323 execution.driver.bind(state);
1324 state.save()?;
1325 execution.driver.record(state)?;
1326 Ok(execution)
1327 }
1328
1329 async fn execute_graph(&mut self) -> Result<()> {
1330 self.state.configure_recovery_resume()?;
1331 self.persist_candidates().await?;
1334 if self
1335 .state
1336 .merge_decision
1337 .as_ref()
1338 .is_some_and(|decision| !decision.closed)
1339 && self.state.status != RunStatus::Landing
1340 {
1341 if crate::merge_decision::resumable_hold(&self.state) {
1342 self.state.merge = Some(MergeOutcome {
1343 mode: MergeMode::Pr,
1344 ok: false,
1345 detail: self
1346 .state
1347 .merge_decision
1348 .as_ref()
1349 .unwrap()
1350 .target
1351 .pr
1352 .clone(),
1353 empty: false,
1354 });
1355 self.run_land().await?;
1356 self.settle_questions();
1357 return Ok(());
1358 }
1359 bail!(
1360 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1361 );
1362 }
1363 if self.state.config.graph.selected_implementer.is_some()
1364 && self.state.status == RunStatus::Blocked
1365 && self.state.tally.is_none()
1366 && self.state.candidate_preservation.is_empty()
1367 && self.state.interrupted_implementations.is_empty()
1368 {
1369 bail!(
1370 "selected implementation is held; explicitly release the owning task to retry it"
1371 );
1372 }
1373
1374 if self.state.merge_repairs.last().is_some_and(|r| {
1375 matches!(
1376 r.phase,
1377 crate::run::MergeRepairPhase::Requested
1378 | crate::run::MergeRepairPhase::Fixing
1379 | crate::run::MergeRepairPhase::Reviewing
1380 )
1381 }) {
1382 if self.repair_merge_review().await? {
1383 self.run_land().await?;
1384 }
1385 self.settle_questions();
1386 return Ok(());
1387 }
1388 if self.state.pr_publication_pending {
1391 self.merge().await?;
1392 if self.resource_halt()? {
1393 return Ok(());
1394 }
1395 self.state.save()?;
1396 self.settle_questions();
1397 return Ok(());
1398 }
1399 if self.state.status == RunStatus::Stalled {
1412 if self.recover_stall().await? {
1413 self.finish_after_tally().await?;
1414 } else {
1415 self.state.save()?;
1417 }
1418 return Ok(());
1419 }
1420 if self.state.status == RunStatus::Landing
1430 || crate::handover::landing_checkpoint(&self.state)
1431 {
1432 self.state.land_sync_pending = true;
1433 self.state.save()?;
1434 self.run_land().await?;
1435 self.settle_questions();
1440 return Ok(());
1441 }
1442 if self.state.github_text.is_some()
1447 && self.state.merge.is_none()
1448 && self.state.status == RunStatus::Gating
1449 {
1450 self.merge().await?;
1451 if self.resource_halt()? {
1452 return Ok(());
1453 }
1454 self.state.save()?;
1455 return Ok(());
1456 }
1457 if let Err(e) = self.prep().await {
1458 if e.is::<crate::repo_lock::ResourceBlocked>() {
1459 return Err(e);
1460 }
1461 self.state.status = RunStatus::Blocked;
1462 self.state
1463 .prep_failure
1464 .get_or_insert_with(|| format!("{e:#}"));
1465 self.state
1466 .event("prep", format!("preparation stopped: {e:#}"));
1467 self.state.save()?;
1468 return Err(e);
1469 }
1470 if self.park_here()? {
1471 return Ok(());
1472 }
1473 self.advise().await?;
1474 if self.park_here()? {
1475 return Ok(());
1476 }
1477 self.implement().await?;
1478 if self.park_here()? {
1479 return Ok(());
1480 }
1481 if self.state.status == RunStatus::VerifiedNoop {
1485 return Ok(());
1486 }
1487 self.judge().await?;
1488 if self.park_here()? {
1489 return Ok(());
1490 }
1491 self.deliberate().await?;
1492 if self.park_here()? {
1493 return Ok(());
1494 }
1495 self.vote().await?;
1496 if self.park_here()? {
1497 return Ok(());
1498 }
1499 self.tally()?;
1500 if self.state.status == RunStatus::Stalled {
1505 self.state.save()?;
1509 return Ok(());
1510 }
1511 self.finish_after_tally().await?;
1512 Ok(())
1513 }
1514
1515 fn stop_for_resource(&mut self) -> Result<()> {
1520 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1522 {
1523 self.state.status = RunStatus::Blocked;
1524 }
1525 self.state.save()
1526 }
1527
1528 fn resource_halt(&mut self) -> Result<bool> {
1530 if self.state.resource_block.is_none() {
1531 return Ok(false);
1532 }
1533 self.stop_for_resource()?;
1534 Ok(true)
1535 }
1536
1537 fn park_here(&mut self) -> Result<bool> {
1544 let parking = self.pause.parked() || self.interrupt.parked();
1545 if !parking && self.state.resource_block.is_some() {
1546 self.stop_for_resource()?;
1547 return Ok(true);
1548 }
1549 if !self.pause.parked() && !self.interrupt.parked() {
1554 return Ok(false);
1555 }
1556 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1557 let why = match cause.as_ref().map(ParkCause::label) {
1558 Some(reason) => format!(
1559 "parked after `{}` ({reason}) — resume to carry on from here",
1560 self.state.status.as_str()
1561 ),
1562 None => format!(
1563 "parked after `{}` — resume to carry on from here",
1564 self.state.status.as_str()
1565 ),
1566 };
1567 self.state.event("park", why);
1568 self.state
1569 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1570 self.state.parked = true;
1571 self.state.save()?;
1572 Ok(true)
1573 }
1574
1575 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1577 if self.state.prep_failure.is_none() {
1578 return Ok(());
1579 }
1580 let failed_at = self
1581 .state
1582 .events
1583 .iter()
1584 .rev()
1585 .find(|event| event.node == "prep")
1586 .map_or(self.state.updated_at, |event| event.at);
1587 anyhow::ensure!(
1588 approval_at > failed_at,
1589 "preparation retry requires a new explicit release after the failure"
1590 );
1591 self.state.prep_failure = None;
1592 self.state.prep_complete = Some(false);
1593 self.state
1594 .event("prep-retry", "operator approved retry of preparation");
1595 Ok(())
1596 }
1597
1598 pub fn on_pause(&mut self, pause: Pause) {
1600 self.pause = pause;
1601 }
1602
1603 pub fn watch_interrupt(&mut self, pause: Pause) {
1609 self.interrupt = pause;
1610 }
1611
1612 fn settle_questions(&mut self) {
1632 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1633 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1634 }
1635 }
1636
1637 async fn finish_after_tally(&mut self) -> Result<()> {
1640 self.graft().await?;
1643 if self.resource_halt()? {
1644 return Ok(());
1645 }
1646 self.fold_losers().await?;
1647 self.sync_to_base().await?;
1652 if self.resource_halt()? {
1653 return Ok(());
1654 }
1655 if self.state.status == RunStatus::AlreadyInBase {
1656 return Ok(());
1657 }
1658 self.review_loop().await?;
1659 if self.resource_halt()? {
1660 return Ok(());
1661 }
1662 self.sync_to_base().await?;
1663 if self.resource_halt()? {
1664 return Ok(());
1665 }
1666 if self.state.status == RunStatus::AlreadyInBase {
1667 return Ok(());
1668 }
1669 self.gate().await?;
1670 self.merge().await?;
1671 if self.resource_halt()? {
1672 return Ok(());
1673 }
1674 self.state.save()?;
1675 Ok(())
1676 }
1677
1678 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1685 let result = match ensure_setup_config(&mut self.state, repo).await {
1686 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1687 Err(e) => Err(e),
1688 };
1689 if let Err(e) = result {
1690 git::worktree_remove_clean(repo, wt).await.ok();
1691 if in_prep {
1692 self.rollback_prep(repo).await;
1693 }
1694 self.state.status = RunStatus::Blocked;
1695 self.state.event(
1696 "setup",
1697 format!("worktree setup failed in {}: {e:#}", wt.display()),
1698 );
1699 self.state.save()?;
1700 return Err(e);
1701 }
1702 Ok(())
1703 }
1704
1705 async fn rollback_prep(&mut self, repo: &Path) {
1707 let root = self.state.worktree_root();
1708 for c in std::mem::take(&mut self.state.candidates) {
1709 if matches!(
1710 git::worktree_remove_clean(repo, &c.worktree).await,
1711 Ok(true)
1712 ) {
1713 git::branch_delete(repo, &c.branch).await.ok();
1714 } else {
1715 self.state.candidates.push(c);
1716 }
1717 }
1718 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1719 for j in 1..=seats {
1720 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1721 .await
1722 .ok();
1723 }
1724 for k in 1..=self.state.config.graph.advisors {
1725 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1726 .await
1727 .ok();
1728 }
1729 }
1730
1731 async fn prep(&mut self) -> Result<()> {
1732 if let Some(reason) = &self.state.prep_failure {
1733 bail!(
1734 "candidate preparation previously failed: {reason}; correct the source before retrying"
1735 );
1736 }
1737 if self
1738 .state
1739 .prep_complete
1740 .unwrap_or(!self.state.candidates.is_empty())
1741 {
1742 return Ok(());
1743 }
1744 let repo = self.state.repo.clone();
1745 let _coordination = self.coordinate(&repo).await?;
1748 let base = self.state.base_commit.clone();
1749 let plan = self.state.sources.plan(&repo, &base).await?;
1750 self.rollback_prep(&repo).await;
1751 if !self.state.candidates.is_empty() {
1752 bail!(
1753 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1754 );
1755 }
1756 self.state.prep_complete = Some(false);
1757 self.state.status = RunStatus::Prep;
1758 self.state.save()?;
1759 let repo = self.state.repo.clone();
1760 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1761 let root = self.state.worktree_root();
1762 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1763
1764 let hooks_dir = self.state.dir().join("hooks");
1767 if self.state.config.blind.commit_msg_hook {
1768 std::fs::create_dir_all(&hooks_dir)
1769 .with_context(|| format!("create {}", hooks_dir.display()))?;
1770 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1771 let path = hooks_dir.join("commit-msg");
1772 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1773 make_executable(&path)?;
1774 git::acquire_worktree_config(&repo).await?;
1782 self.state.enabled_worktree_config = true;
1783 }
1784
1785 ensure_setup_config(&mut self.state, &repo).await?;
1788
1789 for (index, (spec, label)) in self
1790 .roles
1791 .implementers
1792 .clone()
1793 .into_iter()
1794 .zip(labels)
1795 .enumerate()
1796 {
1797 let branch = self.state.branch_for(label);
1798 let worktree = root.join(format!("cand-{label}"));
1799 self.state.event(
1803 "prep-intent",
1804 serde_json::json!({
1805 "branch": branch,
1806 "worktree": worktree,
1807 "start": start,
1808 "picks": plan.picks,
1809 })
1810 .to_string(),
1811 );
1812 self.state.save()?;
1813 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1814 if self.state.config.blind.commit_msg_hook {
1815 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1816 }
1817 git::local_exclude(&worktree, "/.magi/").await?;
1818 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1819 git::worktree_remove_clean(&repo, &worktree).await.ok();
1822 git::branch_delete(&repo, &branch).await.ok();
1823 self.rollback_prep(&repo).await;
1824 self.state.status = RunStatus::Blocked;
1825 self.state
1826 .event("prep", format!("worktree setup failed: {e:#}"));
1827 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1828 self.state.save()?;
1829 return Err(e);
1830 }
1831 self.state.candidates.push(Candidate {
1832 index,
1833 label,
1834 agent: spec.id.clone(),
1835 branch,
1836 worktree,
1837 summary: String::new(),
1838 stat: String::new(),
1839 files: 0,
1840 commits: 0,
1841 empty: false,
1842 failed: None,
1843 verified_noop: None,
1844 duration_ms: 0,
1845 folded: false,
1846 });
1847 self.state.save()?;
1848 for pick in &plan.picks {
1849 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1850 {
1851 self.state.status = RunStatus::Blocked;
1852 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1853 self.state
1854 .event("prep", format!("cannot apply referenced commit: {e}"));
1855 self.state.save()?;
1856 return Err(e);
1857 }
1858 }
1859 }
1860
1861 for j in 1..=self.roles.judges.len() {
1862 let wt = root.join(format!("judge-{j}"));
1863 if !wt.exists() {
1864 git::worktree_add_detached(&repo, &wt, &base).await?;
1865 self.setup_seat_worktree(&repo, &wt, true).await?;
1866 }
1867 }
1868
1869 if self.state.config.graph.advise {
1877 for k in 1..=self.state.config.graph.advisors {
1878 let wt = root.join(format!("advisor-{k}"));
1879 if !wt.exists() {
1880 git::worktree_add_detached(&repo, &wt, &base).await?;
1881 self.setup_seat_worktree(&repo, &wt, true).await?;
1882 }
1883 }
1884 }
1885
1886 let authors: Vec<&str> = self
1891 .roles
1892 .implementers
1893 .iter()
1894 .map(|a| a.id.as_str())
1895 .collect();
1896 let overlap: Vec<String> = self
1897 .roles
1898 .judges
1899 .iter()
1900 .enumerate()
1901 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1902 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1903 .collect();
1904 if !overlap.is_empty() {
1905 let note = format!(
1906 "{} also authored a candidate; blind, but the panel is less \
1907 independent than {} distinct agents would be",
1908 overlap.join(", "),
1909 self.roles.judges.len()
1910 );
1911 self.state.event("prep", note);
1912 }
1913
1914 self.state.event(
1915 "prep",
1916 format!(
1917 "{} candidates, {} judges, base {} ({})",
1918 self.state.candidates.len(),
1919 self.roles.judges.len(),
1920 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1921 self.state.base_branch
1922 ),
1923 );
1924 self.state.prep_complete = Some(true);
1925 self.state.status = RunStatus::Implementing;
1926 self.state.save()?;
1927 Ok(())
1928 }
1929
1930 async fn advise(&mut self) -> Result<()> {
1963 let implement_untouched = self
1964 .state
1965 .candidates
1966 .iter()
1967 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1968 if !self.state.config.graph.advise || self.state.advise_attempted {
1969 return Ok(());
1970 }
1971 if !implement_untouched {
1972 self.state.event(
1973 "advise",
1974 "skipping the design-deliberation stage: at least one \
1975 candidate already shows implementation progress, so this \
1976 run is past the point the stage exists to run before"
1977 .to_owned(),
1978 );
1979 self.state.advise_attempted = true;
1980 self.state.save()?;
1981 return Ok(());
1982 }
1983 let run_id = self.state.id.clone();
1984 let prompts = self.state.config.prompts.clone();
1985 let instruction = self.state.instruction.clone();
1986 let language = self.state.config.graph.language.clone();
1987 let root = self.state.worktree_root();
1988 let n = self.state.config.graph.advisors;
1989 let where_recorded = self.state.dir().join("run.json");
1990
1991 let seats = match self.state.config.advisors() {
1992 Ok(seats) if !seats.is_empty() => seats,
1993 Ok(_) => {
1994 self.state.event(
1995 "advise",
1996 format!(
1997 "[graph] advisors is 0; skipping the design-deliberation \
1998 stage and continuing without a synthesis brief (see {})",
1999 where_recorded.display()
2000 ),
2001 );
2002 self.state.advise_attempted = true;
2003 self.state.save()?;
2004 return Ok(());
2005 }
2006 Err(e) => {
2007 self.state.event(
2008 "advise",
2009 format!(
2010 "could not resolve advisor seats ({e:#}); continuing \
2011 without a design-deliberation brief (see {})",
2012 where_recorded.display()
2013 ),
2014 );
2015 self.state.advise_attempted = true;
2016 self.state.save()?;
2017 return Ok(());
2018 }
2019 };
2020
2021 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2022 let artifacts = agent::artifacts_dir(&self.state.dir());
2023 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2024
2025 let mut jobs = Vec::new();
2026 for (i, spec) in seats.iter().cloned().enumerate() {
2027 let seat_key = format!("advisor-{}", i + 1);
2028 let seat = self.seat(&seat_key, &spec.id);
2029 jobs.push(SeatJob {
2030 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2031 spec,
2032 seat,
2033 cwd: worktrees[i % worktrees.len()].clone(),
2034 timeout,
2035 allow_write: false,
2036 sessions: false,
2037 artifacts: artifacts.clone(),
2038 stem: seat_key,
2039 handover: None,
2040 });
2041 }
2042
2043 self.state.event(
2044 "advise",
2045 format!(
2046 "{} advisor seat(s) sketching a design in parallel",
2047 jobs.len()
2048 ),
2049 );
2050 let mut quota_losses = Vec::new();
2051 let cache = self.state.config.cache_dir();
2052 let ctx = WaveCtx {
2053 carry_seats: false,
2054 pauses: Some((&self.pause, &self.interrupt)),
2055 run: &run_id,
2056 node: "advise",
2057 prompts: &prompts,
2058 cache: cache.as_deref(),
2059 round: None,
2060 };
2061 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2062 let results = ask_json_wave::<Proposal>(
2063 jobs,
2064 Arc::clone(&self.sem),
2065 self.state.config.graph.retries,
2066 &advisor_roster,
2067 &ctx,
2068 &mut quota_losses,
2069 &mut self.state,
2070 &|p: &Proposal| p.validate(),
2071 )
2072 .await;
2073 if self.state.resource_block.is_some() {
2074 return self.stop_for_resource();
2075 }
2076 self.state.quota.extend(quota_losses);
2077
2078 let mut records = Vec::with_capacity(results.len());
2079 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2080 let agent_id = seat.agent.clone();
2081 self.state.seats.insert(seat.key.clone(), seat);
2082 match res {
2083 Ok((proposal, out)) => {
2084 self.state
2085 .event("advise", format!("advisor-{} proposed a design", i + 1));
2086 records.push(advise::AdvisorRecord::proposed(
2087 i + 1,
2088 agent_id,
2089 proposal,
2090 out.duration_ms,
2091 ));
2092 }
2093 Err(e) => {
2094 self.state.event(
2095 "advise",
2096 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2097 );
2098 records.push(advise::AdvisorRecord::failed(
2099 i + 1,
2100 agent_id,
2101 e.to_string(),
2102 ));
2103 }
2104 }
2105 }
2106
2107 let mut advice = advise::Advice {
2108 records,
2109 synthesis: None,
2110 };
2111 if advice.proposals().is_empty() {
2112 self.state.event(
2113 "advise",
2114 "no advisor produced a usable proposal; continuing without a \
2115 synthesis brief"
2116 .to_owned(),
2117 );
2118 } else {
2119 match self
2120 .synthesize_brief(
2121 &advice,
2122 &instruction,
2123 &language,
2124 &worktrees[0],
2125 &artifacts,
2126 &run_id,
2127 &prompts,
2128 cache.as_deref(),
2129 )
2130 .await
2131 {
2132 Ok(Some(text)) => {
2133 self.state.event(
2134 "advise",
2135 "synthesized a design brief for the implementer".to_owned(),
2136 );
2137 advice.synthesis = Some(text);
2138 }
2139 Ok(None) => {
2140 self.state.event(
2141 "advise",
2142 "the synthesis seat produced nothing usable; continuing \
2143 without a design brief"
2144 .to_owned(),
2145 );
2146 }
2147 Err(e) => {
2148 if e.is::<crate::repo_lock::ResourceBlocked>() {
2149 return Err(e);
2150 }
2151 self.state.event(
2152 "advise",
2153 format!("could not synthesize a design brief: {e:#}"),
2154 );
2155 }
2156 }
2157 }
2158 advise::apply_reflection(&mut advice);
2159
2160 self.state.advice = Some(advice);
2161 self.state.advise_attempted = true;
2162 self.state.save()?;
2163 Ok(())
2164 }
2165
2166 #[allow(clippy::too_many_arguments)]
2178 async fn synthesize_brief(
2179 &mut self,
2180 advice: &advise::Advice,
2181 instruction: &str,
2182 language: &str,
2183 cwd: &Path,
2184 artifacts: &Path,
2185 run_id: &str,
2186 prompts: &Prompts,
2187 cache: Option<&Path>,
2188 ) -> Result<Option<String>> {
2189 let chain = agent::pick_chain(
2190 &self.state.config.agents,
2191 self.state.config.roles.synthesizer.as_ref(),
2192 &agent::installed,
2193 "synthesizer",
2194 )?;
2195 let proposals = advice.proposals();
2196 let mut prompt = prompt::with_overlay(
2197 prompt::synthesize_brief(instruction, &proposals, language),
2198 prompts.overlay("advise"),
2199 );
2200 if cache.is_some() {
2201 prompt.push('\n');
2206 prompt.push_str(&prompt::build_cache_note("advise", false));
2207 }
2208 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2209 let mut last = None;
2214 for (n, spec) in chain.iter().enumerate() {
2215 if n > 0 {
2216 self.state
2217 .event("advise", format!("synthesis falling back to {}", spec.id));
2218 }
2219 let mut seat = self.seat("advise-synthesis", &spec.id);
2220 let wait = self.repository_wait();
2221 let outcome = wait
2222 .scope(agent::invoke(
2223 spec,
2224 &mut seat,
2225 &Invocation {
2226 scope: crate::agent::InvocationScope::Repository,
2227 coordination: self.state.config.coordination.as_ref(),
2228 fetched_base: None,
2229 cwd,
2230 prompt: &prompt,
2231 timeout,
2232 allow_write: false,
2233 unsandboxed: false,
2234 sessions: false,
2235 artifacts,
2236 stem: &if n == 0 {
2237 "advise-synthesis".to_owned()
2238 } else {
2239 format!("advise-synthesis-{}", spec.id)
2240 },
2241 run: run_id,
2242 node: "advise",
2243 cache_dir: None,
2244 attachments: &[],
2245 writable: &[],
2246 },
2247 ))
2248 .await;
2249 if outcome.is_ok() {
2250 self.state.seats.insert(seat.key.clone(), seat);
2251 }
2252 let advance = agent::chain_advances(&outcome);
2253 last = Some(outcome);
2254 if !advance {
2255 break;
2256 }
2257 }
2258 let out = last.expect("a chain holds at least one agent")?;
2261 if !out.usable() {
2262 return Ok(None);
2263 }
2264 let text =
2265 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2266 Ok((!text.trim().is_empty()).then_some(text))
2267 }
2268
2269 async fn implement(&mut self) -> Result<()> {
2272 self.persist_candidates().await?;
2273 let run_id = self.state.id.clone();
2278 let prompts = self.state.config.prompts.clone();
2279 let todo: Vec<usize> = self
2280 .state
2281 .candidates
2282 .iter()
2283 .enumerate()
2284 .filter(|(_, c)| {
2285 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2286 && c.failed.is_none()
2287 && !c.empty
2288 })
2289 .map(|(i, _)| i)
2290 .collect();
2291 if todo.is_empty() {
2292 return self.after_implement();
2293 }
2294 self.state.status = RunStatus::Implementing;
2295
2296 let language = self.state.config.graph.language.clone();
2297 let github_language = self.state.config.graph.github_language.clone();
2298 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2299 let sessions = self.state.config.graph.sessions;
2300 let artifacts = agent::artifacts_dir(&self.state.dir());
2301 let brief = self
2305 .state
2306 .advice
2307 .as_ref()
2308 .and_then(|a| a.synthesis.as_deref())
2309 .map(str::to_owned);
2310 let attachments = self.state.attachments.clone();
2311
2312 let mut jobs = Vec::new();
2313 for &i in &todo {
2314 let (index, label, worktree) = {
2315 let c = &self.state.candidates[i];
2316 (c.index, c.label, c.worktree.clone())
2317 };
2318 let spec = self.roles.implementers[index].clone();
2319 let seat_key = format!("impl-{label}");
2320 let seat = self.seat(&seat_key, &spec.id);
2321 let instruction = seeded_instruction(&self.state);
2322 jobs.push(SeatJob {
2323 spec,
2324 seat,
2325 prompt: prompt::implement_in(
2326 &instruction,
2327 &worktree.to_string_lossy(),
2328 &language,
2329 &github_language,
2330 brief.as_deref(),
2331 &attachments,
2332 ),
2333 cwd: worktree,
2334 timeout,
2335 allow_write: true,
2336 sessions,
2337 artifacts: artifacts.clone(),
2338 stem: format!("impl-{label}"),
2339 handover: None,
2340 });
2341 }
2342
2343 self.state.event(
2344 "implement",
2345 format!("{} candidates in parallel", jobs.len()),
2346 );
2347 let mut sent = jobs.clone();
2353 let cache = self.state.config.cache_dir();
2354 let ctx = WaveCtx {
2355 carry_seats: false,
2356 pauses: Some((&self.pause, &self.interrupt)),
2357 run: &run_id,
2358 node: "implement",
2359 prompts: &prompts,
2360 cache: cache.as_deref(),
2361 round: None,
2362 };
2363 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2364 if !wave_blocked(&results) {
2365 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2366 .await;
2367 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2368 .await?;
2369 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2370 .await;
2371 }
2372 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2373
2374 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2375 && results
2376 .iter()
2377 .any(|(_, _, out)| FailClass::of(out).is_some());
2378 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2379 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2380 if self
2384 .state
2385 .candidate_preservation
2386 .contains_key(&self.state.candidates[i].label)
2387 {
2388 self.state.seats.insert(seat.key.clone(), seat);
2389 }
2390 continue;
2391 }
2392 let seat_key = seat.key.clone();
2393 let agent = seat.agent.clone();
2402 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2403 self.state.seats.insert(seat.key.clone(), seat);
2404 let label = self.state.candidates[i].label;
2405 self.state.interrupted_implementations.remove(&label);
2406
2407 let (summary, duration, failed, verified_claim) = match out {
2408 AgentOutcome::Ok(o) => {
2409 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2410 let failed = (!o.usable()).then(|| {
2411 if o.timed_out {
2412 "agent timed out".to_owned()
2413 } else {
2414 format!(
2415 "agent exited with {}",
2416 crate::notices::exit_status(o.exit_code)
2417 )
2418 }
2419 });
2420 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2421 (text, o.duration_ms, failed, verified_claim)
2422 }
2423 AgentOutcome::Dropped(o) => {
2429 let why = o
2430 .dropped
2431 .as_ref()
2432 .map(|d| d.why.as_str())
2433 .unwrap_or("the CLI ended the stream without delivering its answer");
2434 (
2435 String::new(),
2436 o.duration_ms,
2437 Some(format!("the CLI dropped the stream ({why})")),
2438 None,
2439 )
2440 }
2441 AgentOutcome::Quota(o) => {
2442 self.state.quota.push(QuotaLoss {
2443 seat: seat_key,
2444 node: "implement".to_owned(),
2445 at: Timestamp::now(),
2446 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2447 });
2448 (
2449 String::new(),
2450 o.duration_ms,
2451 Some("rate limited (quota); produced no change".to_owned()),
2452 None,
2453 )
2454 }
2455 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2456 (String::new(), 0, Some(e), None)
2457 }
2458 };
2459
2460 let c = &mut self.state.candidates[i];
2463 if !exhausted_the_fallback_chain {
2464 c.agent = agent;
2465 }
2466 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2467 c.duration_ms = duration;
2468 c.failed = failed;
2469 c.verified_noop = verified_claim;
2470 self.state
2471 .candidate_preservation
2472 .insert(label, "awaiting persistence".to_owned());
2473 }
2474 self.state.save()?;
2475 self.persist_candidates().await?;
2476
2477 if resource_blocked {
2478 return self.stop_for_resource();
2479 }
2480 if selected_failed {
2481 self.state.status = RunStatus::Blocked;
2482 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2483 self.state.save()?;
2484 bail!(
2485 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2486 );
2487 }
2488 self.after_implement()
2489 }
2490
2491 async fn persist_candidates(&mut self) -> Result<()> {
2493 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2494 for label in labels {
2495 let i = self
2496 .state
2497 .candidates
2498 .iter()
2499 .position(|c| c.label == label)
2500 .context("pending candidate is missing")?;
2501 let c = self.state.candidates[i].clone();
2502 let interrupted = self.state.interrupted_implementations.contains(&label);
2503 let result = async {
2504 let wait = crate::repo_lock::WaitPolicy::new(
2505 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2506 || false,
2507 );
2508 let _coordination =
2509 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2510 if let Err(e) = git::snapshot_uncommitted(
2513 &c.worktree,
2514 &self
2515 .state
2516 .dir()
2517 .join("artifacts")
2518 .join(format!("candidate-{label}-recovery")),
2519 )
2520 .await
2521 {
2522 self.state.event(
2523 "preserve",
2524 format!("candidate {label} snapshot incomplete: {e:#}"),
2525 );
2526 }
2527 let rescued = git::rescue_commit(
2528 &c.worktree,
2529 &format!("magi: candidate {label} (uncommitted work)"),
2530 )
2531 .await?;
2532 self.state.note_withheld("implement", &rescued.withheld);
2533 let base = &self.state.base_commit;
2534 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2535 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2536 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2537 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2538 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2539 if c.empty && commits > 0 && !patch.trim().is_empty()
2540 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2541 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2542 || self.state.tally.is_some())
2543 {
2544 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2548 }
2549 let c = &mut self.state.candidates[i];
2550 c.commits = commits;
2551 c.stat = stat;
2552 c.files = files;
2553 let no_changes = commits == 0 || patch.trim().is_empty();
2554 c.empty = no_changes && !interrupted;
2555 if interrupted && no_changes {
2556 c.failed = None;
2557 c.verified_noop = None;
2558 }
2559 if !c.empty {
2560 c.failed = None;
2561 c.verified_noop = None;
2562 }
2563 self.state.event(
2564 "preserve",
2565 format!("candidate {label}: {files} files, {commits} commits"),
2566 );
2567 Ok::<_, anyhow::Error>(())
2568 }
2569 .await;
2570 match result {
2571 Ok(()) => {
2572 self.state.candidate_preservation.remove(&label);
2573 if !interrupted || self.state.candidates[i].files > 0 {
2574 self.state.interrupted_implementations.remove(&label);
2575 }
2576 }
2577 Err(e) => {
2578 let reason = format!("candidate {label} preservation failed: {e:#}");
2579 self.state.event("preserve", &reason);
2580 self.state.candidate_preservation.insert(label, reason);
2581 }
2582 }
2583 if self.state.candidate_preservation.is_empty()
2584 && self.state.status == RunStatus::Blocked
2585 && self.state.candidates.iter().all(|c| c.failed.is_none())
2586 {
2587 self.state.status = RunStatus::Implementing;
2591 self.state.event(
2592 "preserve",
2593 "candidate persistence recovered; implementation can continue",
2594 );
2595 }
2596 self.state.save()?;
2597 }
2598 if !self.state.candidate_preservation.is_empty() {
2599 self.state.status = RunStatus::Blocked;
2600 self.state.save()?;
2601 bail!(
2602 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2603 );
2604 }
2605 Ok(())
2606 }
2607
2608 async fn resume_undelivered(
2636 &mut self,
2637 results: &mut [(usize, SeatState, AgentOutcome)],
2638 sent: &[SeatJob],
2639 prompts: &Prompts,
2640 run_id: &str,
2641 ) {
2642 for (wi, seat, out) in results.iter_mut() {
2643 let Some(dropped) = (match &*out {
2644 AgentOutcome::Dropped(o) => o.dropped.clone(),
2645 _ => None,
2646 }) else {
2647 continue;
2648 };
2649 let Some(job) = sent.get(*wi) else { continue };
2650 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2652 self.state.event(
2653 "implement",
2654 format!(
2655 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2656 work is in the tree",
2657 seat.key, dropped.output_tokens, dropped.why
2658 ),
2659 );
2660 continue;
2661 }
2662 if !has_context(&job.spec, seat, job.sessions) {
2670 self.state.event(
2671 "implement",
2672 format!(
2673 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2674 is no session left to resume",
2675 seat.key, dropped.output_tokens, dropped.why
2676 ),
2677 );
2678 continue;
2679 }
2680 self.state.event(
2681 "implement",
2682 format!(
2683 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2684 conversation",
2685 seat.key, dropped.output_tokens, dropped.why
2686 ),
2687 );
2688 let mut retry = job.clone();
2689 retry.seat = seat.clone();
2690 retry.prompt = prompt::resume_after_drop(&dropped.why);
2691 retry.timeout = retry_budget(job.timeout, true);
2692 retry.stem = format!("{}-resume", job.stem);
2693 let cache = self.state.config.cache_dir();
2694 let ctx = WaveCtx {
2695 carry_seats: false,
2696 pauses: Some((&self.pause, &self.interrupt)),
2697 run: run_id,
2698 node: "implement",
2699 prompts,
2700 cache: cache.as_deref(),
2701 round: None,
2702 };
2703 let (resumed_seat, resumed) =
2704 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2705 *seat = resumed_seat;
2706 *out = resumed;
2707 }
2708 }
2709
2710 async fn resume_seat_handovers(
2775 &mut self,
2776 results: &mut [(usize, SeatState, AgentOutcome)],
2777 sent: &mut [SeatJob],
2778 prompts: &Prompts,
2779 run_id: &str,
2780 ) -> Result<()> {
2781 let instruction = seeded_instruction(&self.state);
2782 let language = self.state.config.graph.language.clone();
2783 let github_language = self.state.config.graph.github_language.clone();
2784 let brief = self
2785 .state
2786 .advice
2787 .as_ref()
2788 .and_then(|a| a.synthesis.as_deref())
2789 .map(str::to_owned);
2790 let attachments = self.state.attachments.clone();
2791 for (wi, seat, out) in results.iter_mut() {
2792 let others: BTreeSet<String> = sent
2795 .iter()
2796 .enumerate()
2797 .filter(|(j, _)| j != wi)
2798 .map(|(_, j)| j.spec.id.clone())
2799 .collect();
2800 let Some(job) = sent.get_mut(*wi) else {
2801 continue;
2802 };
2803 let start = self
2808 .roles
2809 .implementer_roster
2810 .iter()
2811 .position(|s| s.id == job.spec.id)
2812 .unwrap_or(0);
2813 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2814 let mut fallback_attempt = 0usize;
2815 let mut prev: Option<FailClass> = None;
2816 while let Some(cur) = FailClass::of(&*out) {
2817 if !should_hand_over(prev.as_ref(), &cur) {
2818 break;
2819 }
2820 let Some(next) =
2821 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2822 .cloned()
2823 else {
2824 break;
2825 };
2826 tried.insert(next.id.clone());
2827 fallback_attempt += 1;
2828
2829 let coordination = self.coordinate(&job.cwd).await?;
2830 if let Ok(r) = git::rescue_commit(
2831 &job.cwd,
2832 &format!(
2833 "magi: candidate {} (uncommitted work before {} fallback)",
2834 seat.key,
2835 if cur == FailClass::Quota {
2836 "quota"
2837 } else {
2838 "handover"
2839 }
2840 ),
2841 )
2842 .await
2843 {
2844 self.state.note_withheld("implement", &r.withheld);
2845 }
2846 drop(coordination);
2847
2848 record_handover(
2849 &mut self.state,
2850 "implement",
2851 &seat.key,
2852 &seat.agent,
2853 &next.id,
2854 &cur,
2855 &fail_reason(&*out),
2856 );
2857 prev = Some(cur.clone());
2858
2859 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2860 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2861 job.spec = next.clone();
2869 let mut retry = job.clone();
2870 retry.seat = new_seat;
2871 retry.prompt = prompt::implement_in(
2872 &instruction,
2873 &job.cwd.to_string_lossy(),
2874 &language,
2875 &github_language,
2876 brief.as_deref(),
2877 &attachments,
2878 );
2879 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2880 let cache = self.state.config.cache_dir();
2881 let ctx = WaveCtx {
2882 carry_seats: false,
2883 pauses: Some((&self.pause, &self.interrupt)),
2884 run: run_id,
2885 node: "implement",
2886 prompts,
2887 cache: cache.as_deref(),
2888 round: None,
2889 };
2890 let (fallback_seat, fallback_out) = run_one(
2891 retry,
2892 Arc::clone(&self.sem),
2893 &ctx,
2894 &mut self.state,
2895 fallback_attempt,
2896 )
2897 .await;
2898 *seat = fallback_seat;
2899 *out = fallback_out;
2900 }
2901 }
2902 Ok(())
2903 }
2904
2905 async fn resume_unconfirmed_commands(
2929 &mut self,
2930 results: &mut [(usize, SeatState, AgentOutcome)],
2931 sent: &[SeatJob],
2932 prompts: &Prompts,
2933 run_id: &str,
2934 ) {
2935 for (wi, seat, out) in results.iter_mut() {
2936 let AgentOutcome::Ok(o) = &*out else {
2937 continue;
2938 };
2939 if !has_unconfirmed_command(&o.commands) {
2940 continue;
2941 }
2942 let Some(job) = sent.get(*wi) else { continue };
2943 if !has_context(&job.spec, seat, job.sessions) {
2944 self.state.event(
2945 "implement",
2946 format!(
2947 "{}: the reply named a command whose own CLI never confirmed the exit \
2948 status of, but there is no session left to resume",
2949 seat.key
2950 ),
2951 );
2952 continue;
2953 }
2954 self.state.event(
2955 "implement",
2956 format!(
2957 "{}: the reply named a command whose own CLI never confirmed the exit \
2958 status of; resuming the conversation",
2959 seat.key
2960 ),
2961 );
2962 let mut retry = job.clone();
2963 retry.seat = seat.clone();
2964 retry.prompt = prompt::resume_incomplete(
2965 "a command in your last reply had no confirmed exit status",
2966 );
2967 retry.timeout = retry_budget(job.timeout, true);
2968 retry.stem = format!("{}-confirm", job.stem);
2969 let cache = self.state.config.cache_dir();
2970 let ctx = WaveCtx {
2971 carry_seats: false,
2972 pauses: Some((&self.pause, &self.interrupt)),
2973 run: run_id,
2974 node: "implement",
2975 prompts,
2976 cache: cache.as_deref(),
2977 round: None,
2978 };
2979 let (resumed_seat, resumed) =
2980 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2981 *seat = resumed_seat;
2982 *out = resumed;
2983 }
2984 }
2985
2986 async fn continue_fix_report(
3007 &mut self,
3008 mut seat: SeatState,
3009 parse_err: String,
3010 job: &SeatJob,
3011 prompts: &Prompts,
3012 run_id: &str,
3013 round: usize,
3014 ) -> (
3015 SeatState,
3016 Option<FixReport>,
3017 Option<String>,
3018 ContinuationRecord,
3019 ) {
3020 let mut last_err = parse_err;
3021 let mut cumulative_wait_ms = 0u64;
3022 let mut attempts = 0usize;
3023 loop {
3024 if !has_context(&job.spec, &seat, job.sessions) {
3025 self.state.event(
3026 "fix",
3027 format!(
3028 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3029 session left to resume into"
3030 ),
3031 );
3032 let outcome = if attempts == 0 {
3033 ContinuationOutcome::NoSession
3034 } else {
3035 ContinuationOutcome::Exhausted
3036 };
3037 return (
3038 seat,
3039 None,
3040 Some(format!("unparsable fix report: {last_err}")),
3041 ContinuationRecord {
3042 attempts,
3043 cumulative_wait_ms,
3044 outcome,
3045 },
3046 );
3047 }
3048 if attempts >= MAX_FIX_CONTINUATIONS {
3049 self.state.event(
3050 "fix",
3051 format!(
3052 "round {round}: fixer's reply still had no adoption report after \
3053 {attempts} continuation(s) ({last_err}); giving up"
3054 ),
3055 );
3056 return (
3057 seat,
3058 None,
3059 Some(format!(
3060 "unparsable fix report after {attempts} continuation(s): {last_err}"
3061 )),
3062 ContinuationRecord {
3063 attempts,
3064 cumulative_wait_ms,
3065 outcome: ContinuationOutcome::Exhausted,
3066 },
3067 );
3068 }
3069 attempts += 1;
3070 self.state.event(
3071 "fix",
3072 format!(
3073 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3074 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3075 ),
3076 );
3077 let mut retry = job.clone();
3078 retry.seat = seat.clone();
3079 retry.prompt = prompt::resume_incomplete(&last_err);
3080 retry.timeout = retry_budget(job.timeout, true);
3081 retry.stem = format!("{}-continue{attempts}", job.stem);
3082 let cache = self.state.config.cache_dir();
3083 let ctx = WaveCtx {
3084 carry_seats: false,
3085 pauses: Some((&self.pause, &self.interrupt)),
3086 run: run_id,
3087 node: "fix",
3088 prompts,
3089 cache: cache.as_deref(),
3090 round: Some(round),
3091 };
3092 let (resumed_seat, resumed_out) = run_one(
3093 retry,
3094 Arc::clone(&self.sem),
3095 &ctx,
3096 &mut self.state,
3097 attempts,
3098 )
3099 .await;
3100 seat = resumed_seat;
3101 match resumed_out {
3102 AgentOutcome::Ok(o) => {
3103 cumulative_wait_ms += o.duration_ms;
3104 match verdict::extract_json::<FixReport>(&o.text) {
3105 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3106 self.state.event(
3107 "fix",
3108 format!(
3109 "round {round}: fixer's adoption report recovered after \
3110 {attempts} continuation(s)"
3111 ),
3112 );
3113 return (
3114 seat,
3115 Some(report),
3116 None,
3117 ContinuationRecord {
3118 attempts,
3119 cumulative_wait_ms,
3120 outcome: ContinuationOutcome::Resumed,
3121 },
3122 );
3123 }
3124 Ok(_) => {
3132 last_err = "the reply parsed, but it reported a command whose own CLI \
3133 never confirmed an exit status"
3134 .to_owned();
3135 }
3136 Err(e) => last_err = e.to_string(),
3137 }
3138 }
3139 AgentOutcome::Quota(o) => {
3140 cumulative_wait_ms += o.duration_ms;
3141 self.state.quota.push(QuotaLoss {
3142 seat: seat.key.clone(),
3143 node: "fix".to_owned(),
3144 at: Timestamp::now(),
3145 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3146 });
3147 self.state.event(
3148 "fix",
3149 format!(
3150 "round {round}: continuation rate limited (quota); not retrying now"
3151 ),
3152 );
3153 return (
3154 seat,
3155 None,
3156 Some("rate limited (quota) while recovering the fix report".to_owned()),
3157 ContinuationRecord {
3158 attempts,
3159 cumulative_wait_ms,
3160 outcome: ContinuationOutcome::QuotaLost,
3161 },
3162 );
3163 }
3164 AgentOutcome::Dropped(o) => {
3165 cumulative_wait_ms += o.duration_ms;
3166 let why = o
3167 .dropped
3168 .as_ref()
3169 .map(|d| d.why.as_str())
3170 .unwrap_or("the CLI ended the stream without delivering its answer");
3171 last_err = format!("the CLI dropped the stream ({why})");
3172 }
3173 AgentOutcome::Failed(e) => last_err = e,
3174 AgentOutcome::ResourceBlocked(e) => {
3177 return (
3178 seat,
3179 None,
3180 Some(format!("the continuation was not started: {e}")),
3181 ContinuationRecord {
3182 attempts: attempts.saturating_sub(1),
3183 cumulative_wait_ms,
3184 outcome: ContinuationOutcome::NoSession,
3185 },
3186 );
3187 }
3188 }
3189 }
3190 }
3191
3192 fn after_implement(&mut self) -> Result<()> {
3193 if !self.state.candidate_preservation.is_empty() {
3194 bail!("candidate preservation incomplete; judging is unsafe");
3195 }
3196 if self.state.leaks.is_empty() {
3198 let cfg = self.state.config.blind.clone();
3199 let mut leaks = Vec::new();
3200 for c in &self.state.candidates {
3201 let Some(patch) =
3202 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3203 else {
3204 continue;
3205 };
3206 leaks.extend(blind::scan(
3207 &format!("candidate {} patch", c.label),
3208 &patch,
3209 &cfg.vendor_tokens,
3210 ));
3211 }
3212 if !leaks.is_empty() {
3213 let summary = leaks
3214 .iter()
3215 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3216 .collect::<Vec<_>>()
3217 .join(", ");
3218 match cfg.on_leak {
3219 LeakPolicy::Fail => {
3220 self.state.status = RunStatus::Failed;
3221 self.state
3222 .event("blind", format!("vendor text in a patch: {summary}"));
3223 self.state.leaks = leaks;
3224 self.state.save()?;
3225 self.settle_questions();
3226 bail!(
3227 "blind.on_leak = \"fail\" and vendor text reached a \
3228 judged patch: {summary}"
3229 );
3230 }
3231 LeakPolicy::Redact => self.state.event(
3232 "blind",
3233 format!("redacting vendor text for judging: {summary}"),
3234 ),
3235 LeakPolicy::Warn => self.state.event(
3236 "blind",
3237 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3238 ),
3239 }
3240 self.state.leaks = leaks;
3241 }
3242 }
3243
3244 if self.state.viable().is_empty() {
3245 if self.state.all_candidates_verified_noop() {
3246 self.state.status = RunStatus::VerifiedNoop;
3257 self.state.save()?;
3258 self.settle_questions();
3259 return Ok(());
3260 }
3261 self.state.status = RunStatus::Failed;
3262 self.state.save()?;
3263 self.settle_questions();
3264 bail!("no candidate produced a change; nothing to judge");
3265 }
3266 self.state.status = RunStatus::Judging;
3267 self.state.save()?;
3268 Ok(())
3269 }
3270
3271 async fn judge(&mut self) -> Result<()> {
3274 let run_id = self.state.id.clone();
3279 let prompts = self.state.config.prompts.clone();
3280 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3281 return Ok(());
3282 }
3283 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3284 if viable.len() == 1 {
3285 self.state.judge_skipped = true;
3292 self.state.event(
3293 "judge",
3294 format!(
3295 "only candidate {} produced a change; judging skipped",
3296 viable[0].label
3297 ),
3298 );
3299 self.state.save()?;
3300 return Ok(());
3301 }
3302 self.state.status = RunStatus::Judging;
3303
3304 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3305 let language = self.state.config.graph.language.clone();
3306 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3307 let sessions = self.state.config.graph.sessions;
3308 let artifacts = agent::artifacts_dir(&self.state.dir());
3309 let root = self.state.worktree_root();
3310 let base_short = short(&self.state.base_commit);
3311
3312 let mut jobs = Vec::new();
3313 let mut orders = Vec::new();
3314 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3315 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3316 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3317 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3318 let seat_key = format!("judge-{}", j + 1);
3319 let seat = self.seat(&seat_key, &spec.id);
3320 jobs.push(SeatJob {
3321 prompt: prompt::judge(
3322 &self.state.instruction,
3323 &views,
3324 self.roles.judges.len(),
3325 &base_short,
3326 &language,
3327 ),
3328 spec,
3329 seat,
3330 cwd: root.join(format!("judge-{}", j + 1)),
3331 timeout,
3332 allow_write: false,
3333 sessions,
3334 artifacts: artifacts.clone(),
3335 stem: format!("judge-{}", j + 1),
3336 handover: None,
3337 });
3338 }
3339
3340 self.state.event(
3341 "judge",
3342 format!(
3343 "{} judges ranking {} candidates blind",
3344 jobs.len(),
3345 viable.len()
3346 ),
3347 );
3348 let labels_for_check = labels.clone();
3349 let mut quota_losses = Vec::new();
3350 let cache = self.state.config.cache_dir();
3351 let ctx = WaveCtx {
3352 carry_seats: false,
3353 pauses: Some((&self.pause, &self.interrupt)),
3354 run: &run_id,
3355 node: "judge",
3356 prompts: &prompts,
3357 cache: cache.as_deref(),
3358 round: None,
3359 };
3360 let results = ask_json_wave::<Ranking>(
3361 jobs,
3362 Arc::clone(&self.sem),
3363 self.state.config.graph.retries,
3364 &self.roles.judge_roster,
3365 &ctx,
3366 &mut quota_losses,
3367 &mut self.state,
3368 &move |r: &Ranking| r.validate(&labels_for_check),
3369 )
3370 .await;
3371 if self.state.resource_block.is_some() {
3372 return self.stop_for_resource();
3373 }
3374 self.state.quota.extend(quota_losses);
3375
3376 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3377 let agent_id = seat.agent.clone();
3378 self.state.seats.insert(seat.key.clone(), seat);
3379 let mut record = Judgement {
3380 judge: j + 1,
3381 seat: format!("judge-{}", j + 1),
3382 agent: agent_id,
3383 ranking: Vec::new(),
3384 reasons: BTreeMap::new(),
3385 confidence: None,
3386 strengths: BTreeMap::new(),
3387 order: orders[j].clone(),
3388 failed: None,
3389 duration_ms: 0,
3390 };
3391 match res {
3392 Ok((ranking, out)) => {
3393 record.ranking = ranking.normalized();
3394 record.reasons = ranking.reasons;
3395 record.strengths = ranking.strengths;
3396 record.confidence = ranking.confidence;
3397 record.duration_ms = out.duration_ms;
3398 self.state.event(
3399 "judge",
3400 format!(
3401 "judge {} ranked {}",
3402 j + 1,
3403 record.ranking.iter().collect::<String>()
3404 ),
3405 );
3406 }
3407 Err(e) => {
3408 record.failed = Some(e.to_string());
3409 self.state
3410 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3411 }
3412 }
3413 self.state.judgements.push(record);
3414 self.state.save()?;
3415 }
3416 Ok(())
3417 }
3418
3419 async fn deliberate(&mut self) -> Result<()> {
3422 let run_id = self.state.id.clone();
3427 let prompts = self.state.config.prompts.clone();
3428 if !self.state.deliberation.is_empty() {
3429 return Ok(());
3430 }
3431 let tops: Vec<char> = self
3432 .state
3433 .judgements
3434 .iter()
3435 .filter_map(|j| j.ranking.first().copied())
3436 .collect();
3437 let rounds = self.state.config.graph.deliberate_rounds;
3438 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3439 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3440 self.state.event(
3441 "deliberate",
3442 format!("judges agreed on {} outright; no deliberation", tops[0]),
3443 );
3444 }
3445 self.state.status = RunStatus::Voting;
3446 self.state.save()?;
3447 return Ok(());
3448 }
3449
3450 self.state.status = RunStatus::Deliberating;
3451 self.state.event(
3452 "deliberate",
3453 format!(
3454 "split: first choices were {} — opening {rounds} round(s)",
3455 tops.iter().collect::<String>()
3456 ),
3457 );
3458
3459 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3460 let language = self.state.config.graph.language.clone();
3461 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3462 let sessions = self.state.config.graph.sessions;
3463 let artifacts = agent::artifacts_dir(&self.state.dir());
3464 let root = self.state.worktree_root();
3465 let base_short = short(&self.state.base_commit);
3466
3467 for round in 1..=rounds {
3471 let mut turns: Vec<DeliberationTurn> = Vec::new();
3472 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3473 if self.state.judgements[j].failed.is_some() {
3474 continue;
3475 }
3476 let seat_key = format!("judge-{}", j + 1);
3477 let spec = self.occupant(&seat_key, spec);
3478 let mut seat = self.seat(&seat_key, &spec.id);
3479 let transcript = self.transcript(&turns, j);
3480 let build = |context: Option<&str>| {
3481 prompt::deliberate(
3482 &self.state.instruction,
3483 context,
3484 &transcript,
3485 round,
3486 rounds,
3487 &language,
3488 )
3489 };
3490 let block = self.candidate_block(&viable, &base_short);
3491 let full = build(Some(&block));
3492 let text = if has_context(&spec, &seat, sessions) {
3493 build(None)
3494 } else {
3495 full.clone()
3496 };
3497 let job = SeatJob {
3498 spec,
3499 seat: seat.clone(),
3500 prompt: text,
3501 cwd: root.join(format!("judge-{}", j + 1)),
3502 timeout,
3503 allow_write: false,
3504 sessions,
3505 artifacts: artifacts.clone(),
3506 stem: format!("delib-{round}-judge-{}", j + 1),
3507 handover: Some(full),
3508 };
3509 let cache = self.state.config.cache_dir();
3510 let ctx = WaveCtx {
3511 carry_seats: false,
3512 pauses: Some((&self.pause, &self.interrupt)),
3513 run: &run_id,
3514 node: "deliberate",
3515 prompts: &prompts,
3516 cache: cache.as_deref(),
3517 round: None,
3518 };
3519 let mut losses = Vec::new();
3523 let mut results = ask_wave_with::<String>(
3524 vec![job],
3525 Arc::clone(&self.sem),
3526 0,
3527 &self.roles.judge_roster,
3528 &ctx,
3529 &mut losses,
3530 &mut self.state,
3531 &|text: &str| {
3532 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3533 },
3534 )
3535 .await;
3536 self.state.quota.extend(losses);
3537 let (updated, res, _) = results.pop().expect("one job in, one result out");
3538 seat = updated;
3539 let agent_id = seat.agent.clone();
3540 self.state.seats.insert(seat.key.clone(), seat);
3541 let body = match res {
3542 Ok((body, _)) => body,
3543 Err(e) => {
3546 self.state
3547 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3548 continue;
3549 }
3550 };
3551 let tentative = verdict::extract_json::<Position>(&body)
3552 .ok()
3553 .and_then(|p| p.tentative)
3554 .and_then(|s| s.trim().chars().next())
3555 .map(|c| c.to_ascii_uppercase());
3556 self.state.event(
3557 "deliberate",
3558 format!(
3559 "round {round}: judge {} now favours {}",
3560 j + 1,
3561 tentative.map_or("—".to_owned(), |c| c.to_string())
3562 ),
3563 );
3564 turns.push(DeliberationTurn {
3565 judge: j + 1,
3566 agent: agent_id,
3567 body: blind::sanitize_prose(&body, &self.state.config.blind),
3568 tentative,
3569 });
3570 }
3571 self.state
3572 .deliberation
3573 .push(DeliberationRound { round, turns });
3574 self.state.save()?;
3575 }
3576
3577 self.state.status = RunStatus::Voting;
3578 self.state.save()?;
3579 Ok(())
3580 }
3581
3582 async fn vote(&mut self) -> Result<()> {
3585 let run_id = self.state.id.clone();
3590 let prompts = self.state.config.prompts.clone();
3591 if !self.state.votes.is_empty() {
3592 return Ok(());
3593 }
3594 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3595 if viable.len() == 1 {
3596 return Ok(());
3597 }
3598 self.state.status = RunStatus::Voting;
3599
3600 let language = self.state.config.graph.language.clone();
3601 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3602 let sessions = self.state.config.graph.sessions;
3603 let artifacts = agent::artifacts_dir(&self.state.dir());
3604 let root = self.state.worktree_root();
3605 let base_short = short(&self.state.base_commit);
3606 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3607
3608 let mut jobs = Vec::new();
3609 let mut seats_at = Vec::new();
3610 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3611 if self
3612 .state
3613 .judgements
3614 .get(j)
3615 .is_some_and(|r| r.failed.is_some())
3616 {
3617 continue;
3618 }
3619 let seat_key = format!("judge-{}", j + 1);
3620 let spec = self.occupant(&seat_key, spec);
3621 let seat = self.seat(&seat_key, &spec.id);
3622 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3623 let text = if has_context(&spec, &seat, sessions) {
3624 prompt::final_vote(&viable, &language)
3625 } else {
3626 full.clone()
3627 };
3628 jobs.push(SeatJob {
3629 spec,
3630 seat,
3631 prompt: text,
3632 cwd: root.join(format!("judge-{}", j + 1)),
3633 timeout,
3634 allow_write: false,
3635 sessions,
3636 artifacts: artifacts.clone(),
3637 stem: format!("vote-judge-{}", j + 1),
3638 handover: Some(full),
3639 });
3640 seats_at.push(j);
3641 }
3642
3643 self.state.event(
3644 "vote",
3645 format!(
3646 "collecting {} final votes one by one, privately",
3647 jobs.len()
3648 ),
3649 );
3650 let allowed = viable.clone();
3651 let mut quota_losses = Vec::new();
3652 let cache = self.state.config.cache_dir();
3653 let ctx = WaveCtx {
3654 carry_seats: false,
3655 pauses: Some((&self.pause, &self.interrupt)),
3656 run: &run_id,
3657 node: "vote",
3658 prompts: &prompts,
3659 cache: cache.as_deref(),
3660 round: None,
3661 };
3662 let results = ask_json_wave::<FinalVote>(
3663 jobs,
3664 Arc::clone(&self.sem),
3665 self.state.config.graph.retries,
3666 &self.roles.judge_roster,
3667 &ctx,
3668 &mut quota_losses,
3669 &mut self.state,
3670 &move |v: &FinalVote| match v.label() {
3671 Some(c) if allowed.contains(&c) => Ok(()),
3672 other => bail!("vote {other:?} is not one of {allowed:?}"),
3673 },
3674 )
3675 .await;
3676 if self.state.resource_block.is_some() {
3677 return self.stop_for_resource();
3678 }
3679 self.state.quota.extend(quota_losses);
3680
3681 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3682 let agent_id = seat.agent.clone();
3683 self.state.seats.insert(seat.key.clone(), seat);
3684 let initial = self
3685 .state
3686 .judgements
3687 .get(j)
3688 .and_then(|r| r.ranking.first().copied());
3689 let mut record = VoteRecord {
3690 judge: j + 1,
3691 agent: agent_id,
3692 vote: None,
3693 reason: String::new(),
3694 changed: false,
3695 };
3696 match res {
3697 Ok((v, _)) => {
3698 record.vote = v.label();
3699 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3700 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3701 self.state.event(
3702 "vote",
3703 format!(
3704 "judge {} voted {}{}",
3705 j + 1,
3706 record.vote.unwrap_or('?'),
3707 if record.changed { " (changed)" } else { "" }
3708 ),
3709 );
3710 }
3711 Err(e) => {
3712 self.state
3713 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3714 }
3715 }
3716 self.state.votes.push(record);
3717 self.state.save()?;
3718 }
3719 Ok(())
3720 }
3721
3722 fn tally(&mut self) -> Result<()> {
3725 if self.state.tally.is_some() {
3726 return Ok(());
3727 }
3728 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3729 let tops: Vec<char> = self
3730 .state
3731 .judgements
3732 .iter()
3733 .filter_map(|j| j.ranking.first().copied())
3734 .collect();
3735 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3736
3737 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3740 let mut cast: Vec<char> = Vec::new();
3741 for (i, j) in self.state.judgements.iter().enumerate() {
3742 let vote = self
3743 .state
3744 .votes
3745 .iter()
3746 .find(|v| v.judge == i + 1)
3747 .and_then(|v| v.vote)
3748 .or_else(|| j.ranking.first().copied());
3749 if let Some(v) = vote {
3750 *first_choice.entry(v).or_insert(0) += 1;
3751 cast.push(v);
3752 }
3753 }
3754
3755 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3756 for j in &self.state.judgements {
3757 let n = j.ranking.len();
3758 for (pos, label) in j.ranking.iter().enumerate() {
3759 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3760 }
3761 }
3762
3763 let best = first_choice.values().copied().max().unwrap_or(0);
3764 let mut leaders: Vec<char> = first_choice
3765 .iter()
3766 .filter(|(_, v)| **v == best)
3767 .map(|(k, _)| *k)
3768 .collect();
3769 let mut tie_break = None;
3770 if leaders.len() > 1 {
3771 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3772 let borda_leaders: Vec<char> = leaders
3773 .iter()
3774 .copied()
3775 .filter(|l| borda[l] == top_borda)
3776 .collect();
3777 tie_break = Some(if borda_leaders.len() == 1 {
3778 format!(
3779 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3780 leaders.len()
3781 )
3782 } else {
3783 format!(
3784 "{} way tie on both first-choice votes and Borda points, broken by label order",
3785 leaders.len()
3786 )
3787 });
3788 leaders = borda_leaders;
3789 leaders.sort_unstable();
3790 }
3791 let winner = *leaders
3792 .first()
3793 .or(viable.first())
3794 .context("no candidate to declare a winner from")?;
3795
3796 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3797 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3798 let deliberated = !self.state.deliberation.is_empty();
3799
3800 let quota_seats: std::collections::BTreeSet<&str> =
3804 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3805 let mut present = 0usize;
3806 for (i, j) in self.state.judgements.iter().enumerate() {
3807 if quota_seats.contains(j.seat.as_str()) {
3808 continue;
3809 }
3810 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3811 let voted = self
3812 .state
3813 .votes
3814 .iter()
3815 .any(|v| v.judge == i + 1 && v.vote.is_some());
3816 if ranked || voted {
3817 present += 1;
3818 }
3819 }
3820 let needs_quorum = viable.len() > 1;
3826 let judges_total = if needs_quorum {
3827 self.roles.judges.len()
3828 } else {
3829 0
3830 };
3831 let quorum = if needs_quorum {
3832 judges_total / 2 + 1
3833 } else {
3834 0
3835 };
3836 let met_quorum = !needs_quorum || present >= quorum;
3837 let uncontested = (!needs_quorum).then(|| {
3838 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3839 });
3840
3841 self.state.event(
3842 "tally",
3843 match &uncontested {
3844 Some(reason) => format!("winner {winner} — {reason}"),
3845 None => format!(
3846 "winner {winner} — votes {} | initial {} | {} changed | \
3847 {present}/{judges_total} judges{}",
3848 first_choice
3849 .iter()
3850 .map(|(k, v)| format!("{k}:{v}"))
3851 .collect::<Vec<_>>()
3852 .join(" "),
3853 if unanimous_initial {
3854 "unanimous"
3855 } else {
3856 "split"
3857 },
3858 changed_votes,
3859 if met_quorum {
3860 String::new()
3861 } else {
3862 format!(" — below quorum ({quorum} required)")
3863 },
3864 ),
3865 },
3866 );
3867 if !met_quorum {
3868 self.state.event(
3869 "stall",
3870 format!(
3871 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3872 the run stops here, resumable"
3873 ),
3874 );
3875 }
3876 self.state.tally = Some(Tally {
3877 first_choice,
3878 borda,
3879 winner,
3880 rankings: tops.len(),
3881 unanimous_initial,
3882 deliberated,
3883 changed_votes,
3884 unanimous_final,
3885 tie_break,
3886 judges: judges_total,
3887 present,
3888 quorum,
3889 met_quorum,
3890 uncontested,
3891 });
3892 self.state.status = if met_quorum {
3893 RunStatus::Reviewing
3894 } else {
3895 RunStatus::Stalled
3896 };
3897 self.state.save()?;
3898 Ok(())
3899 }
3900
3901 #[allow(clippy::too_many_lines)]
3922 async fn recover_stall(&mut self) -> Result<bool> {
3923 let run_id = self.state.id.clone();
3928 let prompts = self.state.config.prompts.clone();
3929 let quota_seats: BTreeSet<&str> =
3934 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3935 let absent: Vec<String> = self
3936 .state
3937 .judgements
3938 .iter()
3939 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3940 .map(|j| j.seat.clone())
3941 .collect();
3942 if absent.is_empty() {
3943 return Ok(false);
3944 }
3945 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3946 if viable.len() <= 1 {
3947 return Ok(false);
3948 }
3949 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3950 let language = self.state.config.graph.language.clone();
3951 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3952 let sessions = self.state.config.graph.sessions;
3953 let artifacts = agent::artifacts_dir(&self.state.dir());
3954 let root = self.state.worktree_root();
3955 let base_short = short(&self.state.base_commit);
3956 let candidates: Vec<Candidate> = viable.clone();
3957
3958 let mut positions: Vec<usize> = absent
3960 .iter()
3961 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3962 .collect();
3963 if positions.is_empty() {
3964 return Ok(false);
3965 }
3966 positions.sort_unstable();
3967 positions.dedup();
3968
3969 let mut judge_jobs = Vec::new();
3971 for &j in &positions {
3972 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3973 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3974 let seat_key = format!("judge-{}", j + 1);
3975 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3976 let seat = self.seat(&seat_key, &spec.id);
3977 judge_jobs.push(SeatJob {
3978 spec,
3979 seat,
3980 prompt: prompt::judge(
3981 &self.state.instruction,
3982 &views,
3983 self.roles.judges.len(),
3984 &base_short,
3985 &language,
3986 ),
3987 cwd: root.join(seat_key),
3988 timeout,
3989 allow_write: false,
3990 sessions,
3991 artifacts: artifacts.clone(),
3992 stem: format!("judge-{}-recover", j + 1),
3993 handover: None,
3994 });
3995 }
3996
3997 let labels_for_check = labels.clone();
3998 let mut judge_losses = Vec::new();
3999 let retries = self.state.config.graph.retries;
4000 let cache = self.state.config.cache_dir();
4001 let ctx = WaveCtx {
4002 carry_seats: false,
4003 pauses: Some((&self.pause, &self.interrupt)),
4004 run: &run_id,
4005 node: "judge",
4006 prompts: &prompts,
4007 cache: cache.as_deref(),
4008 round: None,
4009 };
4010 let results = ask_json_wave::<Ranking>(
4011 judge_jobs,
4012 Arc::clone(&self.sem),
4013 retries,
4014 &self.roles.judge_roster,
4015 &ctx,
4016 &mut judge_losses,
4017 &mut self.state,
4018 &move |r: &Ranking| r.validate(&labels_for_check),
4019 )
4020 .await;
4021 if self.state.resource_block.is_some() {
4022 self.stop_for_resource()?;
4023 return Ok(false);
4024 }
4025
4026 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4028 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4029 let agent_id = seat.agent.clone();
4030 self.state.seats.insert(seat.key.clone(), seat);
4031 let record = &mut self.state.judgements[j];
4032 match res {
4033 Ok((ranking, out)) => {
4034 record.agent = agent_id;
4035 record.ranking = ranking.normalized();
4036 record.reasons = ranking.reasons;
4037 record.strengths = ranking.strengths;
4038 record.confidence = ranking.confidence;
4039 record.failed = None;
4040 record.duration_ms = out.duration_ms;
4041 recovered.insert(j);
4042 self.state.event(
4043 "recover",
4044 format!("judge {} ranked again after the limit", j + 1),
4045 );
4046 }
4047 Err(e) => {
4048 self.state
4049 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4050 }
4051 }
4052 }
4053
4054 let mut vote_jobs = Vec::new();
4056 let mut vote_pos: Vec<usize> = Vec::new();
4057 for &j in &recovered {
4058 let seat_key = format!("judge-{}", j + 1);
4059 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4060 let seat = self.seat(&seat_key, &spec.id);
4061 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4062 let text = if has_context(&spec, &seat, sessions) {
4063 prompt::final_vote(&labels, &language)
4064 } else {
4065 full.clone()
4066 };
4067 vote_jobs.push(SeatJob {
4068 spec,
4069 seat,
4070 prompt: text,
4071 cwd: root.join(seat_key),
4072 timeout,
4073 allow_write: false,
4074 sessions,
4075 artifacts: artifacts.clone(),
4076 stem: format!("vote-judge-{}-recover", j + 1),
4077 handover: Some(full),
4078 });
4079 vote_pos.push(j);
4080 }
4081 let allowed = labels.clone();
4082 let mut vote_losses = Vec::new();
4083 let vote_retries = self.state.config.graph.retries;
4084 let vote_cache = self.state.config.cache_dir();
4085 let ctx = WaveCtx {
4086 carry_seats: false,
4087 pauses: Some((&self.pause, &self.interrupt)),
4088 run: &run_id,
4089 node: "vote",
4090 prompts: &prompts,
4091 cache: vote_cache.as_deref(),
4092 round: None,
4093 };
4094 let votes = ask_json_wave::<FinalVote>(
4095 vote_jobs,
4096 Arc::clone(&self.sem),
4097 vote_retries,
4098 &self.roles.judge_roster,
4099 &ctx,
4100 &mut vote_losses,
4101 &mut self.state,
4102 &move |v: &FinalVote| match v.label() {
4103 Some(c) if allowed.contains(&c) => Ok(()),
4104 other => bail!("vote {other:?} is not one of {allowed:?}"),
4105 },
4106 )
4107 .await;
4108 if self.state.resource_block.is_some() {
4109 self.stop_for_resource()?;
4110 return Ok(false);
4111 }
4112 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4113 let agent_id = seat.agent.clone();
4114 self.state.seats.insert(seat.key.clone(), seat);
4115 match res {
4116 Ok((v, _)) => {
4117 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4118 rec.vote = v.label();
4119 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4120 } else {
4121 self.state.votes.push(VoteRecord {
4122 judge: j + 1,
4123 agent: agent_id,
4124 vote: v.label(),
4125 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4126 changed: false,
4127 });
4128 }
4129 self.state.event(
4130 "recover",
4131 format!("judge {} voted again after the limit", j + 1),
4132 );
4133 }
4134 Err(e) => {
4135 self.state
4136 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4137 }
4138 }
4139 }
4140
4141 let recovered_keys: BTreeSet<String> = recovered
4145 .iter()
4146 .map(|&j| format!("judge-{}", j + 1))
4147 .collect();
4148 self.state
4149 .quota
4150 .retain(|q| !recovered_keys.contains(&q.seat));
4151 for loss in judge_losses.into_iter().chain(vote_losses) {
4155 if recovered_keys.contains(&loss.seat) {
4156 continue;
4157 }
4158 self.state.quota.retain(|q| q.seat != loss.seat);
4159 self.state.quota.push(loss);
4160 }
4161
4162 self.state.tally = None;
4164 self.tally()?;
4165 Ok(self
4166 .state
4167 .tally
4168 .as_ref()
4169 .map(|t| t.met_quorum)
4170 .unwrap_or(false))
4171 }
4172
4173 async fn graft(&mut self) -> Result<()> {
4189 let Err(e) = self.graft_inner().await else {
4190 return Ok(());
4191 };
4192 if e.is::<crate::repo_lock::ResourceBlocked>() {
4193 return Err(e);
4194 }
4195 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4196 self.state.status = RunStatus::Blocked;
4197 self.state.event("graft", format!("blocked: {e:#}"));
4198 self.state.save()?;
4199 return Err(e);
4200 }
4201 if let Some(rec) = self.state.graft.clone()
4206 && !rec.outcome.settled()
4207 {
4208 if let Some(winner) = self.state.winner().cloned() {
4209 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4210 if re.is::<crate::repo_lock::ResourceBlocked>() {
4211 return Err(re);
4212 }
4213 self.state.status = RunStatus::Blocked;
4214 self.state.event("graft", format!("blocked: {re:#}"));
4215 self.state.save()?;
4216 return Err(re);
4217 }
4218 }
4219 if let Some(g) = self.state.graft.as_mut() {
4220 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4221 }
4222 }
4223 self.state.event("graft", format!("skipped: {e:#}"));
4224 if let Err(e) = self.state.save() {
4225 tracing::warn!("save after graft: {e:#}");
4226 }
4227 Ok(())
4228 }
4229
4230 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4233 let _coordination = self.coordinate(wt).await?;
4234 crate::recovery::restore_repair(wt, pre)
4235 .await
4236 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4237 }
4238
4239 fn graft_skipped(&mut self, why: &str) {
4241 let message = format!("skipped: {why}");
4242 if !self
4243 .state
4244 .events
4245 .iter()
4246 .any(|e| e.node == "graft" && e.message == message)
4247 {
4248 self.state.event("graft", message);
4249 }
4250 }
4251
4252 async fn graft_inner(&mut self) -> Result<()> {
4253 if let Some(rec) = self.state.graft.clone()
4256 && !rec.outcome.settled()
4257 {
4258 if self.state.reviews.is_empty()
4262 && let Some(winner) = self.state.winner().cloned()
4263 {
4264 self.graft_restore(&winner.worktree, &rec.pre).await?;
4265 }
4266 if rec.outcome == GraftOutcome::WaitingForRestore {
4267 self.state.graft = None;
4268 self.state.save()?;
4269 } else {
4270 if let Some(g) = self.state.graft.as_mut() {
4271 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4272 }
4273 self.state.event(
4274 "graft",
4275 "rolled back: the previous round was interrupted; the winner is untouched",
4276 );
4277 self.state.save()?;
4278 return Ok(());
4279 }
4280 }
4281 let plan = match graft::plan(&self.state) {
4282 Ok(plan) => plan,
4283 Err(skip) => {
4284 if skip != graft::Skip::AlreadyDone {
4285 self.graft_skipped(skip.reason());
4286 self.state.save()?;
4287 }
4288 return Ok(());
4289 }
4290 };
4291 let Some(winner) = self.state.winner().cloned() else {
4292 return Ok(());
4293 };
4294 let wt = winner.worktree.clone();
4295 if !git::is_clean(&wt).await? {
4296 self.graft_skipped("the winner's worktree has uncommitted changes");
4297 self.state.save()?;
4298 return Ok(());
4299 }
4300 let pre = git::rev_parse(&wt, "HEAD").await?;
4301 self.state.graft = Some(GraftRecord {
4304 pre: pre.clone(),
4305 post: None,
4306 lines: None,
4307 limit: None,
4308 agent: None,
4309 outcome: GraftOutcome::Running,
4310 requested: graft::requested(&plan),
4311 adoption: None,
4312 adoption_error: None,
4313 });
4314 self.state.event(
4315 "graft",
4316 format!(
4317 "porting strengths of candidate(s) {} onto {}",
4318 plan.strengths
4319 .iter()
4320 .map(|(l, _, _)| l.to_string())
4321 .collect::<Vec<_>>()
4322 .join(", "),
4323 plan.winner
4324 ),
4325 );
4326 self.state.save()?;
4327
4328 let fix_prompt = prompt::graft(
4329 &self.state.instruction,
4330 plan.winner,
4331 &plan.strengths,
4332 &self.state.config.graph.language,
4333 );
4334 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4335 let sessions = self.state.config.graph.sessions;
4336 let artifacts = agent::artifacts_dir(&self.state.dir());
4337 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4338 let (_, seat, out) = self
4339 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4340 SeatJob {
4341 prompt: fix_prompt.clone(),
4342 spec,
4343 seat,
4344 cwd: wt.clone(),
4345 timeout,
4346 allow_write: true,
4347 sessions,
4348 artifacts: artifacts.clone(),
4349 stem: "graft".to_owned(),
4350 handover: None,
4351 }
4352 })
4353 .await;
4354 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4355 self.state.graft.as_mut().expect("graft record").outcome =
4358 GraftOutcome::WaitingForRestore;
4359 return self.stop_for_resource();
4360 }
4361 let mut why: Option<String> = None;
4362 match &out {
4363 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4364 why = Some("the fixer produced nothing usable".to_owned());
4365 }
4366 AgentOutcome::Ok(_) => {}
4367 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4368 AgentOutcome::Quota(o) => {
4369 self.state.quota.push(QuotaLoss {
4370 seat: seat.key.clone(),
4371 node: "graft".to_owned(),
4372 at: Timestamp::now(),
4373 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4374 });
4375 why = Some("rate limited (quota)".to_owned());
4376 }
4377 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4378 why = Some(format!(
4379 "the fixer failed: {}",
4380 e.lines()
4381 .next()
4382 .unwrap_or("")
4383 .chars()
4384 .take(160)
4385 .collect::<String>()
4386 ));
4387 }
4388 }
4389 let agent_id = seat.agent.clone();
4390 self.state.seats.insert(seat.key.clone(), seat);
4391
4392 let mut post = None;
4393 let mut lines = None;
4394 let _coordination = self.coordinate(&wt).await?;
4395 if why.is_none() {
4396 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4397 if let Ok(r) = &rescued {
4398 self.state.note_withheld("graft", &r.withheld);
4399 }
4400 let head = git::rev_parse(&wt, "HEAD").await?;
4401 let dirty = match &rescued {
4405 Ok(r) => git::status_porcelain(&wt)
4406 .await
4407 .map(|st| {
4408 st.lines().any(|l| {
4409 !r.withheld
4410 .iter()
4411 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4412 })
4413 })
4414 .unwrap_or(true),
4415 Err(_) => true,
4416 };
4417 if dirty {
4418 why = Some("the fixer left changes that could not be committed".to_owned());
4419 } else if head == pre {
4420 why = Some("the fixer changed nothing".to_owned());
4421 } else {
4422 lines = git::changed_lines(&wt, &pre, &head).await;
4423 post = Some(head);
4424 }
4425 }
4426 let (adoption, adoption_error) = match &out {
4427 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4428 _ => (None, None),
4429 };
4430 let requested = graft::requested(&plan);
4431 let outcome = match why {
4432 None => {
4433 self.state.event(
4434 "graft",
4435 format!(
4436 "applied: {}",
4437 lines.map_or_else(
4438 || "changed lines unavailable".to_owned(),
4439 |n| format!("{n} changed lines")
4440 )
4441 ),
4442 );
4443 GraftOutcome::Applied
4444 }
4445 Some(why) => {
4446 crate::recovery::restore_repair(&wt, &pre)
4449 .await
4450 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4451 self.state.event(
4452 "graft",
4453 format!("rolled back: {why}; the winner is untouched"),
4454 );
4455 GraftOutcome::RolledBack(why)
4456 }
4457 };
4458 self.state.graft = Some(GraftRecord {
4459 pre,
4460 post,
4461 lines,
4462 limit: None,
4463 agent: Some(agent_id),
4464 outcome,
4465 requested,
4466 adoption,
4467 adoption_error,
4468 });
4469 self.state.save()?;
4470 Ok(())
4471 }
4472
4473 async fn fold_losers(&mut self) -> Result<()> {
4476 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4477 return Ok(());
4478 };
4479 if !self.state.candidate_preservation.is_empty() {
4480 bail!("candidate preservation incomplete; refusing cleanup");
4481 }
4482 let repo = self.state.repo.clone();
4483 let _coordination = self.coordinate(&repo).await?;
4484 let mut folded = Vec::new();
4485 for i in 0..self.state.candidates.len() {
4486 let c = &self.state.candidates[i];
4487 if c.label == winner || c.folded {
4488 continue;
4489 }
4490 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4491 remove_preserved_worktree(&repo, &wt).await?;
4492 if git::branch_exists(&repo, &branch).await?
4493 && !git::branch_delete(&repo, &branch).await?
4494 {
4495 bail!("candidate branch removal refused");
4496 }
4497 self.state.candidates[i].folded = true;
4498 self.state.save()?;
4499 folded.push(label.to_string());
4500 }
4501 let root = self.state.worktree_root();
4503 for j in 1..=self.roles.judges.len() {
4504 let wt = root.join(format!("judge-{j}"));
4505 if wt.exists() {
4506 remove_preserved_worktree(&repo, &wt).await?;
4507 }
4508 }
4509 if self.state.config.graph.advise {
4512 for k in 1..=self.state.config.graph.advisors {
4513 let wt = root.join(format!("advisor-{k}"));
4514 if wt.exists() {
4515 remove_preserved_worktree(&repo, &wt).await?;
4516 }
4517 }
4518 }
4519 if !folded.is_empty() {
4520 self.state
4521 .event("fold", format!("folded candidates {}", folded.join(", ")));
4522 self.state.save()?;
4523 }
4524 Ok(())
4525 }
4526
4527 async fn sync_to_base(&mut self) -> Result<()> {
4567 if self.state.status == RunStatus::AlreadyInBase {
4568 return Ok(());
4569 }
4570 let conflicted = self
4571 .state
4572 .base_sync
4573 .as_ref()
4574 .is_some_and(|s| s.conflict.is_some());
4575 let Some(winner) = self.state.winner().cloned() else {
4576 return Ok(());
4577 };
4578
4579 let repo = self.state.repo.clone();
4580 let remote = self.state.config.merge.remote.clone();
4581 let base_branch = self.state.base_branch.clone();
4582 let tracking = format!("{remote}/{base_branch}");
4583
4584 git::fetch(&repo, &remote, &base_branch).await.ok();
4585 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4590 return Ok(());
4591 };
4592
4593 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4594 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4595 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4596
4597 if (behind > 0 || conflicted || head == tip)
4606 && self
4607 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4608 .await?
4609 {
4610 return Ok(());
4611 }
4612 if conflicted {
4613 return Ok(());
4614 }
4615
4616 if behind == 0 {
4617 if let Some(from) = self
4626 .state
4627 .rebase_fixes
4628 .iter()
4629 .rev()
4630 .find_map(|r| r.from.clone())
4631 && from != head
4632 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4633 .await
4634 .is_ok_and(|o| o.ok())
4635 {
4636 git::sync_to_head(&winner.worktree).await?;
4637 }
4638 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4644 if let Some(why) = &conflict {
4645 self.state.status = RunStatus::Blocked;
4646 self.state.event("land", why.clone());
4647 }
4648 self.state.base_sync = Some(BaseSync {
4649 tip,
4650 behind: 0,
4651 attempts,
4652 conflict,
4653 already_in: None,
4654 });
4655 self.state.save()?;
4656 return Ok(());
4657 }
4658
4659 if attempts >= BASE_SYNC_ROUNDS {
4660 let why = format!(
4661 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4662 rebase(s); rebasing again would only race it",
4663 winner.branch
4664 );
4665 self.state.status = RunStatus::Blocked;
4666 self.state.base_sync = Some(BaseSync {
4667 tip,
4668 behind,
4669 attempts,
4670 conflict: Some(why.clone()),
4671 already_in: None,
4672 });
4673 self.state.event("land", why);
4674 self.state.save()?;
4675 return Ok(());
4676 }
4677
4678 self.state.event(
4679 "land",
4680 format!(
4681 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4682 winner.branch
4683 ),
4684 );
4685 self.state.save()?;
4686
4687 let branch_tracking = format!("{remote}/{}", winner.branch);
4693 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4694 let remote_tip = if self.state.task_pr_branch().is_none()
4695 && matches!(&fetched_branch, Ok(o) if o.ok())
4696 {
4697 git::rev_parse(&repo, &branch_tracking).await.ok()
4698 } else {
4699 None
4700 };
4701 if let Some(theirs) = &remote_tip
4705 && !git::is_ancestor(&repo, theirs, &head).await
4706 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4707 .await
4708 .is_ok_and(|missing| missing.is_empty())
4709 {
4710 let why = format!(
4711 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4712 them",
4713 short(theirs),
4714 winner.branch
4715 );
4716 self.state.status = RunStatus::Blocked;
4717 self.state.base_sync = Some(BaseSync {
4718 tip,
4719 behind,
4720 attempts,
4721 conflict: Some(why.clone()),
4722 already_in: None,
4723 });
4724 self.state.event("land", why);
4725 self.state.save()?;
4726 return Ok(());
4727 }
4728
4729 let scratch = self.state.dir().join("base-sync");
4730 let rebased = match crate::rebase::rebase_with_fixer(
4731 &mut self.state,
4732 &scratch,
4733 &winner.branch,
4734 &tracking,
4735 )
4736 .await
4737 {
4738 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4739 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4740 Err(e) => Err(e),
4741 };
4742 let attempts = attempts + 1;
4743 match rebased {
4744 Ok(None) => {
4745 git::sync_to_head(&winner.worktree).await?;
4749 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4750 let mut conflict = None;
4751 if let Some(pinned) = &remote_tip {
4752 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4753 match pushed {
4754 Ok(o) if o.ok() => self.state.event(
4755 "land",
4756 format!("pushed rebased {} to {remote}", winner.branch),
4757 ),
4758 Ok(o) => {
4759 conflict = Some(format!(
4760 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4761 winner.branch,
4762 short(pinned),
4763 o.stderr.chars().take(600).collect::<String>()
4764 ));
4765 }
4766 Err(e) => {
4767 conflict = Some(format!(
4768 "rebased {} locally but could not push it: {e:#}",
4769 winner.branch
4770 ));
4771 }
4772 }
4773 }
4774 if let Some(why) = &conflict {
4775 self.state.status = RunStatus::Blocked;
4776 self.state.event("land", why.clone());
4777 }
4778 self.state.base_sync = Some(BaseSync {
4779 tip: tip.clone(),
4780 behind: 0,
4781 attempts,
4782 conflict,
4783 already_in: None,
4784 });
4785 self.state
4786 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4787 }
4788 Ok(Some(conflict)) => {
4789 let why = format!(
4790 "{} conflicts with {tracking} and did not rebase: {}",
4791 winner.branch,
4792 conflict.chars().take(600).collect::<String>()
4793 );
4794 self.state.status = RunStatus::Blocked;
4795 self.state.base_sync = Some(BaseSync {
4796 tip,
4797 behind,
4798 attempts,
4799 conflict: Some(why.clone()),
4800 already_in: None,
4801 });
4802 self.state.event("land", why);
4803 }
4804 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4808 self.state.save()?;
4809 return Ok(());
4810 }
4811 Err(e) => {
4812 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4813 self.state.status = RunStatus::Blocked;
4814 self.state.base_sync = Some(BaseSync {
4815 tip,
4816 behind,
4817 attempts,
4818 conflict: Some(why.clone()),
4819 already_in: None,
4820 });
4821 self.state.event("land", why);
4822 }
4823 }
4824 self.state.save()?;
4825 Ok(())
4826 }
4827
4828 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4834 if self.state.task_pr_branch().is_some() {
4837 return None;
4838 }
4839 let repo = self.state.repo.clone();
4840 let remote = self.state.config.merge.remote.clone();
4841 let fetched = git::fetch(&repo, &remote, branch).await;
4842 if !matches!(&fetched, Ok(o) if o.ok()) {
4843 return None;
4844 }
4845 let branch_tracking = format!("{remote}/{branch}");
4846 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4847 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4848 return None;
4849 }
4850 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4851 .await
4852 .is_ok_and(|missing| missing.is_empty())
4853 {
4854 return Some(format!(
4855 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4856 them",
4857 short(&theirs)
4858 ));
4859 }
4860 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4861 Ok(o) if o.ok() => {
4862 self.state
4863 .event("land", format!("pushed rebased {branch} to {remote}"));
4864 None
4865 }
4866 Ok(o) => Some(format!(
4867 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4868 someone may have pushed): {}",
4869 short(&theirs),
4870 o.stderr.chars().take(600).collect::<String>()
4871 )),
4872 Err(e) => Some(format!(
4873 "{branch} is rebased locally but could not be pushed: {e:#}"
4874 )),
4875 }
4876 }
4877
4878 async fn settle_already_in(
4895 &mut self,
4896 branch: &str,
4897 tip: &str,
4898 head: &str,
4899 attempts: usize,
4900 behind: usize,
4901 ) -> Result<bool> {
4902 let repo = self.state.repo.clone();
4903 let remote = self.state.config.merge.remote.clone();
4904 let start = self.state.base_commit.clone();
4905 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4906 Ok(Some(e)) => e,
4907 Ok(None) => return Ok(false),
4908 Err(e) => {
4909 tracing::warn!("already-in-base check for {branch}: {e:#}");
4910 return Ok(false);
4911 }
4912 };
4913 let mut verified = vec![head.to_owned()];
4914 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4915 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4916 if matches!(&fetched, Ok(o) if o.ok())
4917 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4918 && theirs != head
4919 {
4920 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4921 Ok(Some(_)) => verified.push(theirs),
4922 _ => return Ok(false),
4923 }
4924 }
4925 let base_branch = self.state.base_branch.clone();
4926 let message = format!(
4927 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4928 land",
4929 evidence.names(),
4930 evidence.proof.as_str()
4931 );
4932 let closed =
4933 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4934 .await;
4935 match closed {
4936 Ok(Ok(url)) => self
4937 .state
4938 .event("land", format!("closed {url}: superseded on {base_branch}")),
4939 Ok(Err(why)) => self
4940 .state
4941 .event("land", format!("did not close a pull request: {why}")),
4942 Err(e) => {
4943 let why = format!(
4944 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4945 not be closed ({e:#}); resume to retry"
4946 );
4947 self.state.status = RunStatus::Blocked;
4948 self.state.base_sync = Some(BaseSync {
4949 tip: tip.to_owned(),
4950 behind,
4951 attempts,
4952 conflict: Some(why.clone()),
4953 already_in: None,
4954 });
4955 self.state.event("land", why);
4956 self.state.save()?;
4957 return Ok(true);
4958 }
4959 }
4960 self.state.status = RunStatus::AlreadyInBase;
4961 self.state.base_sync = Some(BaseSync {
4962 tip: tip.to_owned(),
4963 behind,
4964 attempts,
4965 conflict: None,
4966 already_in: Some(evidence),
4967 });
4968 self.state.event("land", message);
4969 self.state.save()?;
4970 self.settle_questions();
4971 Ok(true)
4972 }
4973
4974 fn landing_base(&self) -> String {
4984 self.state
4985 .base_sync
4986 .as_ref()
4987 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4988 }
4989
4990 pub async fn fix_selected(
5023 &mut self,
5024 ids: &[String],
5025 reason: &str,
5026 allow_stale: bool,
5027 ) -> Result<()> {
5028 let reason = reason.trim();
5029 if reason.is_empty() {
5030 bail!("a fix request needs a reason — that is the operator's own record of why");
5031 }
5032 if ids.is_empty() {
5033 bail!("no finding id given");
5034 }
5035 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5036 bail!(
5037 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5038 has already concluded — can be given a targeted fix. A run still \
5039 in progress should simply be resumed; a `merged` run's branch has \
5040 already landed, so its answer is a fresh `magi review <branch>`, \
5041 not reopening this run's own record",
5042 self.state.id,
5043 self.state.status.as_str()
5044 );
5045 }
5046 let Some(winner) = self.state.winner().cloned() else {
5047 bail!("run {} has no winning candidate to fix", self.state.id);
5048 };
5049 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5050 bail!(
5051 "branch `{}` no longer exists; this run cannot be extended",
5052 winner.branch
5053 );
5054 }
5055 let home = crate::run::home();
5056 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5057 bail!(
5058 "run {} is currently being worked on by another magi process",
5059 self.state.id
5060 );
5061 }
5062 let _claim = FixClaim::acquire(&self.state.dir())?;
5068
5069 let mut seen = BTreeSet::new();
5073 let mut findings = Vec::new();
5074 let mut missing = Vec::new();
5075 for id in ids {
5076 if !seen.insert(id.clone()) {
5077 continue;
5078 }
5079 match self.state.finding(id) {
5080 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5081 id: f.id.clone(),
5082 severity: f.severity,
5083 reviewer_vote: rec.vote,
5084 round: round.round,
5085 round_head: round.head.clone(),
5086 reviewer: rec.reviewer,
5087 agent: rec.agent.clone(),
5088 file: f.file.clone(),
5089 line: f.line,
5090 title: f.title.clone(),
5091 detail: f.detail.clone(),
5092 outcome: OperatorFixOutcome::Pending,
5093 }),
5094 None => missing.push(id.clone()),
5095 }
5096 }
5097 if !missing.is_empty() {
5098 bail!(
5099 "unknown finding id(s): {}; nothing was changed",
5100 missing.join(", ")
5101 );
5102 }
5103
5104 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5105 let stale_details: Vec<(String, String)> = findings
5106 .iter()
5107 .filter(|f| f.round_head != head_at_request)
5108 .map(|f| (f.id.clone(), f.round_head.clone()))
5109 .collect();
5110 let stale = !stale_details.is_empty();
5111 if stale && !allow_stale {
5112 bail!(
5113 "the branch has moved since some finding(s) were raised — {} — now \
5114 at {}; pass --allow-stale to fix anyway, or re-run review first",
5115 stale_details
5116 .iter()
5117 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5118 .collect::<Vec<_>>()
5119 .join(", "),
5120 short(&head_at_request)
5121 );
5122 }
5123
5124 let request = OperatorFixRequest {
5125 requested_at: Timestamp::now(),
5126 reason: reason.to_owned(),
5127 findings,
5128 head_at_request: head_at_request.clone(),
5129 allow_stale,
5130 stale,
5131 fix: None,
5132 result_head: None,
5133 follow_up_review_run: None,
5134 };
5135 self.state.event(
5136 "fix",
5137 format!(
5138 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5139 request.findings.len(),
5140 request
5141 .findings
5142 .iter()
5143 .map(|f| f.id.as_str())
5144 .collect::<Vec<_>>()
5145 .join(", "),
5146 ),
5147 );
5148 self.state.operator_fixes.push(request);
5155 self.state.save()?;
5156 let request_index = self.state.operator_fixes.len() - 1;
5157
5158 let repo = self.state.repo.clone();
5167 let coordination = self.coordinate(&repo).await?;
5168 if winner.worktree.exists() {
5169 let dirty = git::git(
5172 &winner.worktree,
5173 &["status", "--porcelain", "--untracked-files=all"],
5174 )
5175 .await?;
5176 let only_withheld = dirty.lines().all(|l| {
5177 l.strip_prefix("?? ")
5178 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5179 });
5180 if !only_withheld {
5181 bail!(
5182 "`{}` has uncommitted changes; refusing to touch it — commit or \
5183 discard them first",
5184 winner.worktree.display()
5185 );
5186 }
5187 git::worktree_remove(&self.state.repo, &winner.worktree)
5188 .await
5189 .ok();
5190 }
5191 let fix_worktree = self.state.worktree_root().join("operator-fix");
5192 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5193 git::git(
5194 &self.state.repo,
5195 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5196 )
5197 .await
5198 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5199 if !git::is_clean(&fix_worktree).await? {
5200 git::worktree_remove(&self.state.repo, &fix_worktree)
5201 .await
5202 .ok();
5203 bail!(
5204 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5205 winner.branch
5206 );
5207 }
5208 let repo_for_setup = self.state.repo.clone();
5209 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5210 Ok(()) => {
5211 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5212 }
5213 Err(e) => Err(e),
5214 };
5215 if let Err(e) = setup {
5216 git::worktree_remove(&self.state.repo, &fix_worktree)
5217 .await
5218 .ok();
5219 return Err(e);
5220 }
5221
5222 let run_id = self.state.id.clone();
5223 let prompts = self.state.config.prompts.clone();
5224 let language = self.state.config.graph.language.clone();
5225 let sessions = self.state.config.graph.sessions;
5226 let artifacts = agent::artifacts_dir(&self.state.dir());
5227 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5228 .findings
5229 .iter()
5230 .map(|f| Finding {
5231 id: f.id.clone(),
5232 severity: f.severity,
5233 file: f.file.clone(),
5234 line: f.line,
5235 title: f.title.clone(),
5236 detail: f.detail.clone(),
5237 })
5238 .collect();
5239 let fix_prompt = prompt::operator_fix_in(
5240 &self.state.instruction,
5241 &finding_list,
5242 reason,
5243 &stale_details,
5244 &head_at_request,
5245 &language,
5246 &self.state.config.graph.github_language,
5247 );
5248 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5249 drop(coordination);
5250 let (job, seat, out) = self
5251 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5252 prompt: fix_prompt.clone(),
5253 spec,
5254 seat,
5255 cwd: fix_worktree.clone(),
5256 timeout,
5257 allow_write: true,
5258 sessions,
5259 artifacts: artifacts.clone(),
5260 stem: "operator-fix".to_owned(),
5261 handover: None,
5262 })
5263 .await;
5264 let agent_id = seat.agent.clone();
5265
5266 let mut fix = FixRecord {
5267 agent: agent_id,
5268 addressed: Vec::new(),
5269 rejected: Vec::new(),
5270 notes: String::new(),
5271 committed: false,
5272 failed: None,
5273 duration_ms: 0,
5274 continuation: None,
5275 };
5276 let mut final_seat = seat.clone();
5277 match out {
5278 AgentOutcome::Ok(o) => {
5279 fix.duration_ms = o.duration_ms;
5280 let parsed = verdict::extract_json::<FixReport>(&o.text);
5281 let incomplete_reason = match &parsed {
5282 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5283 "the reply parsed, but it reported a command whose own CLI \
5284 never confirmed an exit status"
5285 .to_owned(),
5286 ),
5287 Ok(_) => None,
5288 Err(e) => Some(e.to_string()),
5289 };
5290 match incomplete_reason {
5291 None => {
5292 let report = parsed.expect("checked Ok above");
5293 fix.addressed = report.addressed;
5294 fix.rejected = report.rejected;
5295 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5296 }
5297 Some(reason) => {
5298 let (resumed_seat, resolved, failure, cont) = self
5299 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5300 .await;
5301 fix.duration_ms += cont.cumulative_wait_ms;
5302 fix.continuation = Some(cont);
5303 final_seat = resumed_seat;
5304 match resolved {
5305 Some(report) => {
5306 fix.addressed = report.addressed;
5307 fix.rejected = report.rejected;
5308 fix.notes =
5309 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5310 }
5311 None => fix.failed = failure,
5312 }
5313 }
5314 }
5315 }
5316 AgentOutcome::Dropped(o) => {
5317 fix.duration_ms = o.duration_ms;
5318 let why = o
5319 .dropped
5320 .as_ref()
5321 .map(|d| d.why.as_str())
5322 .unwrap_or("the CLI ended the stream without delivering its answer");
5323 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5324 }
5325 AgentOutcome::Quota(o) => {
5326 self.state.quota.push(QuotaLoss {
5327 seat: final_seat.key.clone(),
5328 node: "fix".to_owned(),
5329 at: Timestamp::now(),
5330 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5331 });
5332 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5333 }
5334 AgentOutcome::Failed(e) => fix.failed = Some(e),
5335 AgentOutcome::ResourceBlocked(e) => {
5336 fix.failed = Some(format!("the fixer was not started: {e}"));
5337 }
5338 }
5339 if fix.continuation.is_none() {
5340 fix.continuation = Some(ContinuationRecord::not_needed());
5341 }
5342 self.state.seats.insert(final_seat.key.clone(), final_seat);
5343
5344 let rescue_message = format!(
5345 "magi: operator-selected fix ({}) (uncommitted work)",
5346 self.state.operator_fixes[request_index]
5347 .findings
5348 .iter()
5349 .map(|f| f.id.as_str())
5350 .collect::<Vec<_>>()
5351 .join(", ")
5352 );
5353 let repo = self.state.repo.clone();
5354 let coordination = self.coordinate(&repo).await?;
5355 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5356 self.state.note_withheld("fix", &r.withheld);
5357 }
5358 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5359 fix.committed = after != head_at_request;
5360 git::worktree_remove(&self.state.repo, &fix_worktree)
5361 .await
5362 .ok();
5363 drop(coordination);
5364
5365 self.state.event(
5366 "fix",
5367 match &fix.failed {
5368 Some(reason) => format!(
5369 "operator fix: adoption report was lost ({reason}); {}",
5370 if fix.committed {
5371 "committed"
5372 } else {
5373 "NO new commit"
5374 }
5375 ),
5376 None => format!(
5377 "operator fix: {} addressed, {} rejected, {}",
5378 fix.addressed.len(),
5379 fix.rejected.len(),
5380 if fix.committed {
5381 "committed"
5382 } else {
5383 "NO new commit"
5384 }
5385 ),
5386 },
5387 );
5388
5389 for f in &mut self.state.operator_fixes[request_index].findings {
5396 f.outcome = if fix.failed.is_some() {
5397 OperatorFixOutcome::Unreported
5398 } else if fix.addressed.contains(&f.id) {
5399 OperatorFixOutcome::Addressed
5400 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5401 OperatorFixOutcome::Rejected { why: r.why.clone() }
5402 } else {
5403 OperatorFixOutcome::Unreported
5404 };
5405 }
5406
5407 let committed = fix.committed;
5408 if committed {
5409 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5410 }
5411 self.state.operator_fixes[request_index].fix = Some(fix);
5412 self.state.save()?;
5415
5416 if committed {
5417 self.state.event(
5418 "fix",
5419 format!(
5420 "operator fix committed {}; opening a follow-up review-only run",
5421 short(&after)
5422 ),
5423 );
5424 let origin =
5427 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5428 match Self::review(
5429 &self.state.repo,
5430 &winner.branch,
5431 self.state.config.clone(),
5432 origin,
5433 )
5434 .await
5435 {
5436 Ok(mut follow_up) => {
5437 follow_up.state.event(
5438 "start",
5439 format!(
5440 "requested by an operator fix on run {} for finding(s) {}",
5441 self.state.id,
5442 self.state.operator_fixes[request_index]
5443 .findings
5444 .iter()
5445 .map(|f| f.id.as_str())
5446 .collect::<Vec<_>>()
5447 .join(", "),
5448 ),
5449 );
5450 follow_up.state.save()?;
5451 let follow_up_id = follow_up.state.id.clone();
5452 let adopted = match crate::direct::adopt(
5455 &follow_up.state,
5456 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5457 ) {
5458 Ok(a) => a,
5459 Err(e) => {
5460 self.state.event(
5463 "fix",
5464 format!(
5465 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5466 ),
5467 );
5468 self.state.save()?;
5469 return Ok(());
5470 }
5471 };
5472 let executed = follow_up.execute().await;
5473 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5474 if let Some(a) = adopted {
5475 a.finish(&follow_up.state, executed);
5476 }
5477 if let Some(e) = failure {
5478 self.state.event(
5479 "fix",
5480 format!(
5481 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5482 ),
5483 );
5484 }
5485 self.state.operator_fixes[request_index].follow_up_review_run =
5486 Some(follow_up_id);
5487 }
5488 Err(e) => {
5489 self.state.event(
5490 "fix",
5491 format!("committed the fix but could not open a follow-up review: {e:#}"),
5492 );
5493 }
5494 }
5495 self.state.save()?;
5496 }
5497
5498 Ok(())
5499 }
5500
5501 async fn ask_fixer(
5521 &mut self,
5522 winner: &Candidate,
5523 node: &'static str,
5524 round: Option<usize>,
5525 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5526 ) -> (SeatJob, SeatState, AgentOutcome) {
5527 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5528 self.ask_fixer_among(attempts, node, round, None, build)
5529 .await
5530 }
5531
5532 async fn ask_fixer_among(
5537 &mut self,
5538 attempts: Vec<(AgentSpec, String)>,
5539 node: &'static str,
5540 round: Option<usize>,
5541 restore: Option<(&Path, &str)>,
5542 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5543 ) -> (SeatJob, SeatState, AgentOutcome) {
5544 let run_id = self.state.id.clone();
5545 let prompts = self.state.config.prompts.clone();
5546 let cache = self.state.config.cache_dir();
5547 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5548 let mut last = None;
5549 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5550 let seat = self.seat(&key, &spec.id);
5551 let mut job = build(spec.clone(), seat);
5552 if i > 0 {
5553 job.stem = format!("{}-{}", job.stem, spec.id);
5554 }
5555 let ctx = WaveCtx {
5556 carry_seats: false,
5557 pauses: Some((&self.pause, &self.interrupt)),
5558 run: &run_id,
5559 node,
5560 prompts: &prompts,
5561 cache: cache.as_deref(),
5562 round,
5563 };
5564 let (seat, out) =
5565 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5566 let advances = match &out {
5569 AgentOutcome::Ok(o) => agent::output_advances(o),
5570 AgentOutcome::ResourceBlocked(_) => false,
5571 _ => true,
5572 };
5573 if let Some(next) = ids.get(i + 1)
5574 && advances
5575 {
5576 self.state.seats.insert(seat.key.clone(), seat.clone());
5577 let class =
5578 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5579 let reason = match &out {
5580 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5581 other => fail_reason(other),
5582 };
5583 if let Some((wt, rev)) = restore
5587 && let Err(e) = self.graft_restore(wt, rev).await
5588 {
5589 let why =
5590 format!("could not restore the worktree before the next fixer: {e:#}");
5591 tracing::warn!("{why}");
5592 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5593 AgentOutcome::ResourceBlocked(why)
5594 } else {
5595 AgentOutcome::Failed(why)
5596 };
5597 last = Some((job, seat, out));
5598 break;
5599 }
5600 record_handover(
5601 &mut self.state,
5602 node,
5603 &seat.key,
5604 &spec.id,
5605 next,
5606 &class,
5607 &reason,
5608 );
5609 continue;
5610 }
5611 last = Some((job, seat, out));
5612 break;
5613 }
5614 last.expect("the fixer chain always has an entry")
5615 }
5616
5617 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5628 use crate::recovery::{Mode, Repair};
5629 if !self.state.reviews.is_empty() {
5630 return Ok(());
5631 }
5632 let Some(mut rec) = self.state.recovery.clone() else {
5633 return Ok(());
5634 };
5635 if rec.mode != Mode::VerifyRepair {
5636 return Ok(());
5637 }
5638 let wt = winner.worktree.clone();
5639 let mut head = git::rev_parse(&wt, "HEAD").await?;
5640 match rec.repair.clone() {
5641 Repair::Done { .. } => return Ok(()),
5642 Repair::WaitingForRestore { before } => {
5643 self.graft_restore(&wt, &before).await?;
5644 head = before;
5645 rec.repair = Repair::NotStarted;
5646 self.state.recovery = Some(rec.clone());
5647 self.state.save()?;
5648 }
5649 Repair::Started { before } => {
5650 if let Err(e) = self.graft_restore(&wt, &before).await {
5654 if e.is::<crate::repo_lock::ResourceBlocked>() {
5655 return Err(e);
5656 }
5657 return self
5658 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5659 .await;
5660 }
5661 rec.repair = Repair::Done {
5662 before: before.clone(),
5663 after: before,
5664 committed: false,
5665 failed: Some("interrupted; rolled back".to_owned()),
5666 };
5667 self.state
5668 .event("recovery", "the repair was interrupted and rolled back");
5669 self.state.recovery = Some(rec);
5670 self.state.save()?;
5671 return Ok(());
5672 }
5673 Repair::NotStarted => {}
5674 }
5675 if crate::recovery::recheck(&mut rec, &head) {
5676 self.state.event("recovery", rec.summary());
5677 self.state.recovery = Some(rec);
5678 self.state.save()?;
5679 return Ok(());
5680 }
5681 rec.repair = Repair::Started {
5682 before: head.clone(),
5683 };
5684 self.state.recovery = Some(rec.clone());
5685 self.state.save()?;
5686
5687 let language = self.state.config.graph.language.clone();
5688 let sessions = self.state.config.graph.sessions;
5689 let artifacts = agent::artifacts_dir(&self.state.dir());
5690 let fix_prompt = prompt::recovery_fix(
5691 &self.state.instruction,
5692 &rec,
5693 self.state.config.graph.review_rounds,
5694 &language,
5695 );
5696 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5697 self.state.event(
5698 "recovery",
5699 format!(
5700 "repair: the fixer starts from the failed verification on {}",
5701 short(&head)
5702 ),
5703 );
5704 let (_job, seat, out) = self
5705 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5706 prompt: fix_prompt.clone(),
5707 spec,
5708 seat,
5709 cwd: wt.clone(),
5710 timeout,
5711 allow_write: true,
5712 sessions,
5713 artifacts: artifacts.clone(),
5714 stem: "fix-recovery".to_owned(),
5715 handover: None,
5716 })
5717 .await;
5718 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5719 rec.repair = Repair::WaitingForRestore { before: head };
5720 self.state.recovery = Some(rec);
5721 return self.stop_for_resource();
5722 }
5723 let failed = match &out {
5724 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5725 AgentOutcome::Quota(o) => {
5726 self.state.quota.push(QuotaLoss {
5727 seat: seat.key.clone(),
5728 node: "fix".to_owned(),
5729 at: Timestamp::now(),
5730 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5731 });
5732 Some("rate limited (quota); the fixer could not run".to_owned())
5733 }
5734 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5735 AgentOutcome::Failed(e) => Some(e.clone()),
5736 };
5737 self.state.seats.insert(seat.key.clone(), seat);
5738 let coordination = self.coordinate(&wt).await?;
5739 let failed = match failed {
5740 Some(why) => {
5742 if let Err(e) = git::restore_to(&wt, &head).await {
5743 return self
5744 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5745 .await;
5746 }
5747 Some(why)
5748 }
5749 None => {
5750 if let Ok(r) = git::rescue_commit(
5751 &wt,
5752 "magi: recovery repair of the failed verification (uncommitted work)",
5753 )
5754 .await
5755 {
5756 self.state.note_withheld("fix", &r.withheld);
5757 }
5758 None
5759 }
5760 };
5761 let after = git::rev_parse(&wt, "HEAD").await?;
5762 drop(coordination);
5763 self.state.event(
5764 "recovery",
5765 match &failed {
5766 Some(why) => format!("repair: nothing kept ({why})"),
5767 None if after != head => {
5768 format!("repair: committed {}", short(&after))
5769 }
5770 None => "repair: the fixer made no commit".to_owned(),
5771 },
5772 );
5773 rec.repair = Repair::Done {
5774 before: head.clone(),
5775 committed: after != head,
5776 after,
5777 failed,
5778 };
5779 self.state.recovery = Some(rec);
5780 self.state.save()?;
5781 Ok(())
5782 }
5783
5784 async fn stop_recovery(
5787 &mut self,
5788 mut rec: crate::recovery::RecoveryRecord,
5789 head: &str,
5790 why: String,
5791 ) -> Result<()> {
5792 rec.repair = crate::recovery::Repair::Started {
5794 before: head.to_owned(),
5795 };
5796 self.state.recovery = Some(rec);
5797 self.state
5798 .event("recovery", format!("repair could not be undone: {why}"));
5799 self.state.status = RunStatus::Blocked;
5800 self.state.save()?;
5801 bail!("recovery repair could not be rolled back: {why}")
5802 }
5803
5804 async fn review_loop(&mut self) -> Result<()> {
5805 if self
5810 .state
5811 .base_sync
5812 .as_ref()
5813 .is_some_and(|s| s.conflict.is_some())
5814 {
5815 return Ok(());
5816 }
5817 let run_id = self.state.id.clone();
5822 let prompts = self.state.config.prompts.clone();
5823 let Some(winner) = self.state.winner().cloned() else {
5824 return Ok(());
5825 };
5826 let max_rounds = self.state.config.graph.review_rounds;
5827 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5837 if status == RunStatus::Gating {
5840 self.record_contested_handoff();
5841 }
5842 self.state.status = status;
5843 self.state.save()?;
5844 return Ok(());
5845 }
5846 self.state.status = RunStatus::Reviewing;
5847 if self
5857 .state
5858 .reviews
5859 .last()
5860 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5861 {
5862 let shell = self.state.config.shell();
5863 return self
5864 .stop_reviewing(
5865 "the last round's own verification never resolved",
5866 &shell,
5867 &winner.worktree,
5868 )
5869 .await;
5870 }
5871
5872 self.recovery_repair(&winner).await?;
5873 if self.resource_halt()? {
5874 return Ok(());
5875 }
5876
5877 let repo = self.state.repo.clone();
5878 let root = self.state.worktree_root();
5879 let language = self.state.config.graph.language.clone();
5880 let sessions = self.state.config.graph.sessions;
5881 let artifacts = self.merge_repair_artifacts();
5882 let base = self.landing_base();
5883 let base_short = short(&base);
5884 let reviewers = self.roles.reviewers.clone();
5885 let shell = self.state.config.shell();
5886
5887 for round in (self.state.reviews.len() + 1)..=max_rounds {
5888 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5889 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5890 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5891 let prev_verification = self
5900 .state
5901 .reviews
5902 .last()
5903 .and_then(|r| r.verification_summary(&head));
5904 let prior_section = match (&self.state.recovery, round) {
5907 (Some(rec), 1) if self.state.reviews.is_empty() => {
5908 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5909 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5910 Some(crate::recovery::head_of(
5911 &git::diff(&winner.worktree, reviewed, "HEAD")
5912 .await
5913 .unwrap_or_default(),
5914 crate::recovery::SINCE_DIFF_MAX,
5915 ))
5916 }
5917 _ => None,
5918 };
5919 Some(prompt::recovery_review(rec, since.as_deref()))
5920 }
5921 _ => None,
5922 };
5923
5924 let mut jobs = Vec::new();
5928 let coordination = self.coordinate(&repo).await?;
5929 for (r, spec) in reviewers.iter().cloned().enumerate() {
5930 let wt = root.join(format!("review-{}", r + 1));
5931 if wt.exists() {
5934 git::reset_detached(&wt, &head).await?;
5935 } else {
5936 git::worktree_add_detached(&repo, &wt, &head).await?;
5937 }
5938 self.setup_seat_worktree(&repo, &wt, false).await?;
5939 let seat_key = format!("review-{}", r + 1);
5940 let spec = pick_start_spec(
5943 &self.roles.reviewer_roster,
5944 spec,
5945 self.state.seat_history.get(&seat_key),
5946 );
5947 let seat = self.seat(&seat_key, &spec.id);
5948 jobs.push(SeatJob {
5949 prompt: prompt::review(&prompt::ReviewCtx {
5950 instruction: &self.state.instruction,
5951 branch: &winner.branch,
5952 base_short: &base_short,
5953 stat: &stat,
5954 patch: &patch,
5955 verification: prev_verification.as_ref(),
5956 prior: prior_section.as_deref(),
5957 reviewers: reviewers.len(),
5958 round,
5959 rounds: max_rounds,
5960 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5963 lens: Lens::for_seat(r),
5964 language: &language,
5965 github_language: &self.state.config.graph.github_language,
5966 }),
5967 spec,
5968 seat,
5969 cwd: wt,
5970 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5971 allow_write: false,
5972 sessions,
5973 artifacts: artifacts.clone(),
5974 stem: format!("review-{round}-{}", r + 1),
5975 handover: None,
5976 });
5977 }
5978
5979 drop(coordination);
5980 self.state.event(
5981 "review",
5982 format!(
5983 "round {round}: {} reviewers on {}",
5984 jobs.len(),
5985 short(&head)
5986 ),
5987 );
5988 let mut quota_losses = Vec::new();
5989 let review_retries = self.state.config.graph.retries;
5990 let review_cache = self.state.config.cache_dir();
5991 let ctx = WaveCtx {
5992 carry_seats: true,
5993 pauses: Some((&self.pause, &self.interrupt)),
5994 run: &run_id,
5995 node: "review",
5996 prompts: &prompts,
5997 cache: review_cache.as_deref(),
5998 round: Some(round),
5999 };
6000 let results = ask_json_wave::<Review>(
6001 jobs,
6002 Arc::clone(&self.sem),
6003 review_retries,
6004 &self.roles.reviewer_roster,
6005 &ctx,
6006 &mut quota_losses,
6007 &mut self.state,
6008 &|_: &Review| Ok(()),
6009 )
6010 .await;
6011 if self.state.resource_block.is_some() {
6012 return self.stop_for_resource();
6013 }
6014 let round_quota_missing = quota_losses.len();
6018 self.state.quota.extend(quota_losses);
6019
6020 let mut records = Vec::new();
6021 let mut all_findings = Vec::new();
6022 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6023 let agent_id = seat.agent.clone();
6024 self.state.seats.insert(seat.key.clone(), seat);
6025 let mut record = ReviewRecord {
6026 reviewer: r + 1,
6027 agent: agent_id,
6028 summary: String::new(),
6029 findings: Vec::new(),
6030 vote: None,
6031 failed: None,
6032 duration_ms: 0,
6033 attempts,
6039 };
6040 match res {
6041 Ok((review, out)) => {
6042 record.summary =
6050 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6051 record.vote = Some(review.vote);
6052 record.duration_ms = out.duration_ms;
6053 for (n, mut f) in review.findings.into_iter().enumerate() {
6054 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6057 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6058 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6059 f.file = f
6065 .file
6066 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6067 all_findings.push(f.clone());
6068 record.findings.push(f);
6069 }
6070 self.state.event(
6071 "review",
6072 format!(
6073 "round {round}: reviewer {} voted {} with {} finding(s)",
6074 r + 1,
6075 review.vote.label(),
6076 record.findings.len()
6077 ),
6078 );
6079 }
6080 Err(e) => {
6081 record.failed = Some(e.to_string());
6082 self.state.event(
6083 "review",
6084 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6085 );
6086 }
6087 }
6088 records.push(record);
6089 }
6090
6091 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6098 let vote_split =
6099 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6100 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6101 if vote_split {
6102 self.state.event(
6103 "review",
6104 format!(
6105 "round {round}: votes split ({}) — one round of reconsideration",
6106 initial_votes
6107 .iter()
6108 .map(|v| v.label())
6109 .collect::<Vec<_>>()
6110 .join(", ")
6111 ),
6112 );
6113 let panel: Vec<ReviewSeatReport<'_>> = records
6116 .iter()
6117 .filter_map(|r| {
6118 r.vote.map(|vote| ReviewSeatReport {
6119 reviewer: r.reviewer,
6120 vote,
6121 summary: &r.summary,
6122 findings: &r.findings,
6123 })
6124 })
6125 .collect();
6126
6127 let mut jobs = Vec::new();
6128 let mut seats_at = Vec::new();
6129 for (r, spec) in reviewers.iter().cloned().enumerate() {
6130 if records[r].vote.is_none() {
6134 continue;
6135 }
6136 let wt = root.join(format!("review-{}", r + 1));
6137 let seat_key = format!("review-{}", r + 1);
6138 let spec = self.occupant(&seat_key, spec);
6139 let seat = self.seat(&seat_key, &spec.id);
6140 let build = |with_patch: bool| {
6148 prompt::review_reconsider(&ReviewReconsiderCtx {
6149 instruction: &self.state.instruction,
6150 reviewer: r + 1,
6151 lens: Lens::for_seat(r),
6152 panel: &panel,
6153 patch: with_patch.then_some(ReviewPatch {
6154 branch: &winner.branch,
6155 base_short: &base_short,
6156 stat: &stat,
6157 patch: &patch,
6158 }),
6159 round,
6160 rounds: max_rounds,
6161 language: &language,
6162 })
6163 };
6164 let full = build(true);
6165 let prompt = if has_context(&spec, &seat, sessions) {
6166 build(false)
6167 } else {
6168 full.clone()
6169 };
6170 jobs.push(SeatJob {
6171 prompt,
6172 spec,
6173 seat,
6174 cwd: wt,
6175 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6176 allow_write: false,
6177 sessions,
6178 artifacts: artifacts.clone(),
6179 stem: format!("review-{round}-reconsider-{}", r + 1),
6180 handover: Some(full),
6181 });
6182 seats_at.push(r);
6183 }
6184
6185 let mut recon_quota_losses = Vec::new();
6186 let recon_cache = self.state.config.cache_dir();
6187 let recon_ctx = WaveCtx {
6188 carry_seats: true,
6189 pauses: Some((&self.pause, &self.interrupt)),
6190 run: &run_id,
6191 node: "review",
6192 prompts: &prompts,
6193 cache: recon_cache.as_deref(),
6194 round: Some(round),
6195 };
6196 let recon_results = ask_json_wave::<ReviewRevote>(
6197 jobs,
6198 Arc::clone(&self.sem),
6199 review_retries,
6200 &self.roles.reviewer_roster,
6201 &recon_ctx,
6202 &mut recon_quota_losses,
6203 &mut self.state,
6204 &|_: &ReviewRevote| Ok(()),
6205 )
6206 .await;
6207 if self.state.resource_block.is_some() {
6208 return self.stop_for_resource();
6209 }
6210 self.state.quota.extend(recon_quota_losses);
6211
6212 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6213 let agent_id = seat.agent.clone();
6214 self.state.seats.insert(seat.key.clone(), seat);
6215 let mut rec = ReviewRevoteRecord {
6216 reviewer: r + 1,
6217 agent: agent_id,
6218 vote: None,
6219 reason: String::new(),
6220 failed: None,
6221 };
6222 match res {
6223 Ok((rv, _)) => {
6224 rec.vote = Some(rv.vote);
6225 rec.reason =
6226 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6227 self.state.event(
6228 "review",
6229 format!(
6230 "round {round}: reviewer {} revoted {}",
6231 r + 1,
6232 rv.vote.label()
6233 ),
6234 );
6235 }
6236 Err(e) => {
6237 rec.failed = Some(e.to_string());
6238 self.state.event(
6239 "review",
6240 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6241 );
6242 }
6243 }
6244 reconsideration.push(rec);
6245 }
6246 } else if initial_votes.len() > 1 {
6247 self.state.event(
6248 "review",
6249 format!(
6250 "round {round}: votes agreed ({}) — no reconsideration",
6251 initial_votes[0].label()
6252 ),
6253 );
6254 }
6255
6256 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6257 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6258 let defer_e2e =
6269 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6270 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6271 let reason =
6272 format!("{blocking} blocking finding(s) already required a fix this round");
6273 self.state.event(
6274 "verify",
6275 format!(
6276 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6277 {}); it will run once a round has none left",
6278 short(&head)
6279 ),
6280 );
6281 (Vec::new(), false, true, Some(reason))
6282 } else {
6283 let e2e_commands = self.state.config.verify.e2e.clone();
6284 let cache_dir = self.state.config.cache_dir();
6285 let context = format!("round {round}");
6286 let wait = self.repository_wait();
6287 let (e2e, verify_retried) = with_cache_lease(
6288 &mut self.state,
6289 cache_dir.as_deref(),
6290 "e2e",
6291 "e2e",
6292 &winner.worktree,
6293 &head,
6294 verify_timeout,
6295 &context,
6296 &wait,
6297 |state, budget| {
6298 let shell = shell.clone();
6299 let e2e_commands = e2e_commands.clone();
6300 let worktree = winner.worktree.clone();
6301 let context = context.clone();
6302 async move {
6303 run_e2e_with_retry(
6304 state,
6305 &shell,
6306 &e2e_commands,
6307 &worktree,
6308 budget,
6309 &context,
6310 )
6311 .await
6312 }
6313 },
6314 )
6315 .await?;
6316 (e2e, verify_retried, false, None)
6317 };
6318
6319 let expected = records.len();
6320 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6321 let incomplete = answered < expected;
6322 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6323 let policy = self.state.config.graph.incomplete_review;
6324 let clean = round_is_clean(
6325 blocking,
6326 e2e_ok,
6327 answered,
6328 expected,
6329 round_quota_missing,
6330 policy,
6331 );
6332
6333 let mut round_record = ReviewRound {
6334 round,
6335 head: head.clone(),
6336 verified_head: None,
6337 verified_at: None,
6338 reviews: records,
6339 e2e,
6340 verify_retried,
6341 e2e_deferred,
6342 e2e_defer_reason,
6343 fix: None,
6344 blocking,
6345 answered,
6346 expected,
6347 clean,
6348 progressed: false,
6349 vote_split,
6350 reconsideration,
6351 verdict: None,
6352 };
6353 round_record.verdict = ReviewVote::worst(
6357 round_record
6358 .final_votes()
6359 .into_iter()
6360 .map(|(_, _, vote)| vote),
6361 );
6362 if !matches!(
6373 round_record.e2e_status(),
6374 E2eStatus::Deferred | E2eStatus::NotConfigured
6375 ) {
6376 round_record.verified_head = Some(head.clone());
6377 round_record.verified_at = Some(Timestamp::now());
6378 }
6379 let this_round_verification = round_record.verification_summary(&head);
6380
6381 if incomplete {
6382 let missing: Vec<String> = round_record
6383 .reviews
6384 .iter()
6385 .filter(|r| r.failed.is_some())
6386 .map(|r| format!("review-{}", r.reviewer))
6387 .collect();
6388 self.state.event(
6389 "review",
6390 format!(
6391 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6392 missing.join(", ")
6393 ),
6394 );
6395 }
6396
6397 if clean {
6398 self.state.event(
6399 "review",
6400 if incomplete && policy == IncompleteReviewPolicy::Warn {
6401 format!(
6402 "round {round}: clean (warn policy, incomplete panel) — no \
6403 blocking findings from the seats that answered, verification green"
6404 )
6405 } else if incomplete {
6406 format!(
6407 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6408 quorum) — no blocking findings from the seats that answered, \
6409 verification green",
6410 expected - answered
6411 )
6412 } else {
6413 format!("round {round}: clean — no blocking findings, verification green")
6414 },
6415 );
6416 self.state.reviews.push(round_record);
6417 self.state.status = RunStatus::Gating;
6418 self.state.save()?;
6419 return Ok(());
6420 }
6421
6422 if incomplete && blocking == 0 && e2e_ok {
6430 self.state.reviews.push(round_record);
6431 self.state.save()?;
6432 if round == max_rounds {
6433 self.state.status = RunStatus::Blocked;
6434 self.state.event(
6435 "review",
6436 format!(
6437 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6438 refusing to call it clean",
6439 expected - answered
6440 ),
6441 );
6442 return Ok(());
6443 }
6444 continue;
6445 }
6446
6447 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6459 self.state.reviews.push(round_record);
6460 return self
6461 .stop_reviewing(
6462 "the round's own verification could not run",
6463 &shell,
6464 &winner.worktree,
6465 )
6466 .await;
6467 }
6468
6469 if round == max_rounds {
6470 self.state.reviews.push(round_record);
6471 return self
6472 .stop_reviewing(
6473 &format!(
6474 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6475 ),
6476 &shell,
6477 &winner.worktree,
6478 )
6479 .await;
6480 }
6481
6482 let blocking_findings: Vec<_> = all_findings
6485 .iter()
6486 .filter(|f| f.severity.blocks())
6487 .cloned()
6488 .collect();
6489 let fix_prompt = prompt::fix_in(
6490 &self.state.instruction,
6491 &blocking_findings,
6492 this_round_verification.as_ref(),
6493 round,
6494 max_rounds,
6495 &language,
6496 &self.state.config.graph.github_language,
6497 );
6498 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6499 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6500 let (job, seat, out) = self
6501 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6502 prompt: fix_prompt.clone(),
6503 spec,
6504 seat,
6505 cwd: winner.worktree.clone(),
6506 timeout,
6507 allow_write: true,
6508 sessions,
6509 artifacts: artifacts.clone(),
6510 stem: format!("fix-{round}"),
6511 handover: None,
6512 })
6513 .await;
6514 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6517 return self.stop_for_resource();
6518 }
6519 let agent_id = seat.agent.clone();
6520
6521 let mut fix = FixRecord {
6522 agent: agent_id,
6523 addressed: Vec::new(),
6524 rejected: Vec::new(),
6525 notes: String::new(),
6526 committed: false,
6527 failed: None,
6528 duration_ms: 0,
6529 continuation: None,
6530 };
6531 let mut continuation = ContinuationRecord::not_needed();
6532 let mut final_seat = seat.clone();
6533 match out {
6534 AgentOutcome::Ok(o) => {
6535 fix.duration_ms = o.duration_ms;
6536 let parsed = verdict::extract_json::<FixReport>(&o.text);
6537 let incomplete_reason = match &parsed {
6544 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6545 "the reply parsed, but it reported a command whose own CLI never \
6546 confirmed an exit status"
6547 .to_owned(),
6548 ),
6549 Ok(_) => None,
6550 Err(e) => Some(e.to_string()),
6551 };
6552 match incomplete_reason {
6553 None => {
6554 let report = parsed.expect("checked Ok above");
6555 fix.addressed = report.addressed;
6556 fix.rejected = report.rejected;
6557 fix.notes =
6558 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6559 }
6560 Some(reason) => {
6561 let (resumed_seat, resolved, failure, cont) = self
6562 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6563 .await;
6564 fix.duration_ms += cont.cumulative_wait_ms;
6565 continuation = cont;
6566 final_seat = resumed_seat;
6567 match resolved {
6568 Some(report) => {
6569 fix.addressed = report.addressed;
6570 fix.rejected = report.rejected;
6571 fix.notes = blind::sanitize_prose(
6572 &report.notes,
6573 &self.state.config.blind,
6574 );
6575 }
6576 None => fix.failed = failure,
6577 }
6578 }
6579 }
6580 }
6581 AgentOutcome::Dropped(o) => {
6583 fix.duration_ms = o.duration_ms;
6584 let why = o
6585 .dropped
6586 .as_ref()
6587 .map(|d| d.why.as_str())
6588 .unwrap_or("the CLI ended the stream without delivering its answer");
6589 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6590 }
6591 AgentOutcome::Quota(o) => {
6592 self.state.quota.push(QuotaLoss {
6593 seat: final_seat.key.clone(),
6594 node: "fix".to_owned(),
6595 at: Timestamp::now(),
6596 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6597 });
6598 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6599 }
6600 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6601 }
6602 fix.continuation = Some(continuation);
6603 self.state.seats.insert(final_seat.key.clone(), final_seat);
6604 let coordination = self.coordinate(&winner.worktree).await?;
6605 if let Ok(r) = git::rescue_commit(
6606 &winner.worktree,
6607 &format!("magi: review round {round} fixes (uncommitted work)"),
6608 )
6609 .await
6610 {
6611 self.state.note_withheld("fix", &r.withheld);
6612 }
6613 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6614 fix.committed = after != before;
6615 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6623 drop(coordination);
6624 let progressed = diff_after != patch;
6625 let commit_note = if fix.committed {
6626 "committed"
6627 } else {
6628 "NO new commit"
6629 };
6630 let tree_note = if progressed {
6631 "changed vs base"
6632 } else {
6633 "unchanged vs base"
6634 };
6635 self.state.event(
6636 "fix",
6637 match &fix.failed {
6638 Some(reason) => {
6644 format!(
6645 "round {round}: fixer's adoption report was lost ({reason}); \
6646 {commit_note}, tree {tree_note}"
6647 )
6648 }
6649 None => format!(
6650 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6651 {tree_note}{}",
6652 fix.addressed.len(),
6653 fix.rejected.len(),
6654 if continuation.outcome == ContinuationOutcome::Resumed {
6655 format!(
6656 " (adoption report recovered after {} continuation(s))",
6657 continuation.attempts
6658 )
6659 } else {
6660 String::new()
6661 },
6662 ),
6663 },
6664 );
6665 round_record.fix = Some(fix);
6666 round_record.progressed = progressed;
6667 self.state.reviews.push(round_record);
6668 self.state.save()?;
6669
6670 if matches!(
6683 continuation.outcome,
6684 ContinuationOutcome::Exhausted
6685 | ContinuationOutcome::QuotaLost
6686 | ContinuationOutcome::NoSession
6687 ) {
6688 return self
6689 .stop_reviewing(
6690 "the fixer's adoption report never came back, even after resuming its \
6691 own seat; refusing to start another round against the same worktree \
6692 while that is unresolved",
6693 &shell,
6694 &winner.worktree,
6695 )
6696 .await;
6697 }
6698
6699 let streak = self
6700 .state
6701 .reviews
6702 .iter()
6703 .rev()
6704 .take_while(|r| !r.progressed)
6705 .count();
6706 if streak >= STAGNANT_LIMIT {
6707 return self
6708 .stop_reviewing(
6709 &format!(
6710 "the tree has not moved against base for {streak} round(s) in a row"
6711 ),
6712 &shell,
6713 &winner.worktree,
6714 )
6715 .await;
6716 }
6717 }
6718 Ok(())
6719 }
6720
6721 fn record_contested_handoff(&mut self) {
6755 if self.state.contested_handoff.is_some() {
6756 return;
6757 }
6758 let Some(contested) = self
6759 .state
6760 .reviews
6761 .last()
6762 .and_then(ReviewRound::contested_handoff)
6763 else {
6764 return;
6765 };
6766 self.state.event(
6767 "review",
6768 format!(
6769 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6770 wait for the owner's approval",
6771 contested.findings.len(),
6772 contested.rejecters.len()
6773 ),
6774 );
6775 self.state.contested_handoff = Some(contested);
6776 }
6777
6778 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6779 let round_idx = self.state.reviews.len() - 1;
6780 let needs_catchup_run = matches!(
6788 self.state.reviews[round_idx].e2e_status(),
6789 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6790 );
6791 if needs_catchup_run {
6792 let round = self.state.reviews[round_idx].round;
6793 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6794 let commands = self.state.config.verify.e2e.clone();
6795 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6796 let cache_dir = self.state.config.cache_dir();
6797 let context = format!(
6798 "round {round}: verification unresolved, catching up before the final decision"
6799 );
6800 let wait = self.repository_wait();
6801 let (outcomes, verify_retried) = with_cache_lease(
6802 &mut self.state,
6803 cache_dir.as_deref(),
6804 "e2e",
6805 "e2e",
6806 worktree,
6807 &attempted_head,
6808 timeout,
6809 &context,
6810 &wait,
6811 |state, budget| {
6812 let shell = shell.to_vec();
6813 let commands = commands.clone();
6814 let context = context.clone();
6815 async move {
6816 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6817 .await
6818 }
6819 },
6820 )
6821 .await?;
6822 let last = &mut self.state.reviews[round_idx];
6823 last.e2e = outcomes;
6824 last.verify_retried = verify_retried;
6825 last.verified_head = Some(attempted_head);
6832 last.verified_at = Some(Timestamp::now());
6833 if verify_inconclusive(&last.e2e) {
6834 self.state.save()?;
6841 return Ok(());
6842 }
6843 last.e2e_deferred = false;
6844 }
6845 let last = &self.state.reviews[round_idx];
6846 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6847
6848 match last.e2e_status() {
6849 E2eStatus::Failed => {
6850 let red: Vec<String> = last
6851 .e2e
6852 .iter()
6853 .filter(|o| !o.ok())
6854 .map(|o| {
6855 format!(
6856 "`{}` -> {:?}\n{}",
6857 o.command,
6858 o.code,
6859 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6860 )
6861 })
6862 .collect();
6863 self.state
6864 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6865 self.state.status = RunStatus::Blocked;
6866 }
6867 E2eStatus::ResourceBlocked => {
6872 self.state.event(
6873 "review",
6874 format!(
6875 "{why}; e2e could not run (shared build cache unavailable); not \
6876 deciding yet"
6877 ),
6878 );
6879 }
6880 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6881 self.state.event(
6882 "review",
6883 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6884 );
6885 self.record_contested_handoff();
6886 self.state.status = RunStatus::Gating;
6887 }
6888 }
6889 self.state.save()?;
6890 Ok(())
6891 }
6892
6893 async fn gate(&mut self) -> Result<()> {
6896 if self.state.status == RunStatus::Failed
6908 || self
6909 .state
6910 .base_sync
6911 .as_ref()
6912 .is_some_and(|s| s.conflict.is_some())
6913 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6914 != Some(RunStatus::Gating)
6915 {
6916 return Ok(());
6917 }
6918 if self.state.gate_ran {
6919 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6930 self.state.status = RunStatus::Blocked;
6931 self.state.save()?;
6932 }
6933 return Ok(());
6934 }
6935 let Some(winner) = self.state.winner().cloned() else {
6936 return Ok(());
6937 };
6938 self.state.status = RunStatus::Gating;
6939 let mut outcomes = self.run_gate(&winner).await?;
6940 loop {
6941 if verify_inconclusive(&outcomes) {
6952 self.state.save()?;
6953 return Ok(());
6954 }
6955 if outcomes.iter().all(CommandOutcome::ok) {
6956 break;
6957 }
6958 match self.gate_fix_round(&winner, &outcomes).await? {
6959 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6960 GateFix::Stop => break,
6961 GateFix::Defer => {
6962 self.state.save()?;
6963 return Ok(());
6964 }
6965 }
6966 }
6967 let passed = outcomes.iter().all(CommandOutcome::ok);
6968 self.state.gate = outcomes;
6969 self.state.gate_ran = true;
6970 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6971 if !passed {
6972 self.state.status = RunStatus::Blocked;
6973 let spent = self.state.gate_fixes.len();
6974 self.state.event(
6975 "gate",
6976 if spent == 0 {
6977 "gate failed; not merging".to_owned()
6978 } else {
6979 format!("gate failed after {spent} gate-fix round(s); not merging")
6980 },
6981 );
6982 }
6983 self.state.save()?;
6984 Ok(())
6985 }
6986
6987 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6997 let commands = state.config.verify.pre_gate.clone();
6998 if commands.is_empty() {
6999 return;
7000 }
7001 let shell = state.config.shell();
7002 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7003 let (outcomes, _) = run_commands(
7004 state,
7005 "pre_gate",
7006 "pre_gate",
7007 0,
7008 &shell,
7009 &commands,
7010 &winner.worktree,
7011 timeout,
7012 )
7013 .await;
7014 for o in &outcomes {
7015 if !o.ok() {
7016 tracing::warn!(
7017 "pre_gate `{}` failed ({:?}); the gate decides",
7018 o.command,
7019 o.code
7020 );
7021 }
7022 state.event(
7023 "pre_gate",
7024 format!(
7025 "`{}` -> {}",
7026 o.command,
7027 if o.ok() {
7028 "pass".to_owned()
7029 } else {
7030 format!(
7031 "FAIL ({:?})\n{}",
7032 o.code,
7033 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7034 )
7035 }
7036 ),
7037 );
7038 }
7039 state.pre_gate = outcomes;
7040 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7041 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7042 Ok(head) => {
7043 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7044 state.pre_gate_commit = Some(head);
7045 }
7046 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7047 },
7048 Ok(false) => {}
7049 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7050 }
7051 if let Err(e) = state.save() {
7052 tracing::warn!("could not persist the pre_gate record: {e:#}");
7053 }
7054 }
7055
7056 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7059 let wait = self.repository_wait();
7060 run_landing_gate(&mut self.state, winner, &wait).await
7061 }
7062
7063 async fn gate_fix_round(
7076 &mut self,
7077 winner: &Candidate,
7078 outcomes: &[CommandOutcome],
7079 ) -> Result<GateFix> {
7080 let cap = self.state.config.graph.gate_fix_rounds;
7081 let spent = self.state.gate_fixes.len();
7082 if spent >= cap {
7083 if cap > 0 {
7084 self.state.event(
7085 "gate",
7086 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7087 );
7088 }
7089 return Ok(GateFix::Stop);
7090 }
7091 if !gate_fixable(outcomes) {
7092 self.state.event(
7093 "gate",
7094 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7095 command or similar); not spending a fix round on it",
7096 );
7097 return Ok(GateFix::Stop);
7098 }
7099 let min_free = self.state.config.disk.min_free_bytes;
7100 if min_free > 0 {
7101 match crate::disk::free_bytes(&winner.worktree) {
7102 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7103 Ok(free) => {
7104 self.state.event(
7105 "gate",
7106 format!(
7107 "only {free} bytes free ({min_free} required by `[disk] \
7108 min_free_bytes`); not spending a fix round on a failure the disk \
7109 may explain"
7110 ),
7111 );
7112 return Ok(GateFix::Stop);
7113 }
7114 Err(e) => {
7115 self.state.event(
7116 "gate",
7117 format!("free disk space could not be measured ({e:#}); no fix round"),
7118 );
7119 return Ok(GateFix::Stop);
7120 }
7121 }
7122 }
7123
7124 let attempt = spent + 1;
7125 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7126 let base = self.landing_base();
7127 let fix_prompt = prompt::gate_fix_in(
7128 &self.state.instruction,
7129 &failed,
7130 attempt,
7131 cap,
7132 &self.state.config.graph.language,
7133 &self.state.config.graph.github_language,
7134 );
7135 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7136 let sessions = self.state.config.graph.sessions;
7137 let artifacts = self.merge_repair_artifacts();
7138 self.state.event(
7139 "gate",
7140 format!("gate failed; gate-fix round {attempt} of {cap}"),
7141 );
7142 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7143 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7144 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7147 head: before.clone(),
7148 passed: false,
7149 outcomes: Vec::new(),
7150 });
7151 self.state.save()?;
7152 let (_, seat, out) = self
7153 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7154 prompt: fix_prompt.clone(),
7155 spec,
7156 seat,
7157 cwd: winner.worktree.clone(),
7158 timeout,
7159 allow_write: true,
7160 sessions,
7161 artifacts: artifacts.clone(),
7162 stem: format!("gate-fix-{attempt}"),
7163 handover: None,
7164 })
7165 .await;
7166 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7169 return Ok(GateFix::Defer);
7170 }
7171 let mut record = GateFixRecord {
7172 agent: seat.agent.clone(),
7173 failed,
7174 notes: String::new(),
7175 committed: false,
7176 error: None,
7177 };
7178 match out {
7179 AgentOutcome::Ok(o) => {
7180 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7183 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7184 }
7185 }
7186 AgentOutcome::Dropped(_) => {
7187 record.error = Some("the CLI dropped the stream".to_owned());
7188 }
7189 AgentOutcome::Quota(o) => {
7190 self.state.quota.push(QuotaLoss {
7191 seat: seat.key.clone(),
7192 node: "gate-fix".to_owned(),
7193 at: Timestamp::now(),
7194 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7195 });
7196 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7197 }
7198 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7199 }
7200 self.state.seats.insert(seat.key.clone(), seat);
7201 let coordination = self.coordinate(&winner.worktree).await?;
7202 if let Ok(r) = git::rescue_commit(
7203 &winner.worktree,
7204 &format!("magi: gate fix {attempt} (uncommitted work)"),
7205 )
7206 .await
7207 {
7208 self.state.note_withheld("gate-fix", &r.withheld);
7209 }
7210 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7211 record.committed = after != before;
7212 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7213 let note = record.error.clone();
7214 self.state.gate_fixes.push(record);
7215 self.state.save()?;
7216 if !changed {
7217 self.state.event(
7218 "gate",
7219 match note {
7220 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7221 None => format!("gate-fix round {attempt}: the tree did not change"),
7222 },
7223 );
7224 return Ok(GateFix::Stop);
7225 }
7226 self.state.event(
7227 "gate",
7228 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7229 );
7230
7231 drop(coordination);
7232 let wait = self.repository_wait();
7233 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7234 }
7235
7236 async fn guarded_pr_message(
7243 &mut self,
7244 winner: &Candidate,
7245 facts: Option<&BranchFacts>,
7246 posting: bool,
7247 ) -> Result<Option<PrMessage>> {
7248 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7249 let head = match final_head.as_deref() {
7250 Some(h) => crate::pr_outcome::FinalHead::At(h),
7251 None => crate::pr_outcome::FinalHead::Unknown,
7252 };
7253 let summarized = if posting {
7255 Some(self.summarize_change(winner, final_head.as_deref()).await)
7256 } else {
7257 None
7258 };
7259 if self.state.resource_block.is_some() {
7264 self.state.event(
7265 "pr-summary",
7266 "a shared resource was busy; nothing was published, resume to summarize again",
7267 );
7268 return Ok(None);
7269 }
7270 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7271 anyhow::bail!(
7272 "the final commit moved during public description generation; retry before publishing"
7273 );
7274 }
7275 let note = match (&summarized, &self.state.change_summary) {
7276 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7277 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7278 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7279 (None, _) => ChangeNote::NotAsked,
7280 };
7281 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7282 if !posting {
7283 let identity = crate::scrub::Identity::current();
7284 return Ok(Some(PrMessage {
7285 title: crate::scrub::scrub(&pr.title, &identity),
7286 body: crate::scrub::scrub(&pr.body, &identity),
7287 }));
7288 }
7289 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7292 if let Some(known) = self
7293 .state
7294 .github_text
7295 .clone()
7296 .filter(|g| g.fingerprint == raw_fp)
7297 {
7298 return self.settle_withheld(winner, &pr, known).await;
7299 }
7300 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7301 let mut violations =
7302 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7303 let identity = crate::scrub::Identity::current();
7304 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7305 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7306 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7307 {
7308 violations.push(crate::github_text::Violation::SensitiveData);
7309 }
7310 let has_language = violations
7314 .iter()
7315 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7316 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7317 self.state.event(
7318 "github-text",
7319 if has_language {
7320 format!("description rejected: {violations:?}; requesting one rewrite")
7321 } else {
7322 format!("description rejected: {violations:?}; asking the owner")
7323 },
7324 );
7325 let mut rewritten = false;
7326 let mut rejected_rewrite = None;
7327 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7328 {
7329 let output = text;
7330 if let Some(title) = summary_title(&output) {
7331 let raw_body = summary_without_title(&output);
7334 let clean_title = crate::scrub::scrub(&title, &identity);
7335 let mut body = crate::scrub::scrub(&raw_body, &identity);
7336 let redacted = clean_title != title || body != raw_body;
7337 let title = clean_title;
7338 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7339 if !complete {
7340 self.state.event(
7341 "github-text",
7342 "description rewrite omitted required findings",
7343 );
7344 }
7345 if !body.trim().is_empty() {
7346 body.push_str("\n\n");
7347 body.push_str(&public_pr_evidence(&self.state, head));
7348 }
7349 let marker = format!("magi:run/{}", self.state.id);
7350 if !body.trim().is_empty() && !body.contains(&marker) {
7351 body.push_str(&format!("\n\n{marker}"));
7352 }
7353 let rewrite = PrMessage {
7354 title: title.clone(),
7355 body: body.clone(),
7356 };
7357 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7358 let mut failed = crate::github_text::check_for(
7359 &self.state.config.graph,
7360 &title,
7361 &body,
7362 redecision,
7363 );
7364 if (!crate::github_text::shareable(&title)
7365 || !crate::github_text::shareable(&body))
7366 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7367 {
7368 failed.push(crate::github_text::Violation::SensitiveData);
7369 }
7370 if complete
7371 && !title.trim().is_empty()
7372 && !body.trim().is_empty()
7373 && !failed.is_empty()
7374 {
7375 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7376 title: title.clone(),
7377 body: body.clone(),
7378 failed: failed
7379 .iter()
7380 .map(|v| crate::github_text::category(*v).to_owned())
7381 .collect(),
7382 });
7383 }
7384 if complete
7385 && !title.trim().is_empty()
7386 && !raw_body.trim().is_empty()
7387 && failed.is_empty()
7388 {
7389 decision = redecision;
7390 pr = PrMessage { title, body };
7391 rewritten = true;
7392 self.state
7393 .event("github-text", "description rewrite passed");
7394 if redacted {
7395 self.state.event(
7396 "github-text",
7397 "sensitive data removed from description rewrite",
7398 );
7399 }
7400 }
7401 }
7402 }
7403 if !rewritten {
7404 self.state.event(
7405 "github-text",
7406 "description rewrite unavailable or rejected; asking the owner",
7407 );
7408 if let Some(w) =
7409 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7410 {
7411 self.retire_github_text_question("the withheld text changed");
7414 let record = crate::run::GithubTextAsk {
7415 rewrite: rejected_rewrite,
7416 fingerprint: raw_fp,
7417 title: w.title,
7418 body: w.body,
7419 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7420 question: None,
7421 asks: 0,
7422 resolved: false,
7423 chosen_title: None,
7424 choice: None,
7425 };
7426 return self.settle_withheld(winner, &pr, record).await;
7427 }
7428 }
7429 }
7430 let (title, body) =
7431 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7432 Ok(Some(PrMessage { title, body }))
7433 }
7434
7435 async fn summarize_change(
7446 &mut self,
7447 winner: &Candidate,
7448 head: Option<&str>,
7449 ) -> std::result::Result<(), String> {
7450 let Some(head) = head else {
7451 return Err("the final commit could not be read".to_owned());
7452 };
7453 let remote = self.state.config.merge.remote.clone();
7454 let tracking = format!("{remote}/{}", self.state.base_branch);
7455 if !git::rev_exists(&winner.worktree, &tracking).await {
7456 return Err(format!("the remote base `{tracking}` could not be read"));
7457 }
7458 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7459 Ok(b) if !b.is_empty() => b,
7460 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7461 };
7462 let evidence = crate::pr_outcome::summary_evidence(
7463 &self.state,
7464 crate::pr_outcome::FinalHead::At(head),
7465 );
7466 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7467 if self
7468 .state
7469 .change_summary
7470 .as_ref()
7471 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7472 {
7473 return Ok(());
7474 }
7475 if self.state.change_summary.take().is_some() {
7476 self.state.event(
7477 "pr-summary",
7478 "the final diff or evidence changed; the earlier change summary was discarded",
7479 );
7480 }
7481 if let Err(e) = write_artifact(
7483 &self.state,
7484 &format!("public-description-{fingerprint}.md"),
7485 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7486 ) {
7487 tracing::warn!("could not save public description evidence: {e:#}");
7488 }
7489 let chain = match agent::pick_chain(
7490 &self.state.config.agents,
7491 self.state.config.roles.synthesizer.as_ref(),
7492 &agent::installed,
7493 "synthesizer",
7494 ) {
7495 Ok(chain) => chain,
7496 Err(e) => {
7497 tracing::warn!("no agent for the change summary: {e:#}");
7498 self.state.event("pr-summary", "no summary agent available");
7499 return Err("no agent was available to summarize it".to_owned());
7500 }
7501 };
7502 let subjects = git::subjects(&winner.worktree, &base, head)
7503 .await
7504 .unwrap_or_default();
7505 let stat = git::diff_stat(&winner.worktree, &base, head)
7506 .await
7507 .unwrap_or_default();
7508 let stat: String = stat.chars().take(6_000).collect();
7509 let mut prompt = crate::prompt::change_summary_in(
7510 &base,
7511 head,
7512 &subjects,
7513 stat.trim(),
7514 &self.state.config.graph.github_language,
7515 );
7516 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7517 let sessions = self.state.config.graph.sessions;
7518 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7519 let prompts = self.state.config.prompts.clone();
7520 let cache = self.state.config.cache_dir();
7521 let run = self.state.id.clone();
7522 for (i, spec) in chain.into_iter().enumerate() {
7523 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7524 let job = SeatJob {
7525 spec: spec.clone(),
7526 seat,
7527 cwd: winner.worktree.clone(),
7528 prompt: prompt.clone(),
7529 timeout,
7530 allow_write: false,
7531 sessions,
7532 artifacts: agent::artifacts_dir(&self.state.dir()),
7533 stem: if i == 0 {
7534 "change-summary".to_owned()
7535 } else {
7536 format!("change-summary-{}", spec.id)
7537 },
7538 handover: None,
7539 };
7540 let ctx = WaveCtx {
7541 run: &run,
7542 node: "pr-summary",
7543 prompts: &prompts,
7544 cache: cache.as_deref(),
7545 round: None,
7546 carry_seats: false,
7547 pauses: Some((&self.pause, &self.interrupt)),
7548 };
7549 let (seat, outcome) =
7550 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let text = match outcome {
7553 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7554 AgentOutcome::ResourceBlocked(why) => {
7555 if self.state.resource_block.is_none() {
7558 self.state.resource_block = Some(crate::run::ResourceBlock {
7559 node: "pr-summary".to_owned(),
7560 kind: "shared resource".to_owned(),
7561 detail: why,
7562 at: Timestamp::now(),
7563 });
7564 }
7565 return Err("a shared resource was busy".to_owned());
7566 }
7567 _ => continue,
7568 };
7569 let blind = &self.state.config.blind;
7570 let Some(title) = summary_title(&text) else {
7571 continue;
7572 };
7573 let body: String = summary_without_title(&text)
7574 .lines()
7575 .filter(|l| !l.contains("magi:run/"))
7576 .collect::<Vec<_>>()
7577 .join("\n");
7578 let body = blind::sanitize_prose(body.trim(), blind);
7579 let title = blind::sanitize_prose(&title, blind);
7580 if body.trim().is_empty() || title.trim().is_empty() {
7581 continue;
7582 }
7583 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7584 continue;
7585 }
7586 if git::rev_parse(&winner.worktree, "HEAD")
7588 .await
7589 .ok()
7590 .as_deref()
7591 != Some(head)
7592 {
7593 return Err("the branch moved while the summary was written".to_owned());
7594 }
7595 if git::merge_base(&winner.worktree, &tracking, head)
7596 .await
7597 .ok()
7598 .as_deref()
7599 != Some(base.as_str())
7600 || crate::pr_outcome::summary_evidence(
7601 &self.state,
7602 crate::pr_outcome::FinalHead::At(head),
7603 ) != evidence
7604 {
7605 return Err("the summary evidence changed while it was written".to_owned());
7606 }
7607 self.state.change_summary = Some(crate::run::ChangeSummary {
7608 head: head.to_owned(),
7609 base,
7610 evidence: fingerprint,
7611 title,
7612 body,
7613 });
7614 self.state.event(
7615 "pr-summary",
7616 format!("change summary written by {} for the final diff", spec.id),
7617 );
7618 if let Err(e) = self.state.save() {
7619 tracing::warn!("could not save the change summary: {e:#}");
7620 }
7621 return Ok(());
7622 }
7623 self.state
7624 .event("pr-summary", "no agent produced a change summary");
7625 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7626 }
7627
7628 async fn rewrite_pr_text(
7636 &mut self,
7637 winner: &Candidate,
7638 pr: &PrMessage,
7639 violations: &[crate::github_text::Violation],
7640 ) -> Option<String> {
7641 let prompt = format!(
7642 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7643 pr.title, pr.body
7644 );
7645 let sessions = self.state.config.graph.sessions;
7646 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7647 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7649 let key = format!("impl-{}", winner.label);
7650 if let Some(spec) = self
7651 .state
7652 .config
7653 .agents
7654 .iter()
7655 .find(|a| a.id == winner.agent)
7656 .cloned()
7657 && let Some(existing) = self.state.seats.get(&key)
7658 && existing.agent == spec.id
7659 && agent::has_session(spec.kind, existing, sessions)
7660 {
7661 attempts.push((spec, existing.clone(), true));
7662 } else {
7663 let chain = agent::pick_chain(
7664 &self.state.config.agents,
7665 self.state.config.roles.language_judge.as_ref(),
7666 &agent::installed,
7667 "language_judge",
7668 );
7669 match chain {
7670 Ok(chain) => {
7671 for spec in chain {
7672 let seat = crate::fixer::seat_for(
7673 &mut self.state,
7674 "github-text-rewrite",
7675 &spec.id,
7676 );
7677 attempts.push((spec, seat, false));
7678 }
7679 }
7680 Err(e) => {
7681 tracing::warn!("no agent for the description rewrite: {e:#}");
7682 self.state
7683 .event("github-text", "no rewrite agent available");
7684 return None;
7685 }
7686 }
7687 }
7688 let prompts = self.state.config.prompts.clone();
7689 let cache = self.state.config.cache_dir();
7690 let run = self.state.id.clone();
7691 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7692 let job = SeatJob {
7693 spec: spec.clone(),
7694 seat,
7695 cwd: winner.worktree.clone(),
7696 prompt: prompt.clone(),
7697 timeout: retry_budget(timeout_full, resumed),
7698 allow_write: false,
7699 sessions,
7700 artifacts: agent::artifacts_dir(&self.state.dir()),
7701 stem: if i == 0 {
7702 "github-text-rewrite".to_owned()
7703 } else {
7704 format!("github-text-rewrite-{}", spec.id)
7705 },
7706 handover: None,
7707 };
7708 let ctx = WaveCtx {
7709 run: &run,
7710 node: "github-text",
7711 prompts: &prompts,
7712 cache: cache.as_deref(),
7713 round: None,
7714 carry_seats: false,
7715 pauses: Some((&self.pause, &self.interrupt)),
7716 };
7717 let (seat, outcome) =
7718 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7719 self.state.seats.insert(seat.key.clone(), seat);
7720 match outcome {
7721 AgentOutcome::Ok(o)
7722 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7723 {
7724 return Some(o.text);
7725 }
7726 AgentOutcome::ResourceBlocked(_) => return None,
7727 _ => {}
7728 }
7729 }
7730 None
7731 }
7732
7733 fn retire_github_text_question(&mut self, why: &str) {
7735 let Some(id) = self
7736 .state
7737 .github_text
7738 .as_ref()
7739 .and_then(|g| g.question.clone())
7740 else {
7741 return;
7742 };
7743 let _ = ask::Questions::open().update(&id, |q| {
7744 q.abandon(why);
7745 Ok(())
7746 });
7747 }
7748
7749 async fn settle_withheld(
7753 &mut self,
7754 winner: &Candidate,
7755 pr: &PrMessage,
7756 mut g: crate::run::GithubTextAsk,
7757 ) -> Result<Option<PrMessage>> {
7758 use crate::github_text as gt;
7759 let store = ask::Questions::open();
7760 let timeout = self.state.config.graph.answer_timeout;
7761 if !g.resolved {
7762 let mut standing = g
7763 .question
7764 .as_ref()
7765 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7766 if let Some(q) = standing.as_ref()
7767 && q.status.open()
7768 && gt::expired(q, timeout)
7769 {
7770 let why = format!("no answer within {}s of asking", timeout.max(1));
7771 standing = store
7772 .update(&q.id, |q| {
7773 q.abandon(&why);
7774 Ok(())
7775 })
7776 .ok()
7777 .map(|(q, ())| q);
7778 }
7779 let mut ask_again: Option<Option<Vec<String>>> = None;
7780 match standing {
7781 None if g.asks <= 1 => {
7784 g.asks = 0;
7785 ask_again = Some(None);
7786 }
7787 None => {
7788 g.resolved = true;
7789 g.chosen_title = None;
7790 g.choice = Some(GithubTextChoice::Fallback);
7791 }
7792 Some(q) if q.status.open() => {
7793 self.park_on_github_text(&g)?;
7794 return Ok(None);
7795 }
7796 Some(q) => match gt::read_reply(&q) {
7797 gt::Reply::Fallback => {
7798 g.resolved = true;
7799 g.chosen_title = None;
7800 g.choice = Some(GithubTextChoice::Fallback);
7801 }
7802 gt::Reply::Original => {
7803 g.resolved = true;
7804 g.chosen_title = None;
7805 g.choice = Some(GithubTextChoice::Original);
7806 }
7807 gt::Reply::Rewrite => {
7808 g.resolved = true;
7809 g.chosen_title = None;
7810 g.choice = Some(if g.rewrite.is_some() {
7811 GithubTextChoice::Rewrite
7812 } else {
7813 GithubTextChoice::Fallback
7814 });
7815 }
7816 gt::Reply::Title(text) => {
7817 let first = text
7818 .lines()
7819 .map(str::trim)
7820 .find(|l| !l.is_empty())
7821 .unwrap_or("");
7822 let decision = if first.is_empty() {
7823 None
7824 } else {
7825 let candidate = PrMessage {
7826 title: first.to_owned(),
7827 body: String::new(),
7828 };
7829 self.judge_pr_language(&winner.worktree, &candidate).await
7830 };
7831 match gt::vet_title(
7832 &text,
7833 decision,
7834 self.state.config.graph.github_english(),
7835 ) {
7836 Ok(title) => {
7837 g.resolved = true;
7838 g.chosen_title = Some(title);
7839 g.choice = Some(GithubTextChoice::MyTitle);
7840 }
7841 Err(cats) if g.asks < 2 => {
7842 ask_again = Some(Some(
7843 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7844 ))
7845 }
7846 Err(_) => {
7847 g.resolved = true;
7848 g.chosen_title = None;
7849 g.choice = Some(GithubTextChoice::Fallback);
7850 }
7851 }
7852 }
7853 },
7854 }
7855 if let Some(rejected) = ask_again {
7856 let w = gt::Withheld {
7859 title: g.title,
7860 body: g.body,
7861 categories: g.categories.clone(),
7862 };
7863 let artifact = match crate::run::write_artifact(
7866 &self.state,
7867 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7868 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7869 ) {
7870 Ok(p) => Some(p),
7871 Err(e) => {
7872 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7873 None
7874 }
7875 };
7876 let (detail, comparison) = gt::question_content(
7877 &w,
7878 &pr.title,
7879 &pr.body,
7880 &self.state.id,
7881 rejected.as_deref(),
7882 artifact.as_deref(),
7883 g.rewrite.as_ref(),
7884 );
7885 let mut q = ask::Question::new(
7886 self.state.id.clone(),
7887 gt::ASK_NODE.to_owned(),
7888 gt::ASK_SEAT.to_owned(),
7889 gt::question_summary(&self.state.config.graph.language, &w),
7890 detail,
7891 gt::question_choices(&w, g.rewrite.as_ref()),
7892 );
7893 q.posting_comparison = Some(comparison);
7894 q.answer_timeout = timeout;
7895 g.question = Some(q.id.clone());
7898 g.asks += 1;
7899 self.state.github_text = Some(g.clone());
7900 self.state.event(
7901 "github-text",
7902 format!("asking the owner about the withheld text ({})", q.short()),
7903 );
7904 self.state.save()?;
7905 store
7906 .put_executable(&mut q)
7907 .context("file the posting-gate question")?;
7908 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7909 tracing::warn!(
7910 "could not notify about posting-gate question {}: {e:#}",
7911 q.short()
7912 );
7913 }
7914 self.park_on_github_text(&g)?;
7915 return Ok(None);
7916 }
7917 }
7918 let resolved = g.chosen_title.clone();
7919 let original = g.choice == Some(GithubTextChoice::Original);
7920 self.state.event(
7922 "github-text",
7923 if original {
7924 format!(
7925 "posting the original text for: {}",
7926 match (g.title, g.body) {
7927 (true, true) => "title, description",
7928 (true, false) => "title",
7929 _ => "description",
7930 }
7931 )
7932 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7933 "posting the saved rewrite title and description".to_owned()
7934 } else if resolved.is_some() {
7935 "posting the owner's replacement title".to_owned()
7936 } else {
7937 "posting the neutral text for the withheld fields".to_owned()
7938 },
7939 );
7940 self.state.github_text = Some(g.clone());
7941 self.state.save()?;
7942 if g.choice == Some(GithubTextChoice::Rewrite)
7943 && let Some(r) = g.rewrite.as_ref()
7944 {
7945 return Ok(Some(PrMessage {
7946 title: r.title.clone(),
7947 body: r.body.clone(),
7948 }));
7949 }
7950 let w = gt::Withheld {
7951 title: g.title,
7952 body: g.body,
7953 categories: g.categories.clone(),
7954 };
7955 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7956 if let Some(t) = resolved {
7957 title = t;
7958 }
7959 let english = gt::LanguageDecision {
7961 title_english: true,
7962 body_english: true,
7963 body_complete: true,
7964 };
7965 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7966 if original {
7967 if g.title {
7970 title = pr.title.clone();
7971 }
7972 if g.body {
7973 body = pr.body.clone();
7974 }
7975 }
7976 Ok(Some(PrMessage { title, body }))
7977 }
7978
7979 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7980 self.state.github_text = Some(g.clone());
7981 self.state.end_pass_parked(ParkCause::Question);
7982 self.state.parked = true;
7983 self.state.event(
7984 "github-text",
7985 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7986 );
7987 self.state.save()
7988 }
7989
7990 async fn judge_pr_language(
7993 &mut self,
7994 cwd: &Path,
7995 pr: &PrMessage,
7996 ) -> Option<crate::github_text::LanguageDecision> {
7997 self.state.config.roles.language_judge.as_ref()?;
7998 if !self.state.config.graph.language_gate() {
7999 return None;
8000 }
8001 let decision =
8002 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8003 self.state.event(
8004 "github-text",
8005 if decision.is_some() {
8006 "language decided by the language judge"
8007 } else {
8008 "language judge unavailable; using built-in heuristics"
8009 },
8010 );
8011 decision
8012 }
8013
8014 async fn merge(&mut self) -> Result<()> {
8015 if self
8030 .state
8031 .base_sync
8032 .as_ref()
8033 .is_some_and(|s| s.conflict.is_some())
8034 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8035 != Some(RunStatus::Gating)
8036 || !self.state.gate_allows_progress()
8044 {
8045 return Ok(());
8046 }
8047 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8056 return Ok(());
8057 }
8058 let Some(winner) = self.state.winner().cloned() else {
8059 return Ok(());
8060 };
8061 let repo = self.state.repo.clone();
8062 let base = self.state.base_branch.clone();
8063 let mode = self.state.config.merge.mode;
8064 let style = self.state.config.merge.style;
8065 let facts = if is_review_run(&self.state) {
8066 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8067 let start = review_base(
8068 &repo,
8069 &self.state.config.merge.remote,
8070 &base,
8071 &self.state.base_commit,
8072 &winner.branch,
8073 )
8074 .await;
8075 branch_facts(&repo, &start, &winner.branch).await
8076 } else {
8077 None
8078 };
8079 let leaked = if is_review_run(&self.state) {
8082 leaked_subjects(&self.state, &winner.branch).await
8083 } else {
8084 Some(Vec::new())
8085 };
8086 let Some(pr) = self
8090 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8091 .await?
8092 else {
8093 return Ok(());
8094 };
8095 let message = pr.commit_message();
8096
8097 let outcome = match mode {
8098 MergeMode::None => MergeOutcome {
8099 mode,
8100 ok: true,
8101 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8102 empty: false,
8103 },
8104 MergeMode::Pr | MergeMode::Local
8105 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8106 {
8107 MergeOutcome {
8108 mode,
8109 ok: false,
8110 detail: empty_candidate_detail(&self.state, &base),
8111 empty: true,
8112 }
8113 }
8114 MergeMode::Local => {
8115 let on = git::current_branch(&repo).await?;
8116 if on.as_deref() != Some(base.as_str()) {
8117 MergeOutcome {
8118 mode,
8119 ok: false,
8120 detail: format!(
8121 "{} has {} checked out, not the base branch {base}",
8122 repo.display(),
8123 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8124 ),
8125 empty: false,
8126 }
8127 } else if !git::is_clean(&repo).await? {
8128 MergeOutcome {
8129 mode,
8130 ok: false,
8131 detail: format!("{} is dirty; refusing to merge", repo.display()),
8132 empty: false,
8133 }
8134 } else {
8135 let out = match style {
8136 MergeStyle::Merge => {
8137 git::merge_no_ff(&repo, &winner.branch, &message).await?
8138 }
8139 MergeStyle::Squash => {
8140 git::merge_squash(&repo, &winner.branch, &message).await?
8141 }
8142 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8143 };
8144 MergeOutcome {
8145 mode,
8146 ok: out.ok(),
8147 detail: if out.ok() { out.stdout } else { out.stderr },
8148 empty: false,
8149 }
8150 }
8151 }
8152 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8153 self.state.pr_publication_pending = true;
8154 self.state.save()?;
8155 let out =
8156 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8157 self.state.pr_publication_pending = out.is_err();
8158 MergeOutcome {
8159 mode,
8160 ok: out.is_ok(),
8161 detail: match out {
8162 Ok(url) => url,
8163 Err(e) => format!("{e:#}"),
8164 },
8165 empty: false,
8166 }
8167 }
8168 MergeMode::Pr => {
8169 let remote = self.state.config.merge.remote.clone();
8170 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8171 if !pushed.ok() {
8172 MergeOutcome {
8173 mode,
8174 ok: false,
8175 detail: pushed.stderr,
8176 empty: false,
8177 }
8178 } else {
8179 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8185 let out = match pr_merge_plan(found) {
8186 PrPlan::Create => {
8187 gh_pr_create(
8188 &winner.worktree,
8189 &base,
8190 &winner.branch,
8191 &pr.title,
8192 &pr.body,
8193 )
8194 .await
8195 }
8196 PrPlan::Adopt { url, title } => {
8197 self.state
8198 .event("merge", format!("Pr: adopted open pull request {url}"));
8199 if self.state.github_text.as_ref().is_some_and(|g| {
8200 g.resolved
8201 && g.choice == Some(GithubTextChoice::Rewrite)
8202 && g.rewrite.is_some()
8203 }) {
8204 land::set_pr_text_verbatim(
8207 &winner.worktree,
8208 &url,
8209 &pr.title,
8210 &pr.body,
8211 )
8212 .await
8213 .map(|()| url)
8214 } else {
8215 if title != pr.title
8216 && (!is_review_run(&self.state)
8217 || leaked
8218 .as_deref()
8219 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8220 && let Err(e) =
8221 if self.state.github_text.as_ref().is_some_and(|g| {
8222 g.resolved
8223 && g.title
8224 && g.choice == Some(GithubTextChoice::Original)
8225 }) {
8226 land::set_pr_title_verbatim(
8228 &winner.worktree,
8229 &url,
8230 &pr.title,
8231 )
8232 .await
8233 } else {
8234 land::set_pr_title(
8235 &mut self.state,
8236 &winner.worktree,
8237 &url,
8238 &pr.title,
8239 )
8240 .await
8241 }
8242 {
8243 tracing::warn!("could not refresh title of {url}: {e:#}");
8244 self.state
8245 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8246 }
8247 Ok(url)
8248 }
8249 }
8250 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8251 };
8252 match out {
8253 Ok(url) => MergeOutcome {
8254 mode,
8255 ok: true,
8256 detail: url,
8257 empty: false,
8258 },
8259 Err(e) => MergeOutcome {
8260 mode,
8261 ok: false,
8262 detail: e.to_string(),
8263 empty: false,
8264 },
8265 }
8266 }
8267 }
8268 };
8269
8270 self.state.status = match (mode, outcome.ok) {
8271 (MergeMode::None, _) => RunStatus::Ready,
8272 (_, true) if outcome.empty => RunStatus::Ready,
8273 (MergeMode::Pr, true) => RunStatus::Ready,
8274 (_, true) => RunStatus::Merged,
8275 (_, false) => RunStatus::Blocked,
8276 };
8277 self.state.event(
8278 "merge",
8279 format!(
8280 "{:?}: {}",
8281 mode,
8282 outcome.detail.lines().next().unwrap_or("")
8283 ),
8284 );
8285 self.state.merge = Some(outcome);
8286 self.state.save()?;
8287
8288 if self.state.config.graph.land
8294 && mode == MergeMode::Pr
8295 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8296 {
8297 self.run_land().await?;
8298 }
8299 self.settle_questions();
8304 Ok(())
8305 }
8306
8307 async fn run_land(&mut self) -> Result<()> {
8318 crate::pr_head::ensure_owner(&self.state)?;
8319 let url = self
8320 .state
8321 .pr
8322 .as_ref()
8323 .map(|pr| pr.url.clone())
8324 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8325 .unwrap_or_default();
8326 let url = url.lines().next().unwrap_or("").trim().to_owned();
8327 if !url.starts_with("http") {
8328 return Ok(());
8329 }
8330 loop {
8333 match land::land(&mut self.state, &url).await {
8334 Ok(_)
8335 if self
8336 .state
8337 .merge_repairs
8338 .last()
8339 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8340 {
8341 if self.repair_merge_review().await? {
8342 continue;
8343 }
8344 }
8345 Ok(pr) if self.state.parked => {
8346 let _ = pr;
8350 }
8351 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8352 self.state.status = RunStatus::Superseded;
8353 self.state.save()?;
8354 }
8355 Ok(pr) => {
8356 self.state.status = match pr.state {
8357 land::PrLifecycle::Merged => RunStatus::Merged,
8358 _ => RunStatus::Blocked,
8359 };
8360 if bump::should_release_bump(self.state.status)
8367 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8368 {
8369 self.state
8375 .event("bump", format!("release bump skipped: {e:#}"));
8376 }
8377 if self.state.status == RunStatus::Merged {
8380 crate::followup::after_merge(&mut self.state, &pr.url).await;
8381 }
8382 self.state.save()?;
8383 }
8384 Err(e) => {
8385 if e.is::<crate::repo_lock::ResourceBlocked>() {
8386 return Err(e);
8387 }
8388 self.state.status = RunStatus::Blocked;
8389 self.state.event("land", format!("gave up: {e}"));
8390 self.state.save()?;
8391 }
8392 }
8393 break;
8394 }
8395 Ok(())
8396 }
8397
8398 async fn repair_merge_review(&mut self) -> Result<bool> {
8401 use crate::run::MergeRepairPhase as Phase;
8402 crate::pr_head::ensure_owner(&self.state)?;
8403 let i = self.state.merge_repairs.len() - 1;
8404 let request = self.state.merge_repairs[i].clone();
8405 if request.phase == Phase::Fixing {
8406 let restored = match self.state.winner().cloned() {
8407 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8408 None => Err(anyhow::anyhow!("repair has no winning branch")),
8409 };
8410 let why = match restored {
8411 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8412 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8413 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8414 };
8415 self.hold_merge_repair(i, why)?;
8416 return Ok(false);
8417 }
8418 let failure = if i >= self.state.config.graph.land_rounds
8419 || self.state.config.graph.review_rounds == 0
8420 {
8421 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8422 } else {
8423 None
8424 };
8425 if let Some(why) = failure {
8426 self.hold_merge_repair(i, why)?;
8427 return Ok(false);
8428 }
8429 if request.phase == Phase::Requested {
8430 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8431 Ok(pr) => pr,
8432 Err(e) => {
8433 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8434 return Ok(false);
8435 }
8436 };
8437 self.state.merge_repairs[i].phase = Phase::Fixing;
8438 self.state.land_approval = None;
8439 self.state.land_armed_head = None;
8440 self.state.save()?;
8441 let evidence = serde_json::to_string_pretty(&request)?;
8444 let reason = format!(
8445 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8446 );
8447 let budget = self.state.config.graph.land_rounds;
8448 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8449 match result {
8450 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8451 Ok(land::Fixed::Failed(why)) => {
8452 self.hold_merge_repair(
8453 i,
8454 format!(
8455 "repair failed: {why}; inspect the branch and run an explicit review"
8456 ),
8457 )?;
8458 return Ok(false);
8459 }
8460 Err(e) => {
8461 self.hold_merge_repair(
8462 i,
8463 format!(
8464 "repair failed: {e:#}; inspect the branch and run an explicit review"
8465 ),
8466 )?;
8467 return Ok(false);
8468 }
8469 }
8470 self.begin_merge_repair_review(i)?;
8474 }
8475 self.review_loop().await?;
8476 self.gate().await?;
8477 if self.resource_halt()? || self.state.parked {
8478 return Ok(false);
8479 }
8480 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8481 != Some(RunStatus::Gating)
8482 || !self.state.gate_allows_progress()
8483 {
8484 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8485 return Ok(false);
8486 }
8487 let winner = self
8490 .state
8491 .winner()
8492 .cloned()
8493 .context("repair has no winning branch")?;
8494 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8495 if self.state.gate_head.as_deref() != Some(after.as_str())
8496 || !git::is_clean(&winner.worktree).await?
8497 {
8498 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8499 return Ok(false);
8500 }
8501 let expected = self
8502 .state
8503 .pr_pushed_head
8504 .as_deref()
8505 .unwrap_or(&request.head)
8506 .to_owned();
8507 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8508 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8511 let refspec = format!("{after}:refs/heads/{public}");
8512 let push = git::git_raw(
8513 &winner.worktree,
8514 &["push", &lease, &self.state.config.merge.remote, &refspec],
8515 )
8516 .await;
8517 match push {
8518 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8519 Ok(out) => {
8520 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8521 return Ok(false);
8522 }
8523 Err(e) => {
8524 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8525 return Ok(false);
8526 }
8527 }
8528 self.state.merge_repairs[i].phase = Phase::Complete;
8529 self.state.status = RunStatus::Landing;
8530 self.state.event(
8531 "repair",
8532 "new review and gate completed; fresh owner merge decision required",
8533 );
8534 self.state.save()?;
8535 Ok(true)
8536 }
8537
8538 fn merge_repair_artifacts(&self) -> PathBuf {
8539 let artifacts = agent::artifacts_dir(&self.state.dir());
8540 match self
8541 .state
8542 .merge_repairs
8543 .last()
8544 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8545 {
8546 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8547 None => artifacts,
8548 }
8549 }
8550
8551 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8552 self.state.reviews.clear();
8553 self.state.gate.clear();
8554 self.state.gate_ran = false;
8555 self.state.gate_head = None;
8556 self.state.gate_fixes.clear();
8557 self.state.gate_repair_e2e = None;
8558 self.state.pre_gate.clear();
8559 self.state.pre_gate_commit = None;
8560 self.state.contested_handoff = None;
8561 self.state.land_reruns = None;
8562 self.state.land_approval = None;
8563 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8564 self.state.status = RunStatus::Reviewing;
8565 self.state.save()
8566 }
8567
8568 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8569 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8570 self.state.merge_repairs[index].failure = Some(why.clone());
8571 self.state.status = RunStatus::Blocked;
8572 self.state.event("repair", why);
8573 self.state.save()
8574 }
8575
8576 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8580 if let Some(existing) = self.state.seats.get(key)
8581 && existing.agent == agent
8582 {
8583 return existing.clone();
8584 }
8585 let fresh = if self.state.seats.contains_key(key) {
8588 handover_seat(key, agent, self.state.next_seat_seed())
8589 } else {
8590 SeatState::new(key, agent, self.state.seed)
8591 };
8592 self.state.seats.insert(key.to_owned(), fresh.clone());
8593 fresh
8594 }
8595
8596 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8602 match self.state.seats.get(key) {
8603 Some(s) if s.agent != spec.id => {
8604 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8605 }
8606 _ => spec,
8607 }
8608 }
8609
8610 fn view(&self, c: &Candidate) -> CandidateView {
8612 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8613 .unwrap_or_default();
8614 let (patch, _) = blind::sanitize_patch(
8615 &format!("candidate {} patch", c.label),
8616 &raw,
8617 &self.state.config.blind,
8618 );
8619 CandidateView {
8620 label: c.label,
8621 branch: c.branch.clone(),
8622 summary: c.summary.clone(),
8623 stat: c.stat.clone(),
8624 patch,
8625 }
8626 }
8627
8628 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8630 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8631 prompt::judge(
8632 "(see above)",
8633 &views,
8634 self.roles.judges.len(),
8635 base_short,
8636 "en",
8637 )
8638 }
8639
8640 fn vote_prompt_full(
8646 &self,
8647 j: usize,
8648 labels: &[char],
8649 language: &str,
8650 candidates: &[Candidate],
8651 base_short: &str,
8652 ) -> String {
8653 let mut text = format!(
8654 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8655 prompt::final_vote(labels, language),
8656 self.state.instruction,
8657 self.candidate_block(candidates, base_short)
8658 );
8659 if let Some(own) = self
8660 .state
8661 .judgements
8662 .get(j)
8663 .filter(|r| !r.ranking.is_empty())
8664 {
8665 let reasons = own
8666 .reasons
8667 .iter()
8668 .map(|(k, v)| format!("- {k}: {v}"))
8669 .collect::<Vec<_>>()
8670 .join("\n");
8671 text.push_str(&format!(
8672 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8673 own.ranking.iter().collect::<String>(),
8674 if reasons.is_empty() {
8675 ""
8676 } else {
8677 ", because:\n"
8678 }
8679 ));
8680 }
8681 if !self.state.deliberation.is_empty() {
8682 text.push_str("\n# What was argued before this vote\n");
8683 for t in self.transcript(&[], j) {
8684 text.push_str(&format!(
8685 "\n## {}{}\n\n{}\n",
8686 t.who,
8687 if t.is_self { " (you)" } else { "" },
8688 t.body.trim()
8689 ));
8690 }
8691 }
8692 text
8693 }
8694
8695 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8702 let mut turns = Vec::new();
8703 for j in &self.state.judgements {
8704 if j.ranking.is_empty() {
8705 continue;
8706 }
8707 let reasons = j
8708 .reasons
8709 .iter()
8710 .map(|(k, v)| format!("- {k}: {v}"))
8711 .collect::<Vec<_>>()
8712 .join("\n");
8713 turns.push(Turn {
8714 who: format!("Judge {} (opening ranking)", j.judge),
8715 is_self: j.judge == self_idx + 1,
8716 body: format!(
8717 "Ranked {}{}{reasons}",
8718 j.ranking.iter().collect::<String>(),
8719 if reasons.is_empty() {
8720 ""
8721 } else {
8722 ", because:\n"
8723 }
8724 ),
8725 });
8726 }
8727 for t in self
8728 .state
8729 .deliberation
8730 .iter()
8731 .flat_map(|r| r.turns.iter())
8732 .chain(current)
8733 {
8734 turns.push(Turn {
8735 who: format!("Judge {}", t.judge),
8736 is_self: t.judge == self_idx + 1,
8737 body: t.body.clone(),
8738 });
8739 }
8740 turns
8741 }
8742}
8743
8744fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8746 agent::has_session(spec.kind, seat, sessions)
8747}
8748
8749fn next_untried_in_roster<'a>(
8770 roster: &'a [AgentSpec],
8771 start: usize,
8772 tried: &BTreeSet<String>,
8773) -> Option<&'a AgentSpec> {
8774 roster
8775 .get(start + 1..)?
8776 .iter()
8777 .find(|s| !tried.contains(&s.id))
8778}
8779
8780fn pick_successor<'a>(
8801 roster: &'a [AgentSpec],
8802 start: usize,
8803 tried: &BTreeSet<String>,
8804 carried: Option<&BTreeSet<String>>,
8805 others: &BTreeSet<String>,
8806) -> Option<&'a AgentSpec> {
8807 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8808 roster
8809 .get(start + 1..)
8810 .and_then(|tail| {
8811 tail.iter()
8812 .filter(free)
8813 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8814 })
8815 .or_else(|| {
8816 carried
8817 .is_some()
8818 .then(|| roster.iter().find(free))
8819 .flatten()
8820 })
8821 .or_else(|| match carried {
8822 Some(c) => next_for_seat(roster, start, tried, c),
8823 None => next_untried_in_roster(roster, start, tried),
8824 })
8825}
8826
8827fn next_for_seat<'a>(
8837 roster: &'a [AgentSpec],
8838 start: usize,
8839 round_tried: &BTreeSet<String>,
8840 carried_failed: &BTreeSet<String>,
8841) -> Option<&'a AgentSpec> {
8842 roster
8843 .get(start + 1..)?
8844 .iter()
8845 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8846 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8847}
8848
8849fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8855 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8856 return spec;
8857 };
8858 let ok = |id: &str| !h.failed.contains(id);
8859 if let Some(last) = h.last_ok.as_deref()
8860 && ok(last)
8861 && let Some(s) = roster.iter().find(|s| s.id == last)
8862 {
8863 return s.clone();
8864 }
8865 if ok(&spec.id) {
8866 return spec;
8867 }
8868 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8869 roster
8870 .get(from + 1..)
8871 .into_iter()
8872 .flatten()
8873 .chain(roster.iter())
8874 .find(|s| ok(&s.id))
8875 .cloned()
8876 .unwrap_or(spec)
8877}
8878
8879pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8883 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8884}
8885
8886const TIMED_OUT: &str = "timed out";
8889
8890impl FailClass {
8891 fn of(out: &AgentOutcome) -> Option<Self> {
8893 match out {
8894 AgentOutcome::Ok(_) => None,
8895 AgentOutcome::Quota(_) => Some(Self::Quota),
8896 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8897 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8898 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8899 AgentOutcome::ResourceBlocked(_) => None,
8902 }
8903 }
8904
8905 fn stem_word(&self) -> &'static str {
8907 match self {
8908 Self::Quota => "quota",
8909 _ => "handover",
8910 }
8911 }
8912}
8913
8914fn failure_signature(msg: &str) -> String {
8918 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8919 if let Some(code) = line
8922 .strip_prefix("process exited with no usable output (exit code ")
8923 .and_then(|s| s.strip_suffix(')'))
8924 .and_then(|s| s.parse::<i32>().ok())
8925 {
8926 line = format!("exited with some({code}) and no usable output");
8927 } else if matches!(
8928 line.as_str(),
8929 "process exited with no usable output (no exit code reported)"
8930 | "process exited with no usable output (exit code unavailable)"
8931 ) {
8932 line = "exited with none and no usable output".to_owned();
8933 }
8934 let mut out = Vec::new();
8935 for word in line.split_whitespace() {
8936 if word.contains('/') || word.contains('\\') {
8937 out.push("<path>".to_owned());
8938 continue;
8939 }
8940 let mut w = String::new();
8941 let mut in_digits = false;
8942 for c in word.chars() {
8943 if c.is_ascii_digit() {
8944 if !in_digits {
8945 w.push('#');
8946 }
8947 in_digits = true;
8948 } else {
8949 in_digits = false;
8950 w.push(c);
8951 }
8952 }
8953 out.push(w);
8954 }
8955 out.join(" ").chars().take(120).collect()
8956}
8957
8958fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8965 match cur {
8966 FailClass::Quota | FailClass::Timeout => true,
8967 FailClass::Other(_) => prev != Some(cur),
8968 }
8969}
8970
8971fn fail_reason(out: &AgentOutcome) -> String {
8973 match out {
8974 AgentOutcome::Ok(_) => String::new(),
8975 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8976 AgentOutcome::Dropped(o) => format!(
8977 "the CLI dropped the stream ({})",
8978 o.dropped
8979 .as_ref()
8980 .map(|d| d.why.as_str())
8981 .unwrap_or("it ended without delivering its answer")
8982 ),
8983 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8984 e.lines().next().unwrap_or("").chars().take(160).collect()
8985 }
8986 }
8987}
8988
8989pub(crate) fn record_handover(
8992 state: &mut RunState,
8993 node: &str,
8994 seat: &str,
8995 from: &str,
8996 to: &str,
8997 class: &FailClass,
8998 reason: &str,
8999) {
9000 let message = if *class == FailClass::Quota {
9001 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9002 } else {
9003 format!("{seat}: handed over {from} -> {to} ({reason})")
9004 };
9005 state.event(node, message);
9006 state.handovers.push(Handover {
9007 at: Timestamp::now(),
9008 node: node.to_owned(),
9009 seat: seat.to_owned(),
9010 from: from.to_owned(),
9011 to: to.to_owned(),
9012 reason: reason.to_owned(),
9013 });
9014}
9015
9016fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9030 commands.iter().any(|c| c.exit_code.is_none())
9031}
9032
9033fn verified_noop_claim(
9046 usable: bool,
9047 commands: &[agent::CommandEvidence],
9048 text: &str,
9049) -> Option<String> {
9050 (usable && !has_unconfirmed_command(commands))
9051 .then(|| verdict::verified_noop(text))
9052 .flatten()
9053}
9054
9055fn short(commit: &str) -> String {
9056 commit.chars().take(7).collect()
9057}
9058
9059fn make_executable(path: &Path) -> Result<()> {
9060 #[cfg(unix)]
9061 {
9062 use std::os::unix::fs::PermissionsExt as _;
9063 let mut perms = std::fs::metadata(path)?.permissions();
9064 perms.set_mode(0o755);
9065 std::fs::set_permissions(path, perms)?;
9066 }
9067 #[cfg(not(unix))]
9068 {
9069 let _ = path;
9070 }
9071 Ok(())
9072}
9073
9074struct WaveCtx<'a> {
9081 run: &'a str,
9084 node: &'a str,
9086 prompts: &'a Prompts,
9087 cache: Option<&'a Path>,
9089 round: Option<usize>,
9092 carry_seats: bool,
9096 pauses: Option<(&'a Pause, &'a Pause)>,
9099}
9100
9101fn repository_wait(
9103 state: &RunState,
9104 pauses: Option<(&Pause, &Pause)>,
9105) -> crate::repo_lock::WaitPolicy {
9106 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9107 let inherited = crate::repo_lock::WaitPolicy::current();
9108 crate::repo_lock::WaitPolicy::new(
9109 Duration::from_secs(state.config.graph.timeout_resource_wait),
9110 move || {
9111 inherited.cancelled()
9112 || pauses
9113 .as_ref()
9114 .is_some_and(|(a, b)| a.parked() || b.parked())
9115 },
9116 )
9117}
9118
9119fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9120 if error.is::<crate::repo_lock::ResourceBlocked>() {
9121 state.resource_block = Some(crate::run::ResourceBlock {
9122 node: "repository".into(),
9123 kind: if error.to_string().contains("disk pressure") {
9124 "disk pressure"
9125 } else {
9126 "repository coordination"
9127 }
9128 .into(),
9129 detail: error.to_string(),
9130 at: Timestamp::now(),
9131 });
9132 state.event("resource", error.to_string());
9133 }
9134}
9135
9136async fn coordinate_operation(
9137 state: &mut RunState,
9138 cwd: &Path,
9139 wait: &crate::repo_lock::WaitPolicy,
9140) -> Result<Option<crate::repo_lock::SeatLease>> {
9141 let result =
9142 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9143 .await;
9144 if let Err(e) = &result {
9145 record_repository_block(state, e);
9146 }
9147 result
9148}
9149
9150async fn run_one(
9152 job: SeatJob,
9153 sem: Arc<Semaphore>,
9154 ctx: &WaveCtx<'_>,
9155 state: &mut RunState,
9156 attempt: usize,
9157) -> (SeatState, AgentOutcome) {
9158 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9159 .await
9160 .pop()
9161 .expect("one job in, one result out");
9162 (seat, out)
9163}
9164
9165async fn wave(
9171 jobs: Vec<SeatJob>,
9172 sem: Arc<Semaphore>,
9173 ctx: &WaveCtx<'_>,
9174 state: &mut RunState,
9175 attempt: usize,
9176) -> Vec<(usize, SeatState, AgentOutcome)> {
9177 let WaveCtx {
9178 run,
9179 node,
9180 prompts,
9181 cache,
9182 round,
9183 carry_seats: _,
9184 pauses,
9185 } = *ctx;
9186 for job in &jobs {
9187 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9188 }
9189 if let Err(e) = state.save() {
9190 tracing::warn!("could not persist in-progress seats: {e:#}");
9195 }
9196 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9212 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9216 let cache_guard = if let Some(cache_dir) = cache {
9217 if jobs_had_a_writer {
9218 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9219 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9220 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9221 {
9222 Ok(Some(g)) => Some(g),
9223 Ok(None) => {
9226 state.resource_block = Some(crate::run::ResourceBlock {
9227 node: node.to_owned(),
9228 kind: "shared build cache".to_owned(),
9229 detail: "the run is parking".to_owned(),
9230 at: Timestamp::now(),
9231 });
9232 return block_wave(
9233 jobs,
9234 state,
9235 node,
9236 attempt,
9237 "parked while waiting for the shared build cache",
9238 );
9239 }
9240 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9243 Err(e) => {
9244 let why = format!("{e:#}");
9248 state.resource_block = Some(crate::run::ResourceBlock {
9249 node: node.to_owned(),
9250 kind: "shared build cache".to_owned(),
9251 detail: why.clone(),
9252 at: Timestamp::now(),
9253 });
9254 state.event(
9255 "resource",
9256 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9257 );
9258 return block_wave(jobs, state, node, attempt, &why);
9259 }
9260 }
9261 } else {
9262 None
9263 }
9264 } else {
9265 None
9266 };
9267 state.seats_spawning(node, attempt);
9270 if let Err(e) = state.save() {
9271 tracing::warn!("could not persist the start of a wave: {e:#}");
9272 }
9273 let slot_limit = resource_limit
9277 + jobs
9278 .iter()
9279 .map(|j| j.timeout)
9280 .max()
9281 .unwrap_or_default()
9282 .saturating_mul(jobs.len() as u32);
9283 let mut set = tokio::task::JoinSet::new();
9284 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9287 let overlay = prompts.overlay(node);
9288 let repository_wait = repository_wait(state, pauses);
9289 for (i, mut job) in jobs.into_iter().enumerate() {
9290 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9291 if cache.is_some() {
9292 job.prompt.push('\n');
9293 job.prompt
9294 .push_str(&prompt::build_cache_note(node, job.allow_write));
9295 }
9296 let sem = Arc::clone(&sem);
9297 let run = run.to_owned();
9298 let node = node.to_owned();
9299 let attachments = if node == "implement" {
9302 state.attachments.clone()
9303 } else {
9304 Vec::new()
9305 };
9306 let cache = cache
9317 .filter(|_| job.allow_write && cache_guard.is_some())
9318 .map(Path::to_path_buf);
9319 let spawned_tx = spawned_tx.clone();
9320 let repository_wait = repository_wait.clone();
9321 let coordination = state.config.coordination.clone();
9322 let fetched_base = state.fetched_base.clone();
9323 set.spawn(async move {
9324 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9328 Ok(p) => p,
9329 Err(_) => {
9330 return (
9331 i,
9332 job.seat,
9333 AgentOutcome::ResourceBlocked(format!(
9334 "no parallel slot became free within {}s",
9335 slot_limit.as_secs()
9336 )),
9337 false,
9338 );
9339 }
9340 };
9341 spawned_tx.send(job.seat.key.clone()).ok();
9342 let mut seat = job.seat;
9343 let out = repository_wait
9344 .scope(agent::invoke(
9345 &job.spec,
9346 &mut seat,
9347 &Invocation {
9348 scope: crate::agent::InvocationScope::Repository,
9349 coordination: coordination.as_ref(),
9350 fetched_base: fetched_base.as_ref(),
9351 cwd: &job.cwd,
9352 prompt: &job.prompt,
9353 timeout: job.timeout,
9354 allow_write: job.allow_write,
9355 unsandboxed: false,
9356 sessions: job.sessions,
9357 artifacts: &job.artifacts,
9358 stem: &job.stem,
9359 run: &run,
9360 node: &node,
9361 cache_dir: cache.as_deref(),
9362 attachments: &attachments,
9363 writable: &[],
9364 },
9365 ))
9366 .await;
9367 let interrupted = out
9368 .as_ref()
9369 .err()
9370 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9371 let out = match out {
9372 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9373 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9374 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9382 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9383 Ok(o) => AgentOutcome::Failed(format!(
9384 "process exited with no usable output ({})",
9385 crate::notices::exit_status(o.exit_code)
9386 )),
9387 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9388 AgentOutcome::ResourceBlocked(e.to_string())
9389 }
9390 Err(e) => AgentOutcome::Failed(e.to_string()),
9391 };
9392 (i, seat, out, interrupted)
9393 });
9394 }
9395 drop(spawned_tx);
9396 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9397 loop {
9398 let joined = tokio::select! {
9399 Some(key) = spawned_rx.recv() => {
9400 state.seat_spawned(&key);
9402 if let Err(e) = state.save() {
9403 tracing::warn!("could not persist a seat's start: {e:#}");
9404 }
9405 continue;
9406 }
9407 joined = set.join_next() => joined,
9408 };
9409 let Some(joined) = joined else {
9410 break;
9411 };
9412 let (i, seat, out, interrupted) = match joined {
9413 Ok(v) => v,
9414 Err(e) => {
9418 tracing::error!("agent task panicked: {e}");
9419 continue;
9420 }
9421 };
9422 state.seat_finished(&seat.key);
9423 if interrupted
9424 && node == "implement"
9425 && let Some(label) = state
9426 .candidates
9427 .iter()
9428 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9429 .map(|candidate| candidate.label)
9430 {
9431 state.candidate_preservation.insert(
9432 label,
9433 "interrupted implementation awaiting persistence".into(),
9434 );
9435 state.interrupted_implementations.insert(label);
9436 }
9437 if let AgentOutcome::ResourceBlocked(why) = &out
9438 && state.resource_block.is_none()
9439 {
9440 state.resource_block = Some(crate::run::ResourceBlock {
9441 node: node.to_owned(),
9442 kind: if why.contains("disk pressure") {
9443 "disk pressure"
9444 } else if why.contains("repository coordination") {
9445 "repository coordination"
9446 } else {
9447 "parallel slot"
9448 }
9449 .to_owned(),
9450 detail: why.clone(),
9451 at: Timestamp::now(),
9452 });
9453 let activity = if interrupted {
9454 "agent execution interrupted"
9455 } else {
9456 "no agent was started"
9457 };
9458 state.event("resource", format!("{node}: {activity} - {why}"));
9459 }
9460 record_jobs(state, node, round, &seat.key, &out);
9461 if let Err(e) = state.save() {
9462 tracing::warn!("could not persist a seat's completion: {e:#}");
9463 }
9464 if collected.len() <= i {
9465 collected.resize_with(i + 1, || None);
9466 }
9467 collected[i] = Some((i, seat, out));
9468 }
9469 if state
9475 .active
9476 .values()
9477 .any(|a| a.node == node && a.attempt == attempt)
9478 {
9479 state
9480 .active
9481 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9482 if let Err(e) = state.save() {
9483 tracing::warn!("could not persist the end of a wave: {e:#}");
9484 }
9485 }
9486 if let Some(cache_dir) = cache
9493 && jobs_had_a_writer
9494 {
9495 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9496 }
9497 if let Some(guard) = cache_guard {
9498 guard.release();
9499 }
9500 collected.into_iter().flatten().collect()
9501}
9502
9503fn record_jobs(
9514 state: &mut RunState,
9515 node: &str,
9516 round: Option<usize>,
9517 seat: &str,
9518 out: &AgentOutcome,
9519) {
9520 let commands: &[agent::CommandEvidence] = match out {
9521 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9522 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9523 };
9524 let checked_at = Timestamp::now();
9525 for c in commands {
9526 state.jobs.push(JobRecord {
9527 node: node.to_owned(),
9528 round,
9529 seat: seat.to_owned(),
9530 id: c.id.clone(),
9531 description: c.description.clone(),
9532 checked_at,
9533 status: match c.exit_code {
9534 Some(0) => JobStatus::Completed,
9535 Some(_) => JobStatus::Failed,
9536 None => JobStatus::Unknown,
9537 },
9538 exit_code: c.exit_code,
9539 result_summary: c.result_summary.clone(),
9540 source: c.source.clone(),
9541 });
9542 }
9543}
9544
9545fn round_is_clean(
9566 blocking: usize,
9567 e2e_ok: bool,
9568 answered: usize,
9569 expected: usize,
9570 quota_missing: usize,
9571 policy: IncompleteReviewPolicy,
9572) -> bool {
9573 if blocking != 0 || !e2e_ok {
9574 return false;
9575 }
9576 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9577 return true;
9578 }
9579 answered > 0 && expected - answered <= quota_missing
9580}
9581
9582pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9606 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9607 return Some(RunStatus::Gating);
9608 }
9609 let last = reviews.last()?;
9610 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9611 if reviews.len() < max_rounds && !stagnant {
9612 return None;
9613 }
9614 if last.incomplete() && last.blocking == 0 {
9615 return Some(RunStatus::Blocked);
9616 }
9617 if last.e2e_status() == E2eStatus::ResourceBlocked {
9618 return None;
9619 }
9620 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9621 RunStatus::Gating
9622 } else {
9623 RunStatus::Blocked
9624 })
9625}
9626
9627pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9642 if nudged {
9643 (full / 4).max(Duration::from_secs(120)).min(full)
9644 } else {
9645 full
9646 }
9647}
9648
9649#[allow(clippy::too_many_arguments)]
9678async fn ask_json_wave<T>(
9679 jobs: Vec<SeatJob>,
9680 sem: Arc<Semaphore>,
9681 retries: usize,
9682 roster: &[AgentSpec],
9683 ctx: &WaveCtx<'_>,
9684 losses: &mut Vec<QuotaLoss>,
9685 state: &mut RunState,
9686 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9687) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9688where
9689 T: serde::de::DeserializeOwned + Send + 'static,
9690{
9691 ask_wave_with(
9692 jobs,
9693 sem,
9694 retries,
9695 roster,
9696 ctx,
9697 losses,
9698 state,
9699 &|text: &str| {
9700 let v = verdict::extract_json::<T>(text)?;
9701 validate(&v)?;
9702 Ok(v)
9703 },
9704 )
9705 .await
9706}
9707
9708#[allow(clippy::too_many_arguments)]
9717async fn ask_wave_with<T>(
9718 jobs: Vec<SeatJob>,
9719 sem: Arc<Semaphore>,
9720 retries: usize,
9721 roster: &[AgentSpec],
9722 ctx: &WaveCtx<'_>,
9723 losses: &mut Vec<QuotaLoss>,
9724 state: &mut RunState,
9725 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9726) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9727where
9728 T: Send + 'static,
9729{
9730 let n = jobs.len();
9731 let originals: Vec<SeatJob> = jobs;
9732 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9733 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9734 let mut nudges: Vec<usize> = vec![0; n];
9741 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9745 let mut tried: Vec<BTreeSet<String>> = specs
9746 .iter()
9747 .map(|s| BTreeSet::from([s.id.clone()]))
9748 .collect();
9749 let starts: Vec<usize> = specs
9750 .iter()
9751 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9752 .collect();
9753 let carry = ctx.carry_seats && !roster.is_empty();
9754 let carried: Vec<BTreeSet<String>> = originals
9757 .iter()
9758 .map(|j| {
9759 state
9760 .seat_history
9761 .get(&j.seat.key)
9762 .filter(|_| carry)
9763 .map(|h| h.failed.clone())
9764 .unwrap_or_default()
9765 })
9766 .collect();
9767 let mut prev: Vec<Option<FailClass>> = originals
9768 .iter()
9769 .map(|j| {
9770 state
9771 .seat_history
9772 .get(&j.seat.key)
9773 .filter(|_| carry)
9774 .and_then(|h| h.last_fail.clone())
9775 })
9776 .collect();
9777 let next_agent =
9778 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9779 let others: BTreeSet<String> = specs
9780 .iter()
9781 .enumerate()
9782 .filter(|(j, _)| *j != i)
9783 .map(|(_, s)| s.id.clone())
9784 .collect();
9785 pick_successor(
9786 roster,
9787 starts[i],
9788 tried,
9789 carry.then(|| &carried[i]),
9790 &others,
9791 )
9792 .cloned()
9793 };
9794 let mut fresh: Vec<Option<String>> = vec![None; n];
9795 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9796 let mut pending: Vec<usize> = (0..n).collect();
9797
9798 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9803 for round in 0..max_rounds {
9804 if pending.is_empty() {
9805 break;
9806 }
9807 let mut batch = Vec::with_capacity(pending.len());
9808 let mut renudged: Vec<&str> = Vec::new();
9809 for &i in &pending {
9810 let src = &originals[i];
9811 let full: &str = match &src.handover {
9816 Some(h) if specs[i].id != src.spec.id => h,
9817 _ => &src.prompt,
9818 };
9819 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9822 (
9823 full.to_owned(),
9824 src.timeout,
9825 format!("{}-{word}-{}", src.stem, specs[i].id),
9826 )
9827 } else if nudges[i] == 0 {
9828 (full.to_owned(), src.timeout, src.stem.clone())
9829 } else {
9830 renudged.push(src.seat.key.as_str());
9831 let why = done[i]
9832 .as_ref()
9833 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9834 .unwrap_or_else(|| "no parsable answer".to_owned());
9835 let nudge = prompt::nudge(&why);
9836 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9837 let prompt = if nudged {
9838 nudge
9839 } else {
9840 format!("{full}\n\n---\n\n{nudge}")
9841 };
9842 (
9843 prompt,
9844 retry_budget(src.timeout, nudged),
9845 format!("{}-retry{}", src.stem, nudges[i]),
9846 )
9847 };
9848 batch.push(SeatJob {
9849 spec: specs[i].clone(),
9850 seat: seats[i].clone(),
9851 cwd: src.cwd.clone(),
9852 prompt,
9853 timeout,
9854 allow_write: src.allow_write,
9855 sessions: src.sessions,
9856 artifacts: src.artifacts.clone(),
9857 stem,
9858 handover: None,
9859 });
9860 }
9861
9862 if !renudged.is_empty() {
9863 state.event(
9864 ctx.node,
9865 format!("retry {round}: re-asking {}", renudged.join(", ")),
9866 );
9867 }
9868 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9869 let mut still = Vec::new();
9870 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9871 seats[i] = seat;
9872 if let AgentOutcome::ResourceBlocked(why) = &out {
9875 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9876 continue;
9877 }
9878 let class = FailClass::of(&out);
9879 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9882 && nudges[i] < retries
9883 && !roster.is_empty();
9884 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9885 let next = should_hand_over(prev[i].as_ref(), &cur)
9886 .then(|| next_agent(i, &tried[i], &specs))
9887 .flatten();
9888 if carry {
9889 let h = state
9890 .seat_history
9891 .entry(originals[i].seat.key.clone())
9892 .or_default();
9893 h.failed.insert(specs[i].id.clone());
9894 h.last_fail = Some(cur.clone());
9895 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9896 h.last_ok = None;
9897 }
9898 if let Err(e) = state.save() {
9901 tracing::warn!("could not persist a seat's failure history: {e:#}");
9902 }
9903 }
9904 if let Some(next) = next {
9905 record_handover(
9906 state,
9907 ctx.node,
9908 &originals[i].seat.key,
9909 &specs[i].id,
9910 &next.id,
9911 &cur,
9912 &fail_reason(&out),
9913 );
9914 tried[i].insert(next.id.clone());
9915 prev[i] = Some(cur.clone());
9916 seats[i] =
9917 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9918 specs[i] = next;
9919 fresh[i] = Some(cur.stem_word().to_owned());
9920 nudges[i] = 0;
9921 done[i] = Some(Err(anyhow::anyhow!(
9922 "handed over after: {}",
9923 fail_reason(&out)
9924 )));
9925 still.push(i);
9926 continue;
9927 }
9928 }
9929 if carry
9930 && !nudge_first
9931 && let Some(cur) = class.clone()
9932 {
9933 let h = state
9935 .seat_history
9936 .entry(originals[i].seat.key.clone())
9937 .or_default();
9938 h.failed.insert(specs[i].id.clone());
9939 h.last_fail = Some(cur);
9940 }
9941 let parsed = match out {
9942 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9943 AgentOutcome::Quota(o) => {
9944 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9945 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9946 }
9947 AgentOutcome::Dropped(o) => {
9951 let why = o
9952 .dropped
9953 .as_ref()
9954 .map(|d| d.why.as_str())
9955 .unwrap_or("the CLI ended the stream without delivering its answer");
9956 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9957 }
9958 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9959 Err(anyhow::anyhow!(e))
9960 }
9961 };
9962 let quota = class == Some(FailClass::Quota);
9963 let failed = parsed.is_err();
9964 done[i] = Some(parsed);
9965 if carry && !failed {
9966 let h = state
9967 .seat_history
9968 .entry(originals[i].seat.key.clone())
9969 .or_default();
9970 h.failed.remove(&specs[i].id);
9971 h.last_ok = Some(specs[i].id.clone());
9972 h.last_fail = None;
9973 }
9974 let agent_failure = class.is_some()
9982 && !nudge_first
9983 && !roster.is_empty()
9984 && next_agent(i, &tried[i], &specs).is_some();
9985 if failed && !quota && !agent_failure && nudges[i] < retries {
9986 nudges[i] += 1;
9987 still.push(i);
9988 }
9989 }
9990 pending = still;
9991 }
9992
9993 for (i, q) in last_quota.into_iter().enumerate() {
9996 if let Some(reset) = q {
9997 losses.push(QuotaLoss {
9998 seat: originals[i].seat.key.clone(),
9999 node: ctx.node.to_owned(),
10000 at: Timestamp::now(),
10001 reset,
10002 });
10003 }
10004 }
10005
10006 seats
10007 .into_iter()
10008 .zip(done)
10009 .zip(nudges)
10010 .map(|((seat, res), attempts)| {
10011 (
10012 seat,
10013 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10014 attempts,
10015 )
10016 })
10017 .collect()
10018}
10019
10020fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10022 results
10023 .iter()
10024 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10025}
10026
10027fn block_wave(
10031 jobs: Vec<SeatJob>,
10032 state: &mut RunState,
10033 node: &str,
10034 attempt: usize,
10035 why: &str,
10036) -> Vec<(usize, SeatState, AgentOutcome)> {
10037 state.resource_wait = None;
10038 state
10039 .active
10040 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10041 if let Err(e) = state.save() {
10042 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10043 }
10044 jobs.into_iter()
10045 .enumerate()
10046 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10047 .collect()
10048}
10049
10050async fn acquire_cache_lease(
10063 state: &mut RunState,
10064 cache_dir: &Path,
10065 owner: &crate::cache::Owner,
10066 budget: Duration,
10067 context: &str,
10068 cancel: &(dyn Fn() -> bool + Sync),
10069) -> Result<Option<crate::cache::Guard>> {
10070 let home = crate::run::home();
10071 let started = Instant::now();
10072 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10073 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10074 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10075 Err(e) => {
10076 state.event(
10077 "verify",
10078 format!("{context}: could not check the shared build cache: {e:#}"),
10079 );
10080 if let Err(e2) = state.save() {
10081 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10082 }
10083 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10084 }
10085 };
10086 state.event(
10087 "verify",
10088 format!(
10089 "{context}: waiting for the shared build cache at {} ({})",
10090 cache_dir.display(),
10091 busy.describe()
10092 ),
10093 );
10094 state.resource_wait = Some(crate::run::ResourceWait {
10097 node: context.to_owned(),
10098 kind: "shared build cache".to_owned(),
10099 holder: busy.describe(),
10100 since: Timestamp::now(),
10101 limit_secs: budget.as_secs(),
10102 });
10103 if let Err(e) = state.save() {
10104 tracing::warn!("could not persist a cache wait: {e:#}");
10105 }
10106 let remaining = budget.saturating_sub(started.elapsed());
10107 let waited = crate::cache::wait_for_unless(
10108 &home,
10109 cache_dir,
10110 owner,
10111 remaining,
10112 Duration::from_secs(5),
10113 cancel,
10114 )
10115 .await;
10116 state.resource_wait = None;
10117 match waited {
10118 Ok(g) => Ok(g),
10119 Err(e) => {
10120 state.event("verify", format!("{context}: {e:#}"));
10121 if let Err(e2) = state.save() {
10122 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10123 }
10124 Err(e)
10125 }
10126 }
10127}
10128
10129#[derive(Debug)]
10134struct CacheCheckError(String);
10135
10136impl std::fmt::Display for CacheCheckError {
10137 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10138 f.write_str(&self.0)
10139 }
10140}
10141
10142impl std::error::Error for CacheCheckError {}
10143
10144#[allow(clippy::too_many_arguments)]
10165async fn with_cache_lease<'s, F, Fut>(
10166 state: &'s mut RunState,
10167 cache_dir: Option<&Path>,
10168 node: &str,
10169 seat: &str,
10170 worktree: &Path,
10171 head: &str,
10172 budget: Duration,
10173 context: &str,
10174 wait: &crate::repo_lock::WaitPolicy,
10175 body: F,
10176) -> Result<(Vec<CommandOutcome>, bool)>
10177where
10178 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10179 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10180{
10181 let started = Instant::now();
10184 let home = crate::run::home();
10185 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10186 let guard = if let Some(cache_dir) = cache_dir {
10187 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10188 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10189 wait.cancelled()
10190 })
10191 .await
10192 .and_then(|g| {
10193 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10194 }) {
10195 Ok(g) => Some(g),
10196 Err(e) => {
10197 return Ok((
10198 vec![CommandOutcome {
10199 command: "(waiting for the shared build cache)".into(),
10200 code: None,
10201 output_tail: e.to_string(),
10202 duration_ms: started.elapsed().as_millis() as u64,
10203 resource_blocked: true,
10204 }],
10205 false,
10206 ));
10207 }
10208 }
10209 } else {
10210 None
10211 };
10212 let _coordination = coordinate_operation(state, worktree, wait).await?;
10213 if let Some(cache_dir) = cache_dir {
10214 let identity = crate::cache::Identity::new(worktree, head);
10215 if let Err(e) = crate::cache::ensure_fresh(
10216 &home,
10217 cache_dir,
10218 &identity,
10219 &state.config.verify.cargo_manifests(),
10220 ) {
10221 state.event(
10222 "verify",
10223 format!(
10224 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10225 worktree.display(),
10226 short(head)
10227 ),
10228 );
10229 if let Some(guard) = guard {
10230 guard.release();
10231 }
10232 return Ok((
10233 vec![CommandOutcome {
10234 command: "(confirming the shared build cache is fresh)".into(),
10235 code: None,
10236 output_tail: e.to_string(),
10237 duration_ms: started.elapsed().as_millis() as u64,
10238 resource_blocked: true,
10239 }],
10240 false,
10241 ));
10242 }
10243 }
10244 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10245 if !timed_out_pids.is_empty() {
10246 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10247 }
10248 drop(_coordination);
10251 if let Some(guard) = guard {
10252 guard.release();
10253 }
10254 Ok((outcomes, retried))
10255}
10256
10257async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10269 wait_for_pids_with(
10270 pids,
10271 crate::proc::pid_alive,
10272 LEASE_RELEASE_POLL,
10273 LEASE_RELEASE_MAX_WAIT,
10274 )
10275 .await;
10276}
10277
10278async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10284 pids: &[u32],
10285 alive: F,
10286 poll: Duration,
10287 max_wait: Duration,
10288) {
10289 let deadline = Instant::now() + max_wait;
10290 loop {
10291 if pids.iter().all(|&pid| !alive(pid)) {
10292 return;
10293 }
10294 if Instant::now() >= deadline {
10295 return;
10296 }
10297 tokio::time::sleep(poll).await;
10298 }
10299}
10300
10301fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10308 outcomes.iter().any(|o| o.resource_blocked)
10309}
10310
10311pub(crate) enum GateFix {
10313 Retry,
10315 Stop,
10318 Defer,
10321}
10322
10323fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10331 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10332 red.peek().is_some()
10333 && red.all(|o| {
10334 !o.resource_blocked
10335 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10336 && !o.output_tail.trim().is_empty()
10337 })
10338}
10339
10340fn e2e_outcome_label(o: &CommandOutcome) -> String {
10344 if o.ok() {
10345 return "pass".to_owned();
10346 }
10347 let reason = if o.build_failed() {
10348 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10349 } else {
10350 format!("FAIL ({:?})", o.code)
10351 };
10352 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10353}
10354
10355async fn run_e2e_with_retry(
10363 state: &mut RunState,
10364 shell: &[String],
10365 commands: &[String],
10366 worktree: &Path,
10367 timeout: Duration,
10368 context: &str,
10369) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10370 let (mut e2e, mut timed_out_pids) = run_commands(
10371 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10372 )
10373 .await;
10374 for o in &e2e {
10375 state.event(
10376 "verify",
10377 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10378 );
10379 }
10380 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10384 if verify_retried {
10385 state.event(
10386 "verify",
10387 format!(
10388 "{context}: verify could not build/link, not a test result — retrying once \
10389 before concluding"
10390 ),
10391 );
10392 let retried = run_commands(
10393 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10394 )
10395 .await;
10396 e2e = retried.0;
10397 timed_out_pids.extend(retried.1);
10400 for o in &e2e {
10401 state.event(
10402 "verify",
10403 format!(
10404 "{context}: retry `{}` -> {}",
10405 o.command,
10406 e2e_outcome_label(o)
10407 ),
10408 );
10409 }
10410 }
10411 (e2e, verify_retried, timed_out_pids)
10412}
10413
10414#[allow(clippy::too_many_arguments)]
10430async fn run_commands(
10431 state: &mut RunState,
10432 node: &str,
10433 task: &str,
10434 attempt: usize,
10435 shell: &[String],
10436 commands: &[String],
10437 cwd: &Path,
10438 timeout: Duration,
10439) -> (Vec<CommandOutcome>, Vec<u32>) {
10440 if commands.is_empty() {
10441 return (Vec::new(), Vec::new());
10446 }
10447 let mut out = Vec::new();
10448 let mut timed_out_pids = Vec::new();
10449 let total = commands.len();
10450 for (idx, command) in commands.iter().enumerate() {
10451 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10452 if let Err(e) = state.save() {
10453 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10454 }
10455 let started = Instant::now();
10456 let mut cmd = tokio::process::Command::new(&shell[0]);
10457 cmd.quiet();
10458 cmd.args(&shell[1..])
10459 .arg(command)
10460 .current_dir(cwd)
10461 .stdin(std::process::Stdio::null())
10462 .stdout(std::process::Stdio::piped())
10463 .stderr(std::process::Stdio::piped())
10464 .kill_on_drop(true);
10465 let mut outputs = Vec::new();
10466 let protected = (|| -> Result<()> {
10467 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10468 outputs.push(guard);
10469 }
10470 if let Some(cache) = state.config.cache_dir() {
10471 outputs.push(crate::retention::OutputLease::cache(
10472 &crate::run::home(),
10473 &cache,
10474 false,
10475 )?);
10476 }
10477 Ok(())
10478 })();
10479 if let Err(e) = protected {
10480 out.push(CommandOutcome {
10481 command: command.clone(),
10482 code: None,
10483 output_tail: format!("output ownership unavailable: {e:#}"),
10484 duration_ms: 0,
10485 resource_blocked: true,
10486 });
10487 continue;
10488 }
10489 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10490 let (code, body) = match spawned {
10491 Ok(mut running) => {
10492 let pid = running.child().id();
10493 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10494 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10495 let mut paths = vec![cwd.to_owned()];
10496 if let Some(cache) = state.config.cache_dir() {
10497 paths.push(cache);
10498 }
10499 let mut pressure = None;
10500 let status = tokio::select! {
10501 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10502 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10503 pressure = Some(reason); None
10504 }
10505 };
10506 let (code, failure) = match status {
10507 Some(Ok(Ok(status))) => (
10508 status.code(),
10509 status
10510 .code()
10511 .is_none()
10512 .then(|| format!("command terminated: {status}")),
10513 ),
10514 Some(Ok(Err(error))) => {
10515 (None, Some(format!("failed waiting for command: {error}")))
10516 }
10517 Some(Err(_)) => {
10518 if !running.coordinated()
10522 && let Some(pid) = pid
10523 {
10524 timed_out_pids.push(pid);
10525 }
10526 (
10527 None,
10528 Some(format!("timed out after {}s", timeout.as_secs())),
10529 )
10530 }
10531 None => (None, None),
10532 };
10533 running.finish().await;
10534 if let Some(reason) = pressure {
10535 state.resource_block = Some(crate::run::ResourceBlock {
10536 node: node.to_owned(),
10537 kind: "disk pressure".to_owned(),
10538 detail: reason.clone(),
10539 at: Timestamp::now(),
10540 });
10541 out.push(CommandOutcome {
10542 command: command.clone(),
10543 code: None,
10544 output_tail: reason,
10545 duration_ms: started.elapsed().as_millis() as u64,
10546 resource_blocked: true,
10547 });
10548 break;
10549 }
10550 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10551 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10552 if let Some(failure) = failure {
10553 body.push('\n');
10554 body.push_str(&failure);
10555 }
10556 (code, body)
10557 }
10558 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10559 };
10560 out.push(CommandOutcome {
10561 command: command.clone(),
10562 code,
10563 output_tail: tail(&body, OUTPUT_TAIL),
10564 duration_ms: started.elapsed().as_millis() as u64,
10565 resource_blocked: false,
10566 });
10567 }
10568 state.task_finished(task);
10569 if let Err(e) = state.save() {
10570 tracing::warn!("could not persist the end of {task}: {e:#}");
10571 }
10572 (out, timed_out_pids)
10573}
10574
10575fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10587 let repo = repo.display();
10588 match style {
10589 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10590 MergeStyle::Squash => {
10591 let subject = message
10594 .lines()
10595 .next()
10596 .unwrap_or(branch)
10597 .replace(['\\', '"', '$', '`'], "");
10598 format!(
10599 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10600 )
10601 }
10602 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10603 }
10604}
10605
10606const PR_TITLE_MAX: usize = 240;
10617
10618fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10629 let line = queue::first_line(text)?;
10630 let chars: Vec<char> = line.chars().collect();
10631 let (kept, mark) = if chars.len() <= max {
10632 (line.to_owned(), "")
10633 } else if let Some(end) = sentence_end(&chars, max) {
10634 (chars[..end].iter().collect::<String>(), "")
10635 } else {
10636 let room = max.saturating_sub(3);
10637 let cut = if chars[room].is_whitespace() {
10640 room
10641 } else {
10642 chars[..room]
10643 .iter()
10644 .rposition(|c| c.is_whitespace())
10645 .unwrap_or(room)
10646 };
10647 let head: String = chars[..cut].iter().collect();
10648 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10649 (head.to_owned(), "...")
10650 };
10651 if kept.is_empty() {
10652 return None;
10653 }
10654 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10655 return None;
10656 }
10657 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10658 return None;
10659 }
10660 Some(format!("{kept}{mark}"))
10661}
10662
10663fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10667 const MIN: usize = 20;
10668 for i in MIN..max.min(chars.len()) {
10669 if !matches!(chars[i], '.' | '!' | '?') {
10670 continue;
10671 }
10672 let Some(&next) = chars.get(i + 1) else {
10673 continue;
10674 };
10675 if !next.is_whitespace() {
10676 continue;
10677 }
10678 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10679 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10680 continue;
10681 }
10682 let word: String = chars[..i]
10683 .iter()
10684 .rev()
10685 .take_while(|c| !c.is_whitespace())
10686 .collect::<Vec<_>>()
10687 .into_iter()
10688 .rev()
10689 .collect();
10690 let word = word.to_ascii_lowercase();
10691 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10692 continue;
10693 }
10694 let end = chars[..i]
10695 .iter()
10696 .rposition(|c| !c.is_whitespace())
10697 .map_or(i, |p| p + 1);
10698 return Some(end);
10699 }
10700 None
10701}
10702
10703struct PrMessage {
10708 title: String,
10709 body: String,
10710}
10711
10712impl PrMessage {
10713 fn commit_message(&self) -> String {
10717 format!("{}\n\n{}", self.title, self.body)
10718 }
10719}
10720
10721fn title_marker(line: &str) -> Option<&str> {
10723 let line = line.trim();
10724 let head = line.get(..6)?;
10725 head.eq_ignore_ascii_case("title:")
10726 .then(|| line[6..].trim())
10727}
10728
10729fn summary_title(summary: &str) -> Option<String> {
10734 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10735 let raw = title_marker(first)?;
10736 if raw.is_empty() {
10737 return None;
10738 }
10739 let title = queue::title_from(raw, PR_TITLE_MAX);
10740 let lower = title.to_ascii_lowercase();
10741 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10742 return None;
10743 }
10744 Some(title)
10745}
10746
10747const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10749
10750pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10752
10753fn is_review_run(state: &RunState) -> bool {
10757 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10758}
10759
10760fn review_title(state: &RunState) -> Option<String> {
10765 english_subject(
10766 state.reviewed_commits.as_ref()?.first()?,
10767 state.config.graph.github_english(),
10768 )
10769}
10770
10771fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10774 let raw = raw.trim();
10775 if raw.is_empty() {
10776 return None;
10777 }
10778 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10779 return None;
10780 }
10781 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10782 return None;
10783 }
10784 let title = queue::title_from(raw, PR_TITLE_MAX);
10785 let lower = title.to_ascii_lowercase();
10786 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10787 return None;
10788 }
10789 Some(title)
10790}
10791
10792#[derive(Debug, Clone, PartialEq, Eq)]
10795struct BranchFacts {
10796 commits: Vec<(String, String)>,
10798 stat: String,
10800}
10801
10802const STAT_FILE_LINES: usize = 25;
10804const COMMITS_MAX_CHARS: usize = 20_000;
10806
10807async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10810 let commits = git::commit_log(repo, base, branch).await.ok()?;
10811 if commits.is_empty() {
10812 return None;
10813 }
10814 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10815 let lines: Vec<&str> = stat.lines().collect();
10816 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10817 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10818 let more = format!(" ... {omitted} more file(s)");
10819 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10820 kept.push(&more);
10821 kept.push(lines[lines.len() - 1]);
10822 kept.join("\n")
10823 } else {
10824 lines.join("\n")
10825 };
10826 Some(BranchFacts { commits, stat })
10827}
10828
10829fn markdown_safe(text: &str) -> String {
10832 text.replace("</details>", "</details>")
10833 .replace("\x60\x60\x60", "~~~")
10834}
10835
10836fn neutral_title(state: &RunState, winner: char) -> String {
10837 format!(
10838 "chore: land candidate {} of run {}",
10839 winner.to_ascii_uppercase(),
10840 state.id
10841 )
10842}
10843
10844pub(crate) async fn repair_landing_gate(
10847 state: &mut RunState,
10848 winner: &Candidate,
10849 outcomes: &[CommandOutcome],
10850) -> Result<GateFix> {
10851 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10852 return Ok(GateFix::Stop);
10853 }
10854 let mut runner = Runner {
10855 roles: state.config.resolve_roles()?,
10856 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10857 state: state.clone(),
10858 pause: Pause::new(),
10859 interrupt: Pause::new(),
10860 };
10861 let result = runner.gate_fix_round(winner, outcomes).await;
10862 *state = runner.state;
10863 result
10864}
10865
10866pub(crate) async fn check_gate_repair_e2e(
10869 state: &mut RunState,
10870 winner: &Candidate,
10871 wait: &crate::repo_lock::WaitPolicy,
10872) -> Result<GateFix> {
10873 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10874 return Ok(GateFix::Retry);
10875 }
10876 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10877 if state
10878 .gate_repair_e2e
10879 .as_ref()
10880 .is_some_and(|r| r.head == head && r.passed)
10881 {
10882 return Ok(GateFix::Retry);
10883 }
10884 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10885 head: head.clone(),
10886 passed: false,
10887 outcomes: Vec::new(),
10888 });
10889 state.save()?;
10890 let commands = state.config.verify.e2e.clone();
10891 let outcomes = if commands.is_empty() {
10892 Vec::new()
10893 } else {
10894 let shell = state.config.shell();
10895 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10896 let cache_dir = state.config.cache_dir();
10897 let (outcomes, _) = with_cache_lease(
10898 state,
10899 cache_dir.as_deref(),
10900 "e2e",
10901 "e2e",
10902 &winner.worktree,
10903 &head,
10904 timeout,
10905 "gate repair e2e",
10906 wait,
10907 |state, budget| async move {
10908 run_e2e_with_retry(
10909 state,
10910 &shell,
10911 &commands,
10912 &winner.worktree,
10913 budget,
10914 "gate repair e2e",
10915 )
10916 .await
10917 },
10918 )
10919 .await?;
10920 outcomes
10921 };
10922 let deferred = verify_inconclusive(&outcomes);
10923 let passed = outcomes.iter().all(CommandOutcome::ok)
10924 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10925 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10926 head,
10927 passed,
10928 outcomes,
10929 });
10930 state.save()?;
10931 if deferred {
10932 Ok(GateFix::Defer)
10933 } else if !passed {
10934 state.event(
10935 "gate",
10936 "verify.e2e failed after gate repair; nothing published",
10937 );
10938 Ok(GateFix::Stop)
10939 } else {
10940 Ok(GateFix::Retry)
10941 }
10942}
10943
10944pub(crate) async fn run_landing_gate(
10946 state: &mut RunState,
10947 winner: &Candidate,
10948 wait: &crate::repo_lock::WaitPolicy,
10949) -> Result<Vec<CommandOutcome>> {
10950 let shell = state.config.shell();
10951 let gate_commands = state.config.verify.gate.clone();
10952 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10953 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10954 Vec::new()
10955 } else {
10956 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10957 let cache_dir = state.config.cache_dir();
10958 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10959 let (outcomes, _) = with_cache_lease(
10960 state,
10961 cache_dir.as_deref(),
10962 "gate",
10963 "gate",
10964 &winner.worktree,
10965 &head,
10966 timeout,
10967 "final gate",
10968 wait,
10969 |state, budget| {
10970 let shell = shell.clone();
10971 let gate_commands = gate_commands.clone();
10972 let winner = winner.clone();
10973 async move {
10974 Runner::run_pre_gate(state, &winner).await;
10975 let (outcomes, timed_out_pids) = run_commands(
10976 state,
10977 "gate",
10978 "gate",
10979 0,
10980 &shell,
10981 &gate_commands,
10982 &winner.worktree,
10983 budget,
10984 )
10985 .await;
10986 (outcomes, false, timed_out_pids)
10987 }
10988 },
10989 )
10990 .await?;
10991 outcomes
10992 };
10993 if outcomes.is_empty() {
10994 state.event("gate", "no gate commands configured; not tested");
10999 }
11000 for o in &outcomes {
11001 state.event(
11002 "gate",
11003 format!(
11004 "`{}` -> {}",
11005 o.command,
11006 if o.ok() {
11007 "pass".to_owned()
11008 } else {
11009 format!(
11010 "FAIL ({:?})\n{}",
11011 o.code,
11012 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11013 )
11014 }
11015 ),
11016 );
11017 }
11018 Ok(outcomes)
11019}
11020
11021pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11027 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11028 ""
11029 } else {
11030 pr_title
11031 }
11032}
11033
11034pub fn landing_subject_source(state: &RunState) -> String {
11038 if is_review_run(state) {
11039 let winner = state.candidates.first().map_or('A', |c| c.label);
11040 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11041 }
11042 state.instruction.clone()
11043}
11044
11045fn summary_without_title(summary: &str) -> String {
11048 let mut lines = summary.trim().lines().peekable();
11049 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11050 lines.next();
11051 }
11052 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11053}
11054
11055#[cfg(test)]
11067fn pr_message(state: &RunState, winner: char) -> PrMessage {
11068 pr_message_with(state, winner, None)
11069}
11070
11071#[cfg(test)]
11074fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11075 let raw = pr_message_raw(
11076 state,
11077 winner,
11078 facts,
11079 crate::pr_outcome::FinalHead::Unreported,
11080 ChangeNote::NotAsked,
11081 );
11082 let id = crate::scrub::Identity::current();
11083 PrMessage {
11084 title: crate::scrub::scrub(&raw.title, &id),
11085 body: crate::scrub::scrub(&raw.body, &id),
11086 }
11087}
11088
11089#[derive(Debug, Clone, Copy)]
11091enum ChangeNote<'a> {
11092 NotAsked,
11094 Summary(&'a crate::run::ChangeSummary),
11096 Unavailable(&'a str),
11098}
11099
11100fn pr_message_raw(
11101 state: &RunState,
11102 winner: char,
11103 facts: Option<&BranchFacts>,
11104 head: crate::pr_outcome::FinalHead<'_>,
11105 change: ChangeNote<'_>,
11106) -> PrMessage {
11107 if !matches!(change, ChangeNote::NotAsked) {
11108 let (title, mut body) = match change {
11109 ChangeNote::Summary(c) => (
11110 c.title.clone(),
11111 format!("## Summary\n\n{}\n\n", c.body.trim()),
11112 ),
11113 ChangeNote::Unavailable(why) => (
11114 neutral_title(state, winner),
11115 format!(
11116 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11117 ),
11118 ),
11119 ChangeNote::NotAsked => unreachable!(),
11120 };
11121 body.push_str(&public_pr_evidence(state, head));
11122 body.push_str(&format!(
11123 "\n---\nmagi:run/{} magi:candidate-{}\n",
11124 state.id,
11125 winner.to_ascii_lowercase()
11126 ));
11127 return PrMessage { title, body };
11128 }
11129 pr_preview(state, winner, facts, head)
11130}
11131
11132fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11135 let mut body = crate::pr_outcome::review_rounds(state);
11136 body.push_str(&crate::pr_outcome::final_verification(state, head));
11137 let open = state.last_round_findings();
11138 if !open.is_empty() {
11139 body.push_str("## Open review findings\n\n");
11140 for f in open {
11141 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11142 }
11143 body.push('\n');
11144 }
11145 if let Some(c) = state.change_summary.as_ref()
11146 && head == crate::pr_outcome::FinalHead::At(&c.head)
11147 {
11148 body.push_str(&format!(
11149 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11150 c.base, c.head
11151 ));
11152 }
11153 body
11154}
11155
11156fn pr_preview(
11158 state: &RunState,
11159 winner: char,
11160 facts: Option<&BranchFacts>,
11161 head: crate::pr_outcome::FinalHead<'_>,
11162) -> PrMessage {
11163 let summary = state
11164 .candidates
11165 .iter()
11166 .find(|c| c.label == winner)
11167 .map(|c| c.summary.as_str())
11168 .unwrap_or_default();
11169 let review = is_review_run(state);
11172 let title = if review {
11173 facts
11174 .and_then(|f| {
11175 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11176 })
11177 .or_else(|| review_title(state))
11178 .or_else(|| {
11179 state
11180 .candidates
11181 .iter()
11182 .find(|c| c.label == winner)
11183 .filter(|c| !c.branch.starts_with("magi/"))
11184 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11185 })
11186 .unwrap_or_else(|| neutral_title(state, winner))
11187 } else {
11188 summary_title(summary).unwrap_or_else(|| {
11189 english_title(
11190 &state.instruction,
11191 PR_TITLE_MAX,
11192 state.config.graph.github_english(),
11193 )
11194 .unwrap_or_else(|| neutral_title(state, winner))
11195 })
11196 };
11197
11198 let mut body = String::new();
11199 let what = summary_without_title(summary);
11200 if !what.is_empty() {
11201 body.push_str("## Summary\n\n");
11202 body.push_str(&what);
11203 body.push_str("\n\n");
11204 }
11205
11206 body.push_str(&crate::pr_outcome::review_rounds(state));
11208 body.push_str(&crate::pr_outcome::final_verification(state, head));
11209 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11210
11211 let open = state.open_findings();
11212 if !open.is_empty() {
11213 body.push_str("## Open review findings\n\n");
11214 for f in &open {
11215 body.push_str(&format!(
11216 "- `{}` [{:?}] {}\n",
11217 f.id, f.severity, "Context retained in local review records."
11218 ));
11219 }
11220 body.push('\n');
11221 }
11222
11223 if let Some(fix) = fix
11224 && !fix.rejected.is_empty()
11225 {
11226 body.push_str("## Declined by the fixer\n\n");
11227 for r in &fix.rejected {
11228 body.push_str(&format!(
11229 "- `{}`: {}\n",
11230 r.id, "Reason retained in local review records."
11231 ));
11232 }
11233 body.push('\n');
11234 }
11235
11236 if review {
11237 body.push_str("## Commits under review\n\n");
11239 if let Some(facts) = facts {
11240 let mut left = COMMITS_MAX_CHARS;
11241 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11242 let mut entry = format!("- {}\n", markdown_safe(subject));
11243 for l in markdown_safe(text).lines() {
11244 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11245 }
11246 if left == 0 {
11247 body.push_str(&format!(
11248 "- ... {} more commit(s)\n",
11249 facts.commits.len() - i
11250 ));
11251 break;
11252 }
11253 if entry.len() > left {
11254 let mut end = left;
11257 while !entry.is_char_boundary(end) {
11258 end -= 1;
11259 }
11260 entry.truncate(end);
11261 entry.push_str("\n ... (truncated)\n");
11262 left = 0;
11263 } else {
11264 left -= entry.len();
11265 }
11266 body.push_str(&entry);
11267 }
11268 if !facts.stat.trim().is_empty() {
11269 body.push_str(&format!(
11270 "\n## Diff stat\n\n```\n{}\n```\n",
11271 markdown_safe(facts.stat.trim())
11272 ));
11273 }
11274 } else {
11275 match &state.reviewed_commits {
11276 Some(subjects) => {
11277 for s in subjects {
11278 body.push_str(&format!("- {}\n", s.trim()));
11279 }
11280 }
11281 None => {
11282 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11285 body.push_str(rest.trim());
11286 body.push('\n');
11287 }
11288 }
11289 }
11290 } else {
11291 let task = state.instruction.trim();
11292 let task = if task.is_empty() {
11293 "(empty task)"
11294 } else {
11295 task
11296 };
11297 body.push_str(&format!(
11298 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11299 task.replace("</details>", "</details>")
11300 ));
11301 }
11302
11303 body.push_str(&format!(
11304 "\n---\nmagi:run/{} magi:candidate-{}\n",
11305 state.id,
11306 winner.to_ascii_lowercase()
11307 ));
11308
11309 PrMessage { title, body }
11310}
11311
11312fn seeded_instruction(state: &RunState) -> String {
11315 let mut inputs = Vec::new();
11316 if let Some(start) = &state.sources.start_from {
11317 inputs.push(format!(
11318 "Start from `{}` at `{}` (including ancestry).",
11319 start.token, start.sha
11320 ));
11321 }
11322 for pick in &state.sources.pick_commits {
11323 inputs.push(format!(
11324 "Pick `{}` at `{}` (single commit).",
11325 pick.token, pick.sha
11326 ));
11327 }
11328 if inputs.is_empty() {
11329 state.instruction.clone()
11330 } else {
11331 format!(
11332 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11333 state.instruction,
11334 inputs.join("\n")
11335 )
11336 }
11337}
11338
11339async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11343 let base = &state.base_branch;
11344 let mut against = base.clone();
11345 if mode == MergeMode::Pr {
11346 let remote = &state.config.merge.remote;
11350 let tracking = format!("{remote}/{base}");
11351 let fetched = git::fetch(repo, remote, base).await;
11352 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11353 against = tracking;
11354 } else {
11355 return false;
11356 }
11357 }
11358 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11359}
11360
11361fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11364 let mut detail = format!(
11365 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11366 nothing to open a pull request for"
11367 );
11368 detail.push_str(
11369 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11370 );
11371 detail
11372}
11373
11374#[derive(Debug, PartialEq, Eq)]
11377enum PrPlan {
11378 Create,
11379 Adopt { url: String, title: String },
11380 Stop(String),
11381}
11382
11383fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11386 match found {
11387 Ok(land::OpenPr::None) => PrPlan::Create,
11388 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11389 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11390 "several open pull requests exist for this branch, not picking one: {}",
11391 urls.join(" ")
11392 )),
11393 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11394 }
11395}
11396
11397pub(crate) async fn gh_pr_create(
11399 cwd: &Path,
11400 base: &str,
11401 head: &str,
11402 title: &str,
11403 body: &str,
11404) -> Result<String> {
11405 let out = tokio::process::Command::new("gh")
11406 .args([
11407 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11408 ])
11409 .current_dir(cwd)
11410 .quiet()
11411 .stdin(std::process::Stdio::null())
11412 .output()
11413 .await
11414 .context("spawn gh")?;
11415 if out.status.success() {
11416 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11417 } else {
11418 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11419 }
11420}
11421
11422async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11424 match std::fs::symlink_metadata(path) {
11425 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11426 git::worktree_remove_clean(repo, path).await?;
11429 return Ok(());
11430 }
11431 Err(e) => return Err(e.into()),
11432 Ok(_) => {}
11433 }
11434 if !git::is_clean(path).await? {
11435 bail!(
11436 "worktree has uncommitted work; refusing removal: {}",
11437 path.display()
11438 );
11439 }
11440 if !git::worktree_remove_clean(repo, path).await? {
11441 bail!("worktree removal refused: {}", path.display());
11442 }
11443 Ok(())
11444}
11445
11446pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11455 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11456 bail!("refusing fold: {reason}");
11457 }
11458 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11459 bail!("candidate preservation incomplete; refusing cleanup");
11460 }
11461 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11465 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11466 let _driver = crate::driver::maintenance(home, &state.id)?;
11467 let mut output_guards = Vec::new();
11468 for entry in std::fs::read_dir(state.worktree_root())
11469 .into_iter()
11470 .flatten()
11471 .flatten()
11472 {
11473 if entry.path().join(".git").is_file() {
11474 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11475 }
11476 }
11477 let repo = state.repo.clone();
11478 let wait = repository_wait(state, None);
11481 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11482 Ok(lease) => lease,
11483 Err(e) => {
11484 state.save_under(home)?;
11485 return Err(e);
11486 }
11487 };
11488 let _source_outputs = if state.status == RunStatus::Merged
11491 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11492 {
11493 verify_merged_fold_sources(state, drop_winner, home).await?
11494 } else {
11495 Vec::new()
11496 };
11497 let root = state.worktree_root();
11498 let winner = state.tally.as_ref().map(|t| t.winner);
11499 let mut removed = Vec::new();
11500
11501 for i in 0..state.candidates.len() {
11502 let c = state.candidates[i].clone();
11503 let is_winner = Some(c.label) == winner;
11504 if is_winner && !drop_winner {
11505 continue;
11506 }
11507 let existed = c.worktree.exists();
11508 remove_preserved_worktree(&repo, &c.worktree).await?;
11511 if existed {
11512 removed.push(c.worktree.to_string_lossy().into_owned());
11513 }
11514 let handed_over = state.released_branches.contains(&c.branch);
11517 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11518 if !git::branch_delete(&repo, &c.branch).await? {
11519 bail!("candidate branch removal refused");
11520 }
11521 removed.push(c.branch.clone());
11522 }
11523 state.candidates[i].folded = true;
11524 }
11525
11526 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11527 let path = name.path();
11528 let keep = !drop_winner
11529 && winner.is_some_and(|w| {
11530 path.file_name()
11531 .is_some_and(|n| n == format!("cand-{w}").as_str())
11532 });
11533 if keep {
11534 continue;
11535 }
11536 remove_preserved_worktree(&repo, &path).await?;
11537 removed.push(path.to_string_lossy().into_owned());
11538 }
11539
11540 remove_if_empty(&root);
11549
11550 if state.enabled_worktree_config && drop_winner {
11551 git::release_worktree_config(&repo).await.ok();
11555 state.enabled_worktree_config = false;
11556 }
11557 state.save_under(home)?;
11558 Ok(removed)
11559}
11560
11561async fn verify_merged_fold_sources(
11562 state: &RunState,
11563 drop_winner: bool,
11564 home: &Path,
11565) -> Result<Vec<std::fs::File>> {
11566 let winner = state.tally.as_ref().map(|t| t.winner);
11567 let mut heads = BTreeSet::new();
11568 for candidate in &state.candidates {
11569 if Some(candidate.label) == winner && !drop_winner {
11570 continue;
11571 }
11572 anyhow::ensure!(
11573 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11574 "candidate recovery source is outside the inventoried run root: {}",
11575 candidate.worktree.display()
11576 );
11577 match std::fs::symlink_metadata(&candidate.worktree) {
11578 Ok(_) => {}
11579 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11580 Err(error) => return Err(error.into()),
11581 }
11582 if !state.released_branches.contains(&candidate.branch)
11583 && git::branch_exists(&state.repo, &candidate.branch).await?
11584 {
11585 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11586 }
11587 }
11588 let mut cargo_guards = Vec::new();
11589 let entries = match std::fs::read_dir(state.worktree_root()) {
11590 Ok(entries) => Some(entries),
11591 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11592 Err(error) => return Err(error.into()),
11593 };
11594 for entry in entries.into_iter().flatten() {
11595 let path = entry?.path();
11596 let metadata = std::fs::symlink_metadata(&path)?;
11597 anyhow::ensure!(
11598 metadata.is_dir() && !metadata.file_type().is_symlink(),
11599 "non-directory or symlink recovery source retained: {}",
11600 path.display()
11601 );
11602 if !drop_winner
11603 && winner.is_some_and(|w| {
11604 path.file_name()
11605 .is_some_and(|name| name == format!("cand-{w}").as_str())
11606 })
11607 {
11608 continue;
11609 }
11610 anyhow::ensure!(
11611 path.join(".git").is_file(),
11612 "unregistered recovery source retained: {}",
11613 path.display()
11614 );
11615 anyhow::ensure!(
11616 git::git(
11617 &path,
11618 &[
11619 "status",
11620 "--porcelain",
11621 "--untracked-files=all",
11622 "--ignored",
11623 "--",
11624 ".",
11625 ":(exclude)target"
11626 ]
11627 )
11628 .await?
11629 .is_empty(),
11630 "source changes or ignored recovery files retained: {}",
11631 path.display()
11632 );
11633 match std::fs::symlink_metadata(path.join("target")) {
11634 Ok(_) => {
11635 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11636 }
11637 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11638 Err(error) => return Err(error.into()),
11639 }
11640 for checkpoint in [
11641 "rebase-merge",
11642 "rebase-apply",
11643 "sequencer",
11644 "CHERRY_PICK_HEAD",
11645 "MERGE_HEAD",
11646 "REBASE_HEAD",
11647 "REVERT_HEAD",
11648 "BISECT_LOG",
11649 "AUTO_MERGE",
11650 "MERGE_MSG",
11651 ] {
11652 let location = git::git(
11653 &path,
11654 &[
11655 "rev-parse",
11656 "--path-format=absolute",
11657 "--git-path",
11658 checkpoint,
11659 ],
11660 )
11661 .await?;
11662 anyhow::ensure!(
11663 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11664 if e.kind() == std::io::ErrorKind::NotFound),
11665 "Git recovery checkpoint retained: {checkpoint}"
11666 );
11667 }
11668 heads.insert(git::rev_parse(&path, "HEAD").await?);
11669 }
11670 crate::fold_evidence::preserve(state, home, &heads).await?;
11671
11672 Ok(cargo_guards)
11673}
11674
11675fn remove_if_empty(dir: &Path) {
11686 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11687 std::fs::remove_dir(dir).ok();
11688 }
11689}
11690
11691pub fn worst_open(state: &RunState) -> Option<Severity> {
11693 state
11694 .reviews
11695 .last()?
11696 .reviews
11697 .iter()
11698 .flat_map(|r| r.findings.iter())
11699 .map(|f| f.severity)
11700 .max()
11701}
11702
11703async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11707 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11708 git::acquire_worktree_config(repo).await?;
11709 state.enabled_worktree_config = true;
11710 }
11711 Ok(())
11712}
11713
11714#[cfg(test)]
11715mod tests {
11716 #[tokio::test]
11717 async fn launcher_base_fetch_requires_remote_success() {
11718 let tmp = tempfile::tempdir().unwrap();
11719 let repo = tmp.path().join("repo");
11720 let remote = tmp.path().join("remote");
11721 std::fs::create_dir(&repo).unwrap();
11722 std::fs::create_dir(&remote).unwrap();
11723 git::git(&remote, &["init", "--bare", "-b", "main"])
11724 .await
11725 .unwrap();
11726 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11727 git::git(
11728 &repo,
11729 &[
11730 "-c",
11731 "user.name=Fixture",
11732 "-c",
11733 "user.email=fixture@example.invalid",
11734 "commit",
11735 "--allow-empty",
11736 "-m",
11737 "fixture",
11738 ],
11739 )
11740 .await
11741 .unwrap();
11742 git::git(
11743 &repo,
11744 &["remote", "add", "origin", remote.to_str().unwrap()],
11745 )
11746 .await
11747 .unwrap();
11748 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11749 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11750 assert_eq!(
11751 resolve_base(&repo, "main", "origin", None)
11752 .await
11753 .unwrap()
11754 .commit,
11755 head
11756 );
11757 #[cfg(unix)]
11758 {
11759 let policy: crate::config::Coordination = toml::from_str(
11760 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11761 ).unwrap();
11762 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11763 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11764 let held = crate::repo_lock::SeatLease::operation_with(
11765 &repo,
11766 Some(&policy),
11767 &crate::repo_lock::WaitPolicy::default(),
11768 )
11769 .await
11770 .unwrap();
11771 assert!(held.is_some());
11772 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11773 tokio::pin!(fetch);
11774 assert!(
11775 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11776 .await
11777 .is_err()
11778 );
11779 assert!(
11780 !repo.join(".git/FETCH_HEAD").exists(),
11781 "no fetch before admission"
11782 );
11783 drop(held);
11784 let fetched = fetch.await.unwrap();
11785 assert_eq!(fetched.commit, head);
11786 assert_eq!(fetched.remote, "origin");
11787 assert_eq!(fetched.branch, "main");
11788 }
11789 git::git(
11791 &repo,
11792 &[
11793 "remote",
11794 "set-url",
11795 "origin",
11796 tmp.path().join("missing").to_str().unwrap(),
11797 ],
11798 )
11799 .await
11800 .unwrap();
11801 let error = resolve_base(&repo, "main", "origin", None)
11802 .await
11803 .unwrap_err()
11804 .to_string();
11805 assert!(error.contains("cannot read origin/main"));
11806 assert!(error.contains("refusing to branch off"));
11807 }
11808
11809 #[test]
11810 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11811 let leaked = vec!["chore(deps): update a crate".to_owned()];
11812 let own = "fix(daemon): apply a chosen action";
11813 assert!(should_retitle("", own, &leaked));
11814 assert!(should_retitle(
11815 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11816 own,
11817 &leaked
11818 ));
11819 assert!(should_retitle(
11820 "chore: land candidate A of run 1",
11821 own,
11822 &leaked
11823 ));
11824 assert!(should_retitle(
11825 "magi: candidate A (uncommitted work)",
11826 own,
11827 &leaked
11828 ));
11829 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11830 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11831 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11832 }
11833
11834 #[test]
11835 fn pr_merge_plan_creates_adopts_or_stops() {
11836 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11837 assert_eq!(
11838 pr_merge_plan(Ok(land::OpenPr::One {
11839 url: "u".into(),
11840 title: "t".into()
11841 })),
11842 PrPlan::Adopt {
11843 url: "u".into(),
11844 title: "t".into()
11845 }
11846 );
11847 let PrPlan::Stop(many) =
11848 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11849 else {
11850 panic!("many must stop");
11851 };
11852 assert!(many.contains('a') && many.contains('b'));
11853 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11854 panic!("a failed lookup must stop");
11855 };
11856 assert!(err.contains("bad token"));
11857 }
11858
11859 use super::*;
11860 use crate::run::GateStatus;
11861 use std::collections::BTreeMap;
11862 use std::time::Duration;
11863
11864 fn conductor() -> AgentSpec {
11865 AgentSpec {
11866 id: "conductor".to_owned(),
11867 kind: crate::config::AgentKind::Command,
11868 model: None,
11869 command: vec!["true".to_owned()],
11870 extra_args: Vec::new(),
11871 env: BTreeMap::new(),
11872 prompt_delivery: None,
11873 }
11874 }
11875
11876 fn spec(id: &str) -> AgentSpec {
11877 AgentSpec {
11878 id: id.to_owned(),
11879 kind: crate::config::AgentKind::Command,
11880 model: None,
11881 command: vec!["true".to_owned()],
11882 extra_args: Vec::new(),
11883 env: BTreeMap::new(),
11884 prompt_delivery: None,
11885 }
11886 }
11887
11888 fn ids(xs: &[&str]) -> BTreeSet<String> {
11889 xs.iter().map(|s| (*s).to_owned()).collect()
11890 }
11891
11892 #[test]
11893 fn pick_successor_skips_an_agent_another_seat_holds() {
11894 let roster = [spec("a"), spec("b"), spec("c")];
11896 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11897 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11898 }
11899
11900 #[test]
11901 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11902 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11904 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11905 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11906 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11907 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11908 }
11909
11910 #[test]
11911 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11912 let roster = [spec("a"), spec("b")];
11913 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11914 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11915 let carried = ids(&["b"]);
11916 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11917 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11918 }
11919
11920 #[test]
11921 fn pick_successor_returns_none_without_any_untried_successor() {
11922 let roster = [spec("a"), spec("b")];
11923 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11924 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11925 let carried = ids(&["a"]);
11926 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11927 }
11928
11929 #[test]
11930 fn pick_successor_rescue_avoids_another_seats_occupant() {
11931 let roster = [spec("a"), spec("b"), spec("c")];
11933 let carried = ids(&["b", "c"]);
11934 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11935 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11936 }
11937
11938 #[test]
11939 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11940 let roster = [spec("a"), spec("b"), spec("c")];
11941 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11942 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11943 }
11944
11945 #[test]
11946 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11947 let roster = [spec("a"), spec("b"), spec("c")];
11948 let failed = ids(&["a", "b", "c"]);
11949 let mut tried = ids(&["b"]);
11951 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11952 assert_eq!(first.id, "a");
11953 tried.insert(first.id.clone());
11954 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11955 assert_eq!(second.id, "c");
11956 tried.insert(second.id.clone());
11957 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11958 }
11959
11960 #[test]
11961 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11962 let roster = [spec("a"), spec("b")];
11963 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11964 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11965 }
11966
11967 #[test]
11968 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11969 let roster = [spec("a"), spec("b"), spec("c")];
11970 let h = SeatHistory {
11971 failed: ids(&["a"]),
11972 last_ok: Some("b".to_owned()),
11973 last_fail: None,
11974 };
11975 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11976 let gone = SeatHistory {
11978 last_ok: Some("zzz".to_owned()),
11979 ..h.clone()
11980 };
11981 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11982 let failed = SeatHistory {
11983 failed: ids(&["a", "b"]),
11984 last_ok: Some("b".to_owned()),
11985 last_fail: None,
11986 };
11987 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11988 }
11989
11990 #[test]
11991 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11992 let first = SeatState::new("review-1", "alpha", 7);
11993 let next = handover_seat("review-1", "gamma", 7);
11994 assert_ne!(first.claude_session, next.claude_session);
11995 }
11996
11997 #[test]
11998 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11999 let mut state = state_with_summary("x", "y");
12000 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12001 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12002 assert_ne!(a.claude_session, b.claude_session);
12003 }
12004
12005 #[test]
12006 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12007 let roster = [spec("a"), spec("b")];
12008 let h = SeatHistory {
12009 failed: ids(&["a", "b"]),
12010 ..SeatHistory::default()
12011 };
12012 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12013 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12014 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12015 }
12016
12017 #[test]
12024 fn failure_signature_ignores_numbers_and_paths() {
12025 assert_eq!(
12026 failure_signature("exited with Some(2) and no usable output"),
12027 failure_signature("exited with Some(137) and no usable output")
12028 );
12029 assert_eq!(
12030 failure_signature("process exited with no usable output (exit code 2)"),
12031 failure_signature("process exited with no usable output (exit code 137)")
12032 );
12033 for (old, new) in [
12034 (
12035 "exited with Some(2) and no usable output",
12036 "process exited with no usable output (exit code 2)",
12037 ),
12038 (
12039 "exited with None and no usable output",
12040 "process exited with no usable output (exit code unavailable)",
12041 ),
12042 ] {
12043 let previous = FailClass::Other(failure_signature(old));
12044 let current = FailClass::Other(failure_signature(new));
12045 assert_eq!(previous, current, "resumption retains the failure class");
12046 assert!(!should_hand_over(Some(&previous), ¤t));
12047 }
12048 assert_eq!(
12049 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12050 failure_signature("cannot open /var/x.txt: denied")
12051 );
12052 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12053 }
12054
12055 #[test]
12056 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12057 let other = FailClass::Other("x".into());
12058 assert!(should_hand_over(None, &FailClass::Quota));
12059 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12060 assert!(should_hand_over(
12061 Some(&FailClass::Timeout),
12062 &FailClass::Timeout
12063 ));
12064 assert!(should_hand_over(None, &other));
12065 assert!(!should_hand_over(Some(&other), &other));
12066 assert!(should_hand_over(
12067 Some(&other),
12068 &FailClass::Other("y".into())
12069 ));
12070 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12072 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12073 }
12074
12075 #[test]
12076 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12077 let a = SeatState::new("judge-1", "alpha", 7);
12078 let b = handover_seat("judge-1", "beta", 7);
12079 assert_ne!(a.claude_session, b.claude_session);
12080 assert_eq!(b.turns, 0);
12081 }
12082
12083 #[test]
12084 fn a_timeout_is_classified_apart_from_other_failures() {
12085 assert_eq!(
12086 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12087 Some(FailClass::Timeout)
12088 );
12089 assert!(matches!(
12090 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12091 Some(FailClass::Other(_))
12092 ));
12093 }
12094
12095 #[test]
12096 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12097 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12098 let tried = BTreeSet::from(["beta".to_owned()]);
12099 let next = next_untried_in_roster(&roster, 1, &tried);
12102 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12103 }
12104
12105 #[test]
12106 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12107 let roster = vec![spec("alpha"), spec("beta")];
12108 let tried = BTreeSet::from(["beta".to_owned()]);
12109 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12113 }
12114
12115 #[test]
12116 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12117 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12118 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12119 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12123 }
12124
12125 #[test]
12126 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12127 let roster = vec![spec("a"), spec("a"), spec("b")];
12128 let tried = BTreeSet::from(["a".to_owned()]);
12129 let next = next_untried_in_roster(&roster, 0, &tried);
12130 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12131 }
12132
12133 #[test]
12134 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12135 let roster = vec![spec("a"), spec("b")];
12136 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12137 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12138 }
12139
12140 #[test]
12141 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12142 let dir = tempfile::tempdir().unwrap();
12143 let bay = dir.path().join("ffff");
12144
12145 remove_if_empty(&bay);
12147 assert!(!bay.exists());
12148
12149 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12152 remove_if_empty(&bay);
12153 assert!(bay.exists(), "non-empty directory must survive");
12154
12155 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12157 remove_if_empty(&bay);
12158 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12159 }
12160
12161 #[test]
12170 fn a_full_panel_that_found_nothing_is_clean() {
12171 assert!(round_is_clean(
12172 0,
12173 true,
12174 2,
12175 2,
12176 0,
12177 IncompleteReviewPolicy::Block
12178 ));
12179 }
12180
12181 #[test]
12182 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12183 assert!(!round_is_clean(
12184 0,
12185 true,
12186 1,
12187 2,
12188 0,
12189 IncompleteReviewPolicy::Block
12190 ));
12191 }
12192
12193 #[test]
12194 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12195 assert!(!round_is_clean(
12196 1,
12197 true,
12198 1,
12199 2,
12200 0,
12201 IncompleteReviewPolicy::Warn
12202 ));
12203 }
12204
12205 #[test]
12206 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12207 assert!(round_is_clean(
12208 0,
12209 true,
12210 1,
12211 2,
12212 0,
12213 IncompleteReviewPolicy::Warn
12214 ));
12215 }
12216
12217 #[test]
12218 fn a_full_panel_with_an_open_finding_is_not_clean() {
12219 assert!(!round_is_clean(
12220 1,
12221 true,
12222 2,
12223 2,
12224 0,
12225 IncompleteReviewPolicy::Block
12226 ));
12227 }
12228
12229 #[test]
12230 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12231 assert!(!round_is_clean(
12232 0,
12233 false,
12234 2,
12235 2,
12236 0,
12237 IncompleteReviewPolicy::Block
12238 ));
12239 }
12240
12241 #[test]
12248 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12249 assert!(round_is_clean(
12252 0,
12253 true,
12254 1,
12255 2,
12256 1,
12257 IncompleteReviewPolicy::Block
12258 ));
12259 }
12260
12261 #[test]
12262 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12263 assert!(!round_is_clean(
12266 0,
12267 true,
12268 1,
12269 2,
12270 0,
12271 IncompleteReviewPolicy::Block
12272 ));
12273 }
12274
12275 #[test]
12276 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12277 assert!(!round_is_clean(
12278 1,
12279 true,
12280 1,
12281 2,
12282 1,
12283 IncompleteReviewPolicy::Block
12284 ));
12285 assert!(!round_is_clean(
12286 0,
12287 false,
12288 1,
12289 2,
12290 1,
12291 IncompleteReviewPolicy::Block
12292 ));
12293 }
12294
12295 #[test]
12296 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12297 assert!(!round_is_clean(
12301 0,
12302 true,
12303 0,
12304 2,
12305 2,
12306 IncompleteReviewPolicy::Block
12307 ));
12308 }
12309
12310 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12311 CommandOutcome {
12312 command: "test".to_owned(),
12313 code,
12314 output_tail: String::new(),
12315 duration_ms: 0,
12316 resource_blocked,
12317 }
12318 }
12319
12320 #[test]
12321 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12322 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12323 assert!(
12324 !verify_inconclusive(&[outcome(Some(1), false)]),
12325 "an ordinary failure is still evidence about the patch"
12326 );
12327 assert!(verify_inconclusive(&[outcome(None, true)]));
12328 assert!(
12329 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12330 "one inconclusive outcome taints the whole batch"
12331 );
12332 assert!(!verify_inconclusive(&[]));
12333 }
12334
12335 #[tokio::test]
12336 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12337 let calls = std::sync::atomic::AtomicUsize::new(0);
12341 let started = Instant::now();
12342 wait_for_pids_with(
12343 &[123],
12344 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12345 Duration::from_millis(5),
12346 Duration::from_secs(5),
12347 )
12348 .await;
12349 assert!(
12350 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12351 "must keep checking rather than deciding on the first answer"
12352 );
12353 assert!(
12354 started.elapsed() < Duration::from_secs(1),
12355 "must return the moment it is confirmed dead, not wait out the ceiling"
12356 );
12357 }
12358
12359 #[tokio::test]
12360 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12361 let started = Instant::now();
12362 wait_for_pids_with(
12363 &[123],
12364 |_| true, Duration::from_millis(5),
12366 Duration::from_millis(30),
12367 )
12368 .await;
12369 let elapsed = started.elapsed();
12370 assert!(
12371 elapsed >= Duration::from_millis(30),
12372 "must not give up before its own ceiling: {elapsed:?}"
12373 );
12374 assert!(
12375 elapsed < Duration::from_secs(1),
12376 "must not wait past its own ceiling either: {elapsed:?}"
12377 );
12378 }
12379
12380 #[tokio::test]
12381 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12382 let started = Instant::now();
12383 wait_for_pids_with(
12384 &[],
12385 |_| true,
12386 Duration::from_secs(5),
12387 Duration::from_secs(5),
12388 )
12389 .await;
12390 assert!(
12391 started.elapsed() < Duration::from_millis(200),
12392 "an empty pid list has nothing to confirm"
12393 );
12394 }
12395
12396 fn review_round(
12402 clean: bool,
12403 blocking: usize,
12404 answered: usize,
12405 expected: usize,
12406 progressed: bool,
12407 e2e_ok: bool,
12408 ) -> ReviewRound {
12409 ReviewRound {
12410 round: 1,
12411 head: "h".to_owned(),
12412 verified_head: None,
12413 verified_at: None,
12414 reviews: Vec::new(),
12415 e2e: vec![CommandOutcome {
12416 command: "test".to_owned(),
12417 code: Some(if e2e_ok { 0 } else { 1 }),
12418 output_tail: String::new(),
12419 duration_ms: 0,
12420 resource_blocked: false,
12421 }],
12422 verify_retried: false,
12423 e2e_deferred: false,
12424 e2e_defer_reason: None,
12425 fix: None,
12426 blocking,
12427 answered,
12428 expected,
12429 clean,
12430 progressed,
12431 vote_split: false,
12432 reconsideration: Vec::new(),
12433 verdict: None,
12434 }
12435 }
12436
12437 #[test]
12438 fn review_conclusion_is_none_when_nothing_has_run() {
12439 assert_eq!(review_conclusion(&[], 3), None);
12440 }
12441
12442 #[test]
12443 fn review_conclusion_is_none_while_rounds_remain() {
12444 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12445 assert_eq!(review_conclusion(&rounds, 3), None);
12446 }
12447
12448 #[test]
12449 fn review_conclusion_is_gating_once_a_round_is_clean() {
12450 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12451 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12452 }
12453
12454 #[test]
12455 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12456 let rounds = vec![
12457 review_round(false, 1, 2, 2, true, true),
12458 review_round(false, 1, 2, 2, true, true),
12459 ];
12460 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12461 }
12462
12463 #[test]
12464 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12465 let rounds = vec![
12466 review_round(false, 1, 2, 2, true, true),
12467 review_round(false, 1, 2, 2, true, false),
12468 ];
12469 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12470 }
12471
12472 #[test]
12473 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12474 let mut blocked = review_round(false, 1, 2, 2, true, false);
12481 blocked.e2e[0].resource_blocked = true;
12482 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12483 assert_eq!(review_conclusion(&rounds, 2), None);
12484 }
12485
12486 #[test]
12487 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12488 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12490 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12491 }
12492
12493 #[test]
12494 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12495 let rounds = vec![
12496 review_round(false, 1, 2, 2, false, true),
12497 review_round(false, 1, 2, 2, false, true),
12498 ];
12499 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12500 }
12501
12502 fn secs(n: u64) -> Duration {
12503 Duration::from_secs(n)
12504 }
12505
12506 #[tokio::test]
12509 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12510 ask_test_home();
12511 let tmp = tempfile::tempdir().unwrap();
12512 let repo = tmp.path().join("repo");
12513 std::fs::create_dir_all(&repo).unwrap();
12514 init_repo(&repo);
12515 let g = |args: &[&str]| {
12516 let out = std::process::Command::new("git")
12517 .args(args)
12518 .current_dir(&repo)
12519 .quiet()
12520 .output()
12521 .unwrap();
12522 assert!(
12523 out.status.success(),
12524 "{}",
12525 String::from_utf8_lossy(&out.stderr)
12526 );
12527 };
12528 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12529 g(&["checkout", "-b", "historical"]);
12530 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12531 g(&["add", "."]);
12532 g(&["commit", "-m", "old alternative"]);
12533 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12534 g(&["checkout", "-b", "magi/task-selected", &base]);
12535 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12536 g(&["add", "."]);
12537 g(&["commit", "-m", "measurement infrastructure"]);
12538 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12539 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12540 g(&["add", "."]);
12541 g(&["commit", "-m", "dependent implementation"]);
12542 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12543 g(&["checkout", "main"]);
12544 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12545 g(&["add", "."]);
12546 g(&["commit", "-m", "unrelated base change"]);
12547 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12548 let origin = tmp.path().join("origin.git");
12549 g(&[
12550 "clone",
12551 "--bare",
12552 repo.to_str().unwrap(),
12553 origin.to_str().unwrap(),
12554 ]);
12555 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12556 g(&["fetch", "origin"]);
12557 let selection = crate::pr_recovery::Selection {
12558 url: "https://example.test/project/pull/1".into(),
12559 branch: "magi/task-selected".into(),
12560 head: head.clone(),
12561 source_head: None,
12562 source_run: None,
12563 source_proof: None,
12564 disposition: Some(crate::pr_recovery::Disposition::Review),
12565 instruction: "preserve measured behavior".into(),
12566 approved_at: jiff::Timestamp::now(),
12567 };
12568 let mut template = runner_at(RunStatus::Reviewing);
12569 template.state.repo = repo.clone();
12570 template.state.base_commit = current.clone();
12571 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12572 template.state.origin = Some(crate::run::Origin::queue("selected"));
12573 template.state.pr_head = Some(selection.branch.clone());
12574 template.state.pr_recovery = Some(selection.clone());
12575 let mut runner =
12576 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12577 .await
12578 .unwrap();
12579 let wt = runner.state.winner().unwrap().worktree.clone();
12580 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12581 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12582 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12583 assert!(runner.state.seeds.is_empty());
12584 assert_eq!(runner.state.instruction, "preserve measured behavior");
12585 runner.sync_to_base().await.unwrap();
12586 assert_eq!(
12587 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12588 "prerequisite"
12589 );
12590 assert_eq!(
12591 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12592 "measured implementation"
12593 );
12594 assert_eq!(
12595 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12596 "current main change"
12597 );
12598 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12599 assert_eq!(
12600 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12601 head
12602 );
12603
12604 let mut partial = runner_at(RunStatus::Reviewing);
12606 partial.state.repo = repo.clone();
12607 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12608 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12609 run: Some("exit 1".into()),
12610 ..Default::default()
12611 }];
12612 let selection = runner.state.pr_recovery.clone().unwrap();
12613 partial.state.pr_recovery = Some(selection.clone());
12614 partial.state.pr_head = Some(selection.branch.clone());
12615 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12616 let id = partial.state.id.clone();
12617 let error = match Runner::open_selected(
12618 &repo,
12619 partial.state,
12620 partial.roles,
12621 1,
12622 runner.state.base_commit.clone(),
12623 selection,
12624 )
12625 .await
12626 {
12627 Ok(_) => panic!("setup must fail"),
12628 Err(e) => e,
12629 };
12630 assert_eq!(
12631 error
12632 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12633 .unwrap()
12634 .run,
12635 id
12636 );
12637 let failed = RunState::load(&id).unwrap();
12638 assert_eq!(failed.status, RunStatus::Blocked);
12639 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12640 assert!(Runner::resume(&id).is_err());
12641 runner.state.status = RunStatus::Ready;
12642 runner.state.driver_exited = true;
12643 runner.state.save().unwrap();
12644 let mut retry = runner_at(RunStatus::Reviewing);
12645 retry.state.repo = repo.clone();
12646 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12647 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12648 reselected.approved_at = jiff::Timestamp::now();
12649 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12650 retry.state.pr_head = Some(reselected.branch.clone());
12651 retry.state.pr_recovery = Some(reselected.clone());
12652 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12653 .await
12654 .unwrap();
12655 let restarted = Runner::open_selected(
12656 &repo,
12657 retry.state,
12658 retry.roles,
12659 1,
12660 runner.state.base_commit.clone(),
12661 reselected,
12662 )
12663 .await
12664 .unwrap();
12665 assert_eq!(
12666 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12667 .await
12668 .unwrap(),
12669 head
12670 );
12671 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12672 assert_eq!(
12673 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12674 head
12675 );
12676 }
12677
12678 #[tokio::test]
12679 #[cfg(unix)]
12680 async fn fallback_and_interrupted_restoration_require_coordination() {
12681 use std::os::fd::AsRawFd as _;
12682 ask_test_home();
12683 let tmp = tempfile::tempdir().unwrap();
12684 let mut runner = registered_runner(tmp.path()).await;
12685 runner.state.config.graph.timeout_resource_wait = 1;
12686 let winner = runner.state.candidates[0].clone();
12687 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12688 let leftover = winner.worktree.join("leftover");
12689 std::fs::write(&leftover, "untrusted edits").unwrap();
12690 let next = tmp.path().join("next-fixer");
12691 let mut failed = spec("first");
12692 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12693 let mut successor = spec("second");
12694 successor.command = vec![
12695 "sh".into(),
12696 "-c".into(),
12697 format!("touch '{}'; echo done", next.display()),
12698 ];
12699 let attempts = vec![
12700 (failed.clone(), "first".into()),
12701 (successor.clone(), "second".into()),
12702 ];
12703 let lock_path = runner.state.repo.join(".git/updater.lock");
12704 let external = std::fs::File::create(&lock_path).unwrap();
12705 assert_eq!(
12706 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12707 0
12708 );
12709 let artifacts = tmp.path().join("artifacts");
12710 let build = |spec, seat| SeatJob {
12711 spec,
12712 seat,
12713 cwd: winner.worktree.clone(),
12714 prompt: "fixture".into(),
12715 timeout: Duration::from_secs(10),
12716 allow_write: true,
12717 sessions: false,
12718 artifacts: artifacts.clone(),
12719 stem: "fix".into(),
12720 handover: None,
12721 };
12722 let (_, _, out) = runner
12723 .ask_fixer_among(
12724 attempts.clone(),
12725 "graft",
12726 None,
12727 Some((&winner.worktree, &head)),
12728 build,
12729 )
12730 .await;
12731 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12732 assert!(leftover.exists(), "contention must prevent clean");
12733 assert!(!next.exists());
12734 assert!(runner.state.handovers.is_empty());
12735 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12737 from_run: "earlier".into(),
12738 generation: 1,
12739 mode: crate::recovery::Mode::VerifyRepair,
12740 reasons: Vec::new(),
12741 reviewed_head: None,
12742 rounds: Vec::new(),
12743 open_findings: Vec::new(),
12744 evidence: None,
12745 criteria_inherited: true,
12746 repair: crate::recovery::Repair::Started {
12747 before: head.clone(),
12748 },
12749 });
12750 assert!(
12751 runner
12752 .recovery_repair(&winner)
12753 .await
12754 .unwrap_err()
12755 .is::<crate::repo_lock::ResourceBlocked>()
12756 );
12757 assert!(matches!(
12758 runner.state.recovery.as_ref().unwrap().repair,
12759 crate::recovery::Repair::Started { .. }
12760 ));
12761 assert!(leftover.exists());
12762 drop(external);
12763 runner.recovery_repair(&winner).await.unwrap();
12764 assert!(!leftover.exists());
12765 assert!(matches!(
12766 runner.state.recovery.as_ref().unwrap().repair,
12767 crate::recovery::Repair::Done { .. }
12768 ));
12769 std::fs::remove_file(&lock_path).unwrap();
12771 std::fs::create_dir(&lock_path).unwrap();
12772 std::fs::write(&leftover, "untrusted edits").unwrap();
12773 let (_, _, out) = runner
12774 .ask_fixer_among(
12775 attempts,
12776 "graft",
12777 None,
12778 Some((&winner.worktree, &head)),
12779 build,
12780 )
12781 .await;
12782 assert!(matches!(out, AgentOutcome::Failed(_)));
12783 assert!(leftover.exists());
12784 assert!(!next.exists());
12785 assert_eq!(
12786 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12787 head
12788 );
12789 }
12790
12791 #[cfg(unix)]
12792 async fn registered_runner(root: &Path) -> Runner {
12793 ask_test_home();
12794 let repo = root.join("repo");
12795 std::fs::create_dir_all(&repo).unwrap();
12796 init_repo(&repo);
12797 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12798 let mut runner = runner_at(RunStatus::Prep);
12799 runner.state.repo = repo.clone();
12800 runner.state.config.coordination = crate::config::test_coordination();
12801 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12802 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12803 runner.state.config.graph.advise = false;
12804 runner.roles.implementers = vec![spec("implement")];
12805 runner.roles.judges = vec![spec("judge")];
12806 runner.prep().await.unwrap();
12807 runner
12808 }
12809
12810 #[tokio::test]
12811 #[cfg(unix)]
12812 async fn parked_implementer_persists_completed_work() {
12813 for registered in [false, true] {
12814 let tmp = tempfile::tempdir().unwrap();
12815 let mut runner = registered_runner(tmp.path()).await;
12816 if !registered {
12817 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12818 }
12819 let marker = tmp.path().join("started");
12820 let release = tmp.path().join("release");
12821 let mut implementer = spec("implement");
12822 implementer.command = vec![
12823 "sh".into(),
12824 "-c".into(),
12825 format!(
12826 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12827 marker.display(),
12828 release.display()
12829 ),
12830 ];
12831 runner.roles.implementers = vec![implementer.clone()];
12832 runner.state.config.agents = vec![implementer];
12833 let pause = runner.pause.clone();
12834 let (result, ()) = tokio::join!(runner.implement(), async {
12835 tokio::time::timeout(Duration::from_secs(10), async {
12836 while !marker.exists() {
12837 tokio::time::sleep(Duration::from_millis(10)).await;
12838 }
12839 })
12840 .await
12841 .unwrap();
12842 pause.park();
12843 std::fs::write(&release, "go").unwrap();
12844 });
12845 result.unwrap();
12846 let candidate = &runner.state.candidates[0];
12847 assert!(candidate.commits > 0, "{candidate:?}");
12848 assert!(candidate.summary.contains("saved-result"));
12849 assert_eq!(candidate.files, 1);
12850 let before = candidate.commits;
12851 runner.implement().await.unwrap();
12852 assert_eq!(runner.state.candidates[0].commits, before);
12853 }
12854 }
12855
12856 #[tokio::test]
12857 #[cfg(unix)]
12858 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12859 let tmp = tempfile::tempdir().unwrap();
12860 let mut runner = registered_runner(tmp.path()).await;
12861 let winner = runner.state.candidates[0].clone();
12862 let cache = tmp.path().join("cache");
12863 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12864 runner.state.config.graph.timeout_resource_wait = 0;
12865 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12866 let crate::cache::AcquireOutcome::Acquired(held) =
12867 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12868 else {
12869 panic!("cache must be free")
12870 };
12871 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12872 from_run: "earlier".into(),
12873 generation: 1,
12874 mode: crate::recovery::Mode::VerifyRepair,
12875 reasons: Vec::new(),
12876 reviewed_head: None,
12877 rounds: Vec::new(),
12878 open_findings: Vec::new(),
12879 evidence: None,
12880 criteria_inherited: true,
12881 repair: crate::recovery::Repair::NotStarted,
12882 });
12883 runner.recovery_repair(&winner).await.unwrap();
12884 assert!(matches!(
12885 runner.state.recovery.as_ref().unwrap().repair,
12886 crate::recovery::Repair::WaitingForRestore { .. }
12887 ));
12888 drop(held);
12889 runner.state.resource_block = None;
12890 runner.state.config.graph.timeout_resource_wait = 1;
12891 runner.recovery_repair(&winner).await.unwrap();
12892 assert!(matches!(
12893 runner.state.recovery.as_ref().unwrap().repair,
12894 crate::recovery::Repair::Done { .. }
12895 ));
12896 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12897 }
12898
12899 #[tokio::test]
12900 #[cfg(unix)]
12901 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12902 use std::os::fd::AsRawFd as _;
12903 for graft in [false, true] {
12904 let tmp = tempfile::tempdir().unwrap();
12905 let mut runner = registered_runner(tmp.path()).await;
12906 runner.state.config.graph.timeout_resource_wait = 1;
12907 let winner = runner.state.candidates[0].clone();
12908 let marker = tmp.path().join("fixer-started");
12909 let mut fixer = spec("fixer");
12910 fixer.command = vec![
12911 "sh".into(),
12912 "-c".into(),
12913 format!("echo started >> '{}'; echo done", marker.display()),
12914 ];
12915 runner.roles.fixer = Some(vec![fixer]);
12916 if graft {
12917 let mut loser = winner.clone();
12918 loser.label = 'B';
12919 loser.index = 1;
12920 runner.state.candidates.push(loser);
12921 runner.state.config.graph.graft_runner_up = true;
12922 runner.state.tally = Some(Tally {
12923 first_choice: BTreeMap::new(),
12924 borda: BTreeMap::new(),
12925 winner: winner.label,
12926 rankings: 1,
12927 unanimous_initial: true,
12928 deliberated: false,
12929 changed_votes: 0,
12930 unanimous_final: true,
12931 tie_break: None,
12932 judges: 1,
12933 present: 1,
12934 quorum: 1,
12935 met_quorum: true,
12936 uncontested: None,
12937 });
12938 runner.state.judgements.push(crate::run::Judgement {
12939 judge: 1,
12940 seat: "judge-1".into(),
12941 agent: "judge".into(),
12942 ranking: vec![winner.label, 'B'],
12943 reasons: BTreeMap::new(),
12944 confidence: None,
12945 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12946 order: vec![0, 1],
12947 failed: None,
12948 duration_ms: 0,
12949 });
12950 } else {
12951 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12952 from_run: "earlier".into(),
12953 generation: 1,
12954 mode: crate::recovery::Mode::VerifyRepair,
12955 reasons: Vec::new(),
12956 reviewed_head: None,
12957 rounds: Vec::new(),
12958 open_findings: Vec::new(),
12959 evidence: None,
12960 criteria_inherited: true,
12961 repair: crate::recovery::Repair::NotStarted,
12962 });
12963 }
12964 let reader = crate::repo_lock::SeatLease::for_checkout(
12965 &winner.worktree,
12966 false,
12967 crate::config::test_coordination().as_ref(),
12968 )
12969 .await
12970 .unwrap();
12971 let pause = runner.pause.clone();
12972 let (result, ()) = tokio::join!(
12973 async {
12974 if graft {
12975 runner.graft().await
12976 } else {
12977 runner.recovery_repair(&winner).await
12978 }
12979 },
12980 async {
12981 tokio::time::sleep(Duration::from_millis(100)).await;
12982 pause.park();
12983 }
12984 );
12985 result.unwrap();
12986 assert!(!marker.exists(), "a blocked seat must not spawn");
12987 runner.state = RunState::load(&runner.state.id).unwrap();
12989 if graft {
12990 assert_eq!(
12991 runner.state.graft.as_ref().unwrap().outcome,
12992 GraftOutcome::WaitingForRestore
12993 );
12994 } else {
12995 assert!(matches!(
12996 runner.state.recovery.as_ref().unwrap().repair,
12997 crate::recovery::Repair::WaitingForRestore { .. }
12998 ));
12999 }
13000 drop(reader);
13001 runner.pause = Pause::new();
13002 runner.state.resource_block = None;
13003 let external =
13005 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13006 assert_eq!(
13007 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13008 0
13009 );
13010 let leftover = winner.worktree.join("leftover");
13011 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13012 let result = if graft {
13013 runner.graft().await
13014 } else {
13015 runner.recovery_repair(&winner).await
13016 };
13017 assert!(
13018 result
13019 .unwrap_err()
13020 .is::<crate::repo_lock::ResourceBlocked>()
13021 );
13022 assert!(leftover.exists());
13023 assert!(!marker.exists());
13024 drop(external);
13025 runner.state.resource_block = None;
13026 let result = if graft {
13027 runner.graft().await
13028 } else {
13029 runner.recovery_repair(&winner).await
13030 };
13031 result.unwrap();
13032 assert!(!leftover.exists(), "restore must precede retry");
13033 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13034 if graft {
13035 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13036 } else {
13037 assert!(matches!(
13038 runner.state.recovery.as_ref().unwrap().repair,
13039 crate::recovery::Repair::Done { .. }
13040 ));
13041 }
13042 }
13043 }
13044
13045 #[tokio::test]
13046 #[cfg(unix)]
13047 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13048 use std::os::fd::AsRawFd as _;
13049 let tmp = tempfile::tempdir().unwrap();
13050 let mut runner = registered_runner(tmp.path()).await;
13051 runner.state.config.graph.timeout_resource_wait = 1;
13052 let winner = runner.state.candidates[0].clone();
13053 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13054 runner.state.tally = Some(Tally {
13055 first_choice: BTreeMap::new(),
13056 borda: BTreeMap::new(),
13057 winner: winner.label,
13058 rankings: 0,
13059 unanimous_initial: false,
13060 deliberated: false,
13061 changed_votes: 0,
13062 unanimous_final: false,
13063 tie_break: None,
13064 judges: 0,
13065 present: 0,
13066 quorum: 0,
13067 met_quorum: true,
13068 uncontested: Some("fixture".into()),
13069 });
13070 runner.state.graft = Some(GraftRecord {
13071 pre: pre.clone(),
13072 post: None,
13073 lines: None,
13074 limit: None,
13075 agent: None,
13076 outcome: GraftOutcome::Running,
13077 requested: Vec::new(),
13078 adoption: None,
13079 adoption_error: None,
13080 });
13081 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13082 git::commit_all(&winner.worktree, "unfinished graft")
13083 .await
13084 .unwrap();
13085 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13086 let leftover = winner.worktree.join("leftover");
13087 std::fs::write(&leftover, "untrusted edits").unwrap();
13088 let path = runner.state.repo.join(".git/updater.lock");
13089 let external = std::fs::File::create(&path).unwrap();
13090 assert_eq!(
13091 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13092 0
13093 );
13094 assert!(
13095 runner
13096 .graft()
13097 .await
13098 .unwrap_err()
13099 .is::<crate::repo_lock::ResourceBlocked>()
13100 );
13101 assert!(matches!(
13102 runner.state.graft.as_ref().unwrap().outcome,
13103 GraftOutcome::Running
13104 ));
13105 assert_eq!(
13106 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13107 unfinished
13108 );
13109 assert!(leftover.exists());
13110 drop(external);
13111 std::fs::remove_file(&path).unwrap();
13112 std::fs::create_dir(&path).unwrap();
13113 assert!(runner.graft().await.is_err());
13114 assert!(matches!(
13115 runner.state.graft.as_ref().unwrap().outcome,
13116 GraftOutcome::Running
13117 ));
13118 assert_eq!(
13119 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13120 unfinished
13121 );
13122 assert!(leftover.exists());
13123 std::fs::remove_dir(&path).unwrap();
13124 runner.graft().await.unwrap();
13125 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13126 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13127 assert!(!leftover.exists());
13128 }
13129
13130 #[tokio::test]
13131 #[cfg(unix)]
13132 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13133 use std::os::fd::AsRawFd as _;
13134 let tmp = tempfile::tempdir().unwrap();
13135 let mut runner = registered_runner(tmp.path()).await;
13136 runner.state.prep_complete = Some(false);
13137 runner.state.config.graph.timeout_resource_wait = 1;
13138 runner.state.save().unwrap();
13139 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13140 assert_eq!(
13141 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13142 0
13143 );
13144 runner.execute().await.unwrap();
13145 let disk = RunState::load(&runner.state.id).unwrap();
13146 assert!(disk.driver_exited);
13147 assert_eq!(disk.driver_pid, None);
13148 assert_eq!(runner.state.status, RunStatus::Blocked);
13149 assert_eq!(
13150 runner.state.resource_block.as_ref().unwrap().kind,
13151 "repository coordination"
13152 );
13153 assert!(runner.state.prep_failure.is_none());
13154 assert!(runner.state.seats.is_empty());
13155 runner.pause.park();
13156 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13159 .await
13160 .unwrap()
13161 .unwrap();
13162 assert!(runner.state.parked);
13163 assert!(runner.state.prep_failure.is_none());
13164 let disk = RunState::load(&runner.state.id).unwrap();
13165 assert!(disk.driver_exited);
13166 assert_eq!(disk.driver_pid, None);
13167 }
13168
13169 #[tokio::test]
13170 #[cfg(unix)]
13171 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13172 use std::os::fd::AsRawFd as _;
13173 let tmp = tempfile::tempdir().unwrap();
13174 let mut runner = registered_runner(tmp.path()).await;
13175 runner.state.prep_complete = Some(false);
13176 runner.state.config.graph.timeout_resource_wait = 30;
13177 runner.state.save().unwrap();
13178 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13179 assert_eq!(
13180 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13181 0
13182 );
13183 assert!(
13184 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13185 .await
13186 .is_err()
13187 );
13188 let disk = RunState::load(&runner.state.id).unwrap();
13189 assert!(disk.driver_exited);
13190 assert_eq!(disk.driver_pid, None);
13191 assert!(disk.status.resumable());
13192 assert!(runner.state.driver_exited);
13193 assert_eq!(runner.state.driver_pid, None);
13194 runner.state.save().unwrap();
13195 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13196 }
13197
13198 #[tokio::test]
13199 #[cfg(unix)]
13200 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13201 let tmp = tempfile::tempdir().unwrap();
13202 let mut runner = registered_runner(tmp.path()).await;
13203 let winner = runner.state.candidates[0].clone();
13204 let cache = tmp.path().join("cache");
13205 let started = tmp.path().join("pre-gate-started");
13206 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13207 runner.state.config.verify.gate =
13208 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13209 let home = crate::run::home();
13210 let owner =
13211 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13212 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13213 crate::cache::AcquireOutcome::Acquired(g) => g,
13214 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13215 };
13216 let mut gate = Box::pin(runner.run_gate(&winner));
13217 assert!(
13218 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13219 .await
13220 .is_err()
13221 );
13222 assert!(!started.exists(), "pre-gate must wait for the cache too");
13223 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13227 let editor = crate::repo_lock::SeatLease::checkout_with(
13228 &winner.worktree,
13229 true,
13230 crate::config::test_coordination().as_ref(),
13231 &wait,
13232 )
13233 .await
13234 .unwrap();
13235 drop(editor);
13236 guard.release();
13237 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13238 .await
13239 .unwrap()
13240 .unwrap();
13241 drop(gate);
13242 assert!(outcomes.iter().all(CommandOutcome::ok));
13243 assert!(started.exists());
13244 }
13245
13246 #[tokio::test]
13247 #[cfg(unix)]
13248 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13249 use std::os::fd::AsRawFd as _;
13250
13251 ask_test_home();
13252 let tmp = tempfile::tempdir().unwrap();
13253 let repo = tmp.path().join("repo");
13254 std::fs::create_dir_all(&repo).unwrap();
13255 init_repo(&repo);
13256 let common = repo.join(".git");
13257 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13258 let mut runner = runner_at(RunStatus::Prep);
13259 runner.state.repo = repo.clone();
13260 runner.state.config.coordination = crate::config::test_coordination();
13261 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13262 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13263 runner.state.config.graph.advise = false;
13264 runner.roles.implementers = vec![spec("implement")];
13265 runner.roles.judges = vec![spec("judge")];
13266 runner.prep().await.unwrap();
13267 let winner = runner.state.candidates[0].clone();
13268 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13271 let completed = tmp.path().join("seat-completed");
13272 runner.roles.implementers[0].command = vec![
13273 "sh".into(),
13274 "-c".into(),
13275 format!("touch '{}'; echo completed", completed.display()),
13276 ];
13277 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13278 assert_eq!(
13279 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13280 0
13281 );
13282 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13283 let mut implement = Box::pin(runner.implement());
13284 tokio::select! {
13285 result = &mut implement => panic!("rescue must wait: {result:?}"),
13286 result = tokio::time::timeout(Duration::from_secs(10), async {
13287 while !completed.exists() {
13288 tokio::time::sleep(Duration::from_millis(20)).await;
13289 }
13290 }) => result.unwrap(),
13291 }
13292 assert!(
13293 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13294 .await
13295 .is_err()
13296 );
13297 assert_eq!(
13298 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13299 before
13300 );
13301 assert_eq!(
13302 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13303 0
13304 );
13305 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13306 .await
13307 .unwrap()
13308 .unwrap();
13309 drop(implement);
13310 assert_ne!(
13311 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13312 before
13313 );
13314 assert!(git::is_clean(&winner.worktree).await.unwrap());
13315
13316 let started = tmp.path().join("pre-gate-started");
13317 let proceed = tmp.path().join("pre-gate-proceed");
13318 let checked = tmp.path().join("gate-ran");
13319 runner.state.config.verify.pre_gate = vec![format!(
13320 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13321 started.display(),
13322 proceed.display()
13323 )];
13324 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13325 assert_eq!(
13326 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13327 0
13328 );
13329 let mut gate = Box::pin(runner.run_gate(&winner));
13330 assert!(
13331 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13332 .await
13333 .is_err()
13334 );
13335 assert!(!started.exists());
13336 assert!(!checked.exists());
13337 assert!(git::is_clean(&winner.worktree).await.unwrap());
13338 assert_eq!(
13339 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13340 0
13341 );
13342 tokio::select! {
13343 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13344 result = tokio::time::timeout(Duration::from_secs(10), async {
13345 while !started.exists() {
13346 tokio::time::sleep(Duration::from_millis(20)).await;
13347 }
13348 }) => result.unwrap(),
13349 }
13350 assert_eq!(
13351 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13352 -1,
13353 "mechanical edits must hold the real lock"
13354 );
13355 std::fs::write(proceed, "ready").unwrap();
13356 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13357 .await
13358 .unwrap()
13359 .unwrap();
13360 drop(gate);
13361 assert!(outcomes.iter().all(CommandOutcome::ok));
13362 assert!(checked.exists());
13363 assert!(runner.state.pre_gate_commit.is_some());
13364 assert!(git::is_clean(&winner.worktree).await.unwrap());
13365 assert_eq!(
13366 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13367 0,
13368 "gate completion must release coordination"
13369 );
13370 drop(outsider);
13371 std::fs::remove_file(common.join("updater.lock")).unwrap();
13372 std::fs::create_dir(common.join("updater.lock")).unwrap();
13373 std::fs::remove_file(&started).unwrap();
13374 std::fs::remove_file(&checked).unwrap();
13375 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13376 let error = runner.run_gate(&winner).await.unwrap_err();
13377 assert!(
13378 error
13379 .to_string()
13380 .contains("acquire mandatory repository lock")
13381 );
13382 assert!(
13383 !started.exists(),
13384 "failed acquisition must prevent pre-gate"
13385 );
13386 assert!(!checked.exists(), "failed acquisition must prevent gate");
13387 assert!(git::is_clean(&winner.worktree).await.unwrap());
13388 assert_eq!(
13389 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13390 head
13391 );
13392 }
13393
13394 #[tokio::test]
13395 #[cfg(unix)]
13396 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13397 use std::os::fd::AsRawFd as _;
13398 use std::time::Duration;
13399
13400 ask_test_home();
13401 let tmp = tempfile::tempdir().unwrap();
13402 let repo = tmp.path().join("repo");
13403 std::fs::create_dir_all(&repo).unwrap();
13404 init_repo(&repo);
13405 let common = repo.join(".git");
13406 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13407 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13408 assert_eq!(
13409 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13410 0
13411 );
13412 let mut runner = runner_at(RunStatus::Prep);
13413 runner.state.repo = repo.clone();
13414 runner.state.config.coordination = crate::config::test_coordination();
13415 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13416 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13417 runner.state.config.graph.advise = true;
13418 runner.state.config.graph.advisors = 1;
13419 runner.roles.implementers = vec![spec("implement")];
13420 runner.roles.judges = vec![spec("judge")];
13421 let root = runner.state.worktree_root();
13422 let started = tmp.path().join("setup-started");
13423 let proceed = tmp.path().join("setup-proceed");
13424 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13425 run: Some(format!(
13426 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13427 started.display(),
13428 proceed.display()
13429 )),
13430 ..Default::default()
13431 }];
13432 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13433 let config = git::git(&repo, &["config", "--local", "--list"])
13434 .await
13435 .unwrap();
13436 let mut prep = Box::pin(runner.prep());
13437 assert!(
13438 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13439 .await
13440 .is_err()
13441 );
13442 assert!(
13443 !root.exists(),
13444 "contention must prevent even checkout creation"
13445 );
13446 assert!(!started.exists(), "contention must prevent setup");
13447 assert_eq!(
13448 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13449 branches
13450 );
13451 assert_eq!(
13452 git::git(&repo, &["config", "--local", "--list"])
13453 .await
13454 .unwrap(),
13455 config
13456 );
13457 drop(lock);
13458 tokio::select! {
13459 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13460 result = tokio::time::timeout(Duration::from_secs(10), async {
13461 while !started.exists() {
13462 tokio::time::sleep(Duration::from_millis(20)).await;
13463 }
13464 }) => result.unwrap(),
13465 }
13466 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13467 assert_eq!(
13468 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13469 -1
13470 );
13471 assert!(
13472 tokio::time::timeout(
13473 Duration::from_millis(100),
13474 crate::repo_lock::SeatLease::for_checkout(
13475 &repo,
13476 true,
13477 crate::config::test_coordination().as_ref()
13478 )
13479 )
13480 .await
13481 .is_err(),
13482 "preparation must also exclude local editors"
13483 );
13484 std::fs::write(proceed, "ready").unwrap();
13485 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13486 .await
13487 .unwrap()
13488 .unwrap();
13489 drop(prep);
13490 assert_eq!(runner.state.prep_complete, Some(true));
13491 assert!(root.join("judge-1").exists());
13492 assert!(root.join("advisor-1").exists());
13493 assert_eq!(
13494 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13495 0
13496 );
13497 }
13498
13499 #[tokio::test]
13500 #[cfg(unix)]
13501 async fn preparation_lock_failure_prevents_repository_changes() {
13502 ask_test_home();
13503 let tmp = tempfile::tempdir().unwrap();
13504 let repo = tmp.path().join("repo");
13505 std::fs::create_dir_all(&repo).unwrap();
13506 init_repo(&repo);
13507 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13508 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13511 let mut runner = runner_at(RunStatus::Prep);
13512 runner.state.repo = repo.clone();
13513 runner.state.config.coordination = crate::config::test_coordination();
13514 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13515 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13516 runner.roles.implementers = vec![spec("implement")];
13517 let error = runner.prep().await.unwrap_err();
13518 assert!(
13519 error
13520 .to_string()
13521 .contains("acquire mandatory repository lock")
13522 );
13523 assert!(!runner.state.worktree_root().exists());
13524 assert!(runner.state.candidates.is_empty());
13525 assert!(!repo.join(".git/config.worktree").exists());
13526 assert_eq!(
13527 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13528 "* main"
13529 );
13530 }
13531
13532 #[tokio::test]
13533 async fn preparation_records_creation_intent_before_git_can_fail() {
13534 ask_test_home();
13535 let tmp = tempfile::tempdir().unwrap();
13536 let repo = tmp.path().join("repo");
13537 std::fs::create_dir_all(&repo).unwrap();
13538 init_repo(&repo);
13539 let mut runner = runner_at(RunStatus::Prep);
13540 runner.state.repo = repo.clone();
13541 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13542 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13543 runner.state.config.blind.commit_msg_hook = false;
13544 runner.roles.implementers = vec![spec("one")];
13545 let label = blind::assign_labels(1, runner.state.seed)[0];
13546 let branch = runner.state.branch_for(label);
13547 git::git(&repo, &["branch", &branch]).await.unwrap();
13550 assert!(runner.prep().await.is_err());
13551 let saved = RunState::load(&runner.state.id).unwrap();
13552 assert!(saved.candidates.is_empty());
13553 let event = saved
13554 .events
13555 .iter()
13556 .find(|e| e.node == "prep-intent")
13557 .unwrap();
13558 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13559 assert_eq!(intent["branch"], branch);
13560 assert_eq!(intent["start"], runner.state.base_commit);
13561 assert_eq!(
13562 intent["worktree"],
13563 runner
13564 .state
13565 .worktree_root()
13566 .join(format!("cand-{label}"))
13567 .to_string_lossy()
13568 .as_ref()
13569 );
13570 assert_eq!(intent["picks"], serde_json::json!([]));
13571 }
13572
13573 #[tokio::test]
13574 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13575 ask_test_home();
13576 let tmp = tempfile::tempdir().unwrap();
13577 let repo = tmp.path().join("repo");
13578 std::fs::create_dir_all(&repo).unwrap();
13579 init_repo(&repo);
13580 let mut runner = runner_at(RunStatus::Prep);
13581 runner.state.repo = repo.clone();
13582 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13583 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13584 runner.state.sources.pick_commits.push(refs::Input {
13585 token: "missing".into(),
13586 sha: "0".repeat(40),
13587 provenance: refs::Provenance::ExplicitCli,
13588 });
13589 assert!(runner.prep().await.is_err());
13590 assert!(runner.state.candidates.is_empty());
13591 assert!(!runner.state.worktree_root().exists());
13592 }
13593
13594 #[tokio::test]
13595 async fn prose_and_legacy_inference_never_import_into_candidates() {
13596 ask_test_home();
13597 let tmp = tempfile::tempdir().unwrap();
13598 let repo = tmp.path().join("repo");
13599 std::fs::create_dir_all(&repo).unwrap();
13600 init_repo(&repo);
13601 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13602 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13603 .await
13604 .unwrap();
13605 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13606 git::git(&repo, &["add", "."]).await.unwrap();
13607 git::git(&repo, &["commit", "-m", "historical implementation"])
13608 .await
13609 .unwrap();
13610 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13611 git::git(&repo, &["checkout", "main"]).await.unwrap();
13612 for instruction in [
13613 format!("Example: magi/history/A at {source}"),
13614 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13615 format!("Never import magi/history/A or {source}"),
13616 format!("Conversation answer: I mentioned {source} only as background"),
13617 ] {
13618 let mut runner = runner_at(RunStatus::Prep);
13619 runner.state.repo = repo.clone();
13620 runner.state.base_commit = base.clone();
13621 runner.state.instruction = instruction.clone();
13622 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13623 runner.state.config.blind.commit_msg_hook = false;
13624 runner.state.seeds = vec![refs::Seed {
13625 token: "magi/history/A".into(),
13626 sha: source.clone(),
13627 kind: refs::SeedKind::Unmerged,
13628 branch: true,
13629 detail: String::new(),
13630 }];
13631 runner.roles.implementers = vec![spec("one")];
13632 assert_eq!(seeded_instruction(&runner.state), instruction);
13633 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13635 legacy.as_object_mut().unwrap().remove("sources");
13636 runner.state = serde_json::from_value(legacy).unwrap();
13637 assert_eq!(runner.state.sources, refs::Sources::default());
13638 runner.state.save().unwrap();
13639 runner.state = RunState::load(&runner.state.id).unwrap();
13640 runner.prep().await.unwrap();
13641 for candidate in &runner.state.candidates {
13642 assert_eq!(
13643 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13644 base
13645 );
13646 assert!(!candidate.worktree.join("unwanted").exists());
13647 }
13648 }
13649 }
13650
13651 #[tokio::test]
13652 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13653 ask_test_home();
13654 let tmp = tempfile::tempdir().unwrap();
13655 let repo = tmp.path().join("repo");
13656 std::fs::create_dir_all(&repo).unwrap();
13657 init_repo(&repo);
13658 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13659 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13660 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13661 git::git(&repo, &["add", "."]).await.unwrap();
13662 git::git(&repo, &["commit", "-m", "required seed"])
13663 .await
13664 .unwrap();
13665 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13666 git::git(&repo, &["checkout", "main"]).await.unwrap();
13667 let mut runner = runner_at(RunStatus::Prep);
13668 runner.state.repo = repo.clone();
13669 runner.state.base_commit = base.clone();
13670 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13671 runner.state.config.graph.advise = false;
13672 runner.state.config.blind.commit_msg_hook = false;
13673 runner.state.sources =
13674 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13675 .await
13676 .unwrap();
13677 runner.roles.implementers = vec![spec("one"), spec("two")];
13678 runner.roles.judges = vec![spec("judge")];
13679 let label = blind::assign_labels(2, runner.state.seed)[0];
13680 let branch = runner.state.branch_for(label);
13681 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13682 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13683 .await
13684 .unwrap();
13685 runner.state.candidates.push(Candidate {
13686 index: 0,
13687 label,
13688 agent: "one".into(),
13689 branch,
13690 worktree,
13691 summary: String::new(),
13692 stat: String::new(),
13693 files: 0,
13694 commits: 0,
13695 empty: false,
13696 failed: None,
13697 verified_noop: None,
13698 duration_ms: 0,
13699 folded: false,
13700 });
13701 runner.state.save().unwrap();
13702 runner.state = RunState::load(&runner.state.id).unwrap();
13704 assert_eq!(runner.state.prep_complete, Some(false));
13705 runner.prep().await.unwrap();
13706 assert_eq!(runner.state.prep_complete, Some(true));
13707 assert_eq!(runner.state.candidates.len(), 2);
13708 for candidate in &runner.state.candidates {
13709 assert_eq!(
13710 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13711 git::tree_of(&repo, &seed).await.unwrap()
13712 );
13713 assert_eq!(
13714 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13715 "required commit"
13716 );
13717 }
13718 assert!(runner.state.worktree_root().join("judge-1").exists());
13719 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13720 runner.prep().await.unwrap();
13721 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13722
13723 runner.state.prep_complete = Some(false);
13724 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13725 run: Some(format!(
13726 "test -f '{}'",
13727 tmp.path().join("dependency-ready").display()
13728 )),
13729 ..Default::default()
13730 }];
13731 assert!(runner.prep().await.is_err());
13732 assert!(runner.state.prep_failure.is_some());
13733 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13734 runner.state = RunState::load(&runner.state.id).unwrap();
13735 assert!(
13736 runner.prep().await.is_err(),
13737 "correcting the cause alone must not auto-retry"
13738 );
13739 let failed_at = runner.state.events.last().unwrap().at;
13740 assert!(
13741 runner.retry_preparation(failed_at).is_err(),
13742 "old approvals cannot unlock the hold"
13743 );
13744 let mut task = crate::queue::Task::new(
13745 "retry".into(),
13746 "retry setup".into(),
13747 repo.clone(),
13748 crate::queue::Source::Human,
13749 );
13750 task.start(runner.state.id.clone());
13751 task.hold_machine(Some("setup failed".into()));
13752 task.release_operator("cli", None);
13753 runner
13754 .retry_preparation(task.resumption.as_ref().unwrap().at)
13755 .unwrap();
13756 runner.prep().await.unwrap();
13757 assert!(runner.state.prep_failure.is_none());
13758 assert_eq!(runner.state.prep_complete, Some(true));
13759 assert_eq!(runner.state.candidates.len(), 2);
13760 assert!(runner.state.worktree_root().join("judge-1").exists());
13761 for candidate in &runner.state.candidates {
13762 assert_eq!(
13763 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13764 git::tree_of(&repo, &seed).await.unwrap()
13765 );
13766 }
13767 }
13768
13769 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13770 let mut heads = Vec::new();
13771 for candidate in candidates {
13772 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13773 }
13774 heads
13775 }
13776
13777 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13778 ask_test_home();
13779 let tmp = tempfile::tempdir().unwrap();
13780 let repo = tmp.path().join("repo");
13781 std::fs::create_dir(&repo).unwrap();
13782 init_repo(&repo);
13783 let mut runner = runner_at(RunStatus::Reviewing);
13784 runner.state.repo = repo.clone();
13785 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13786 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13787 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13788 let branch = format!("candidate-{label}");
13789 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13790 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13791 .await
13792 .unwrap();
13793 if label == 'B' {
13794 std::fs::write(worktree.join("change"), "winner").unwrap();
13795 git::rescue_commit(&worktree, "candidate change")
13796 .await
13797 .unwrap();
13798 }
13799 runner.state.candidates.push(Candidate {
13800 index,
13801 label,
13802 agent: "alpha".into(),
13803 branch,
13804 worktree,
13805 summary: "completed reply".into(),
13806 stat: String::new(),
13807 files: usize::from(label == 'B'),
13808 commits: usize::from(label == 'B'),
13809 empty: label == 'A',
13810 failed: None,
13811 verified_noop: None,
13812 duration_ms: 1,
13813 folded: false,
13814 });
13815 }
13816 runner.state.judge_skipped = true;
13817 runner.state.tally = Some(Tally {
13818 first_choice: BTreeMap::new(),
13819 borda: BTreeMap::new(),
13820 winner: 'B',
13821 rankings: 0,
13822 unanimous_initial: false,
13823 deliberated: false,
13824 changed_votes: 0,
13825 unanimous_final: false,
13826 tie_break: None,
13827 judges: 0,
13828 present: 0,
13829 quorum: 0,
13830 met_quorum: true,
13831 uncontested: Some("only B produced a change".into()),
13832 });
13833 runner.state.save().unwrap();
13834 (tmp, runner)
13835 }
13836
13837 #[tokio::test]
13838 #[cfg(unix)]
13839 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13840 for (solo, failed_checkpoint) in
13841 [(true, false), (false, false), (true, true), (false, true)]
13842 {
13843 let (_tmp, mut runner) = preservation_resume_fixture().await;
13844 if solo {
13845 runner.state.candidates.retain(|c| c.label == 'A');
13846 }
13847 runner.state.candidates[0].empty = false;
13848 runner.state.candidates[0].summary.clear();
13849 runner.state.tally = None;
13850 runner.state.judge_skipped = false;
13851 runner.state.status = RunStatus::Implementing;
13852 runner.state.config.disk.min_free_bytes = u64::MAX;
13853 let mut implementer = spec("alpha");
13854 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13855 runner.roles.implementers = vec![implementer];
13856 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13857 if failed_checkpoint {
13858 std::fs::create_dir_all(&patch).unwrap();
13859 }
13860 runner.state.save().unwrap();
13861 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13862 runner.state = RunState::load(&runner.state.id).unwrap();
13863 assert_eq!(runner.state.status, RunStatus::Blocked);
13864 assert!(runner.state.interrupted_implementations.contains(&'A'));
13865 assert_eq!(runner.state.candidates[0].commits, 0);
13866 assert!(!runner.state.candidates[0].empty);
13867 assert!(runner.state.candidates[0].failed.is_none());
13868 if failed_checkpoint {
13869 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13870 std::fs::remove_dir(patch).unwrap();
13871 }
13872 runner.state.resource_block = None;
13873 runner.state.config.disk.min_free_bytes = 0;
13874 runner.roles.implementers[0].command = vec![
13875 "sh".into(),
13876 "-c".into(),
13877 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13878 ];
13879 runner.implement().await.unwrap();
13880 assert!(runner.state.interrupted_implementations.is_empty());
13881 assert!(runner.state.candidate_preservation.is_empty());
13882 assert_eq!(runner.state.candidates[0].commits, 1);
13883 assert!(!runner.state.candidates[0].empty);
13884 assert!(
13885 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13886 .unwrap()
13887 .contains("retried empty interrupted seat")
13888 );
13889 if !solo {
13890 assert_eq!(runner.state.candidates[1].commits, 1);
13891 }
13892 }
13893 }
13894
13895 #[tokio::test]
13896 #[cfg(unix)]
13897 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13898 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13899 let (_tmp, mut runner) = preservation_resume_fixture().await;
13900 runner.state.candidates.retain(|c| c.label == 'A');
13901 runner.state.candidates[0].empty = false;
13902 runner.state.candidates[0].summary.clear();
13903 runner.state.tally = None;
13904 runner.state.judge_skipped = false;
13905 runner.state.status = RunStatus::Implementing;
13906 runner.state.config.disk.min_free_bytes = u64::MAX;
13909 let wt = runner.state.candidates[0].worktree.clone();
13910 std::fs::write(
13911 wt.join("interrupted-source"),
13912 "retain implementation work\n",
13913 )
13914 .unwrap();
13915 if committed {
13916 git::rescue_commit(&wt, "implementation checkpoint")
13917 .await
13918 .unwrap();
13919 }
13920 let lock = git::git(
13921 &wt,
13922 &[
13923 "rev-parse",
13924 "--path-format=absolute",
13925 "--git-path",
13926 "index.lock",
13927 ],
13928 )
13929 .await
13930 .unwrap();
13931 if block_rescue {
13932 std::fs::write(&lock, "fixture-owned lock").unwrap();
13933 }
13934 let mut implementer = spec("alpha");
13935 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13936 runner.roles.implementers = vec![implementer];
13937 runner.state.save().unwrap();
13938 let result = runner.implement().await;
13939 assert_eq!(result.is_err(), block_rescue);
13940 assert_eq!(
13941 runner.state.resource_block.as_ref().unwrap().kind,
13942 "disk pressure"
13943 );
13944 assert!(
13945 runner
13946 .state
13947 .events
13948 .iter()
13949 .any(|e| e.message.contains("agent execution interrupted"))
13950 );
13951 assert!(
13952 runner
13953 .state
13954 .dir()
13955 .join("artifacts/candidate-A-recovery/staged.patch")
13956 .exists()
13957 );
13958 if block_rescue {
13959 let saved = RunState::load(&runner.state.id).unwrap();
13960 assert!(saved.candidate_preservation.contains_key(&'A'));
13961 assert!(!saved.candidates[0].empty);
13962 assert_eq!(
13963 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13964 "retain implementation work\n"
13965 );
13966 std::fs::remove_file(lock).unwrap();
13967 runner.state = saved;
13968 } else {
13969 assert_eq!(runner.state.status, RunStatus::Blocked);
13970 assert_eq!(runner.state.candidates[0].commits, 1);
13971 }
13972 runner.roles.implementers.clear();
13975 runner.state.config.disk.min_free_bytes = 0;
13976 runner.state.resource_block = None;
13977 runner.implement().await.unwrap();
13978 assert!(runner.state.candidate_preservation.is_empty());
13979 assert_eq!(runner.state.candidates[0].commits, 1);
13980 assert!(!runner.state.candidates[0].empty);
13981 assert!(git::is_clean(&wt).await.unwrap());
13982 assert!(
13983 crate::run::read_artifact(&runner.state, "cand-A.patch")
13984 .unwrap()
13985 .contains("retain implementation work")
13986 );
13987 }
13988 }
13989
13990 #[tokio::test]
13991 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13992 let (_tmp, mut runner) = preservation_resume_fixture().await;
13993 runner.implement().await.unwrap();
13995 assert!(runner.state.candidate_preservation.is_empty());
13996 assert!(
13997 !runner
13998 .state
13999 .dir()
14000 .join("artifacts/candidate-A-recovery")
14001 .exists()
14002 );
14003 runner.fold_losers().await.unwrap();
14004 assert!(!runner.state.candidates[0].worktree.exists());
14005 for schema in [14, crate::run::SCHEMA] {
14006 runner.state.schema = schema;
14007 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14008 runner.state.save().unwrap();
14009 runner.state = RunState::load(&runner.state.id).unwrap();
14010 runner.implement().await.unwrap();
14011 assert!(runner.state.candidate_preservation.is_empty());
14012 assert!(runner.state.candidates[0].folded);
14013 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14014 }
14015 }
14016
14017 #[tokio::test]
14018 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14019 let (_tmp, mut runner) = preservation_resume_fixture().await;
14020 let wt = runner.state.candidates[0].worktree.clone();
14021 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14022 runner.state.schema = 14;
14023 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14024 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14025 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14026 for _ in 0..2 {
14027 assert!(runner.implement().await.is_err());
14028 assert_eq!(runner.state.status, RunStatus::Blocked);
14029 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14030 assert!(
14031 runner.state.candidates[0].empty,
14032 "old verdict must remain unusable"
14033 );
14034 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14035 assert!(runner.state.judge_skipped);
14036 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14037 assert!(
14038 crate::run::read_artifact(&runner.state, "cand-A.patch")
14039 .unwrap()
14040 .contains("owned edits")
14041 );
14042 assert!(wt.exists());
14043 assert_eq!(
14044 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14045 .await
14046 .unwrap(),
14047 1
14048 );
14049 runner.state = RunState::load(&runner.state.id).unwrap();
14050 }
14051 }
14052
14053 #[tokio::test]
14054 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14055 let (_tmp, mut runner) = preservation_resume_fixture().await;
14056 let wt = runner.state.candidates[0].worktree.clone();
14057 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14058 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14059 let index_lock = PathBuf::from(
14060 git::git(
14061 &wt,
14062 &[
14063 "rev-parse",
14064 "--path-format=absolute",
14065 "--git-path",
14066 "index.lock",
14067 ],
14068 )
14069 .await
14070 .unwrap(),
14071 );
14072 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14073 runner.state.schema = 14;
14074 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14075 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14076 assert!(runner.implement().await.is_err());
14077 assert!(runner.after_implement().is_err());
14078 assert!(runner.fold_losers().await.is_err());
14079 assert_eq!(
14080 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14081 "failed seat edits"
14082 );
14083 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14084 runner.state = RunState::load(&runner.state.id).unwrap();
14085 std::fs::remove_file(index_lock).unwrap();
14086 assert!(runner.implement().await.is_err());
14088 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14089 assert!(runner.state.candidates[0].empty);
14090 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14091 assert!(
14092 crate::run::read_artifact(&runner.state, "cand-A.patch")
14093 .unwrap()
14094 .contains("failed seat edits")
14095 );
14096 assert!(wt.exists());
14097 }
14098
14099 #[tokio::test]
14100 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14101 let (_tmp, mut runner) = preservation_resume_fixture().await;
14102 let repo = runner.state.repo.clone();
14103 let common = git::git(
14104 &repo,
14105 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14106 )
14107 .await
14108 .unwrap();
14109 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14110 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14111 assert!(runner.fold_losers().await.is_err());
14112 assert!(!runner.state.candidates[0].worktree.exists());
14113 assert!(!runner.state.candidates[0].folded);
14114 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14115 runner.state = RunState::load(&runner.state.id).unwrap();
14116 std::fs::remove_file(branch_lock).unwrap();
14117 let judge = runner.state.worktree_root().join("judge-1");
14118 git::worktree_add_detached(&repo, &judge, "main")
14119 .await
14120 .unwrap();
14121 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14122 runner.roles.judges = vec![spec("judge")];
14123 assert!(runner.fold_losers().await.is_err());
14124 runner.state = RunState::load(&runner.state.id).unwrap();
14125 assert!(
14126 runner.state.candidates[0].folded,
14127 "candidate progress saved before judge failure"
14128 );
14129 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14130 assert!(judge.join("uncommitted").exists());
14131 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14132 runner.fold_losers().await.unwrap();
14133 assert!(!judge.exists());
14134 runner.fold_losers().await.unwrap();
14135 }
14136
14137 #[tokio::test]
14138 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14139 let (_tmp, mut runner) = preservation_resume_fixture().await;
14140 runner.state.candidates.retain(|c| c.label == 'B');
14141 runner.state.tally = None;
14142 runner.state.judge_skipped = false;
14143 runner.state.prep_complete = Some(true);
14144 runner.state.config.graph.advise = false;
14145 runner.state.config.graph.selected_implementer = Some("alpha".into());
14146 let wt = runner.state.candidates[0].worktree.clone();
14147 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14148 runner
14149 .state
14150 .candidate_preservation
14151 .insert('B', "awaiting persistence".into());
14152 let index_lock = git::git(
14153 &wt,
14154 &[
14155 "rev-parse",
14156 "--path-format=absolute",
14157 "--git-path",
14158 "index.lock",
14159 ],
14160 )
14161 .await
14162 .unwrap();
14163 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14164 runner.state.save().unwrap();
14165 assert!(runner.execute_graph().await.is_err());
14166 assert_eq!(runner.state.status, RunStatus::Blocked);
14167 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14168 std::fs::remove_file(index_lock).unwrap();
14169 for _ in 0..2 {
14172 runner.state = RunState::load(&runner.state.id).unwrap();
14173 runner.pause.park();
14174 runner.execute_graph().await.unwrap();
14175 assert_eq!(runner.state.status, RunStatus::Implementing);
14176 assert!(runner.state.candidate_preservation.is_empty());
14177 assert_eq!(runner.state.candidates[0].commits, 2);
14178 assert!(runner.state.tally.is_none());
14179 }
14180 runner.implement().await.unwrap();
14181 assert_eq!(runner.state.status, RunStatus::Judging);
14182 runner.state.status = RunStatus::Blocked;
14184 runner.state.candidates[0].failed = Some("implementation failed".into());
14185 runner.state.save().unwrap();
14186 let error = runner.execute_graph().await.unwrap_err();
14187 assert!(
14188 error
14189 .to_string()
14190 .contains("selected implementation is held")
14191 );
14192 }
14193
14194 #[tokio::test]
14195 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14196 let (_tmp, mut runner) = preservation_resume_fixture().await;
14197 let repo = runner.state.repo.clone();
14198 let candidate = runner.state.candidates[0].clone();
14199 let registered = git::worktree_holding(&repo, &candidate.branch)
14202 .await
14203 .unwrap()
14204 .unwrap();
14205 assert_eq!(
14206 registered.canonicalize().unwrap(),
14207 candidate.worktree.canonicalize().unwrap()
14208 );
14209 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14210 assert_eq!(
14211 git::worktree_holding(&repo, &candidate.branch)
14212 .await
14213 .unwrap(),
14214 Some(registered)
14215 );
14216 assert!(
14217 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14218 "the stale registration initially keeps the branch checked out"
14219 );
14220 runner.fold_losers().await.unwrap();
14221 assert!(runner.state.candidates[0].folded);
14222 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14223 assert_eq!(
14224 git::worktree_holding(&repo, &candidate.branch)
14225 .await
14226 .unwrap(),
14227 None
14228 );
14229 runner.state = RunState::load(&runner.state.id).unwrap();
14230 runner.fold_losers().await.unwrap();
14231 }
14232
14233 #[tokio::test]
14234 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14235 let (_tmp, mut runner) = preservation_resume_fixture().await;
14236 let repo = runner.state.repo.clone();
14237 let candidate = runner.state.candidates[0].clone();
14238 let winner = runner.state.candidates[1].clone();
14239 let registered = git::worktree_holding(&repo, &candidate.branch)
14240 .await
14241 .unwrap()
14242 .unwrap();
14243 assert_eq!(
14244 registered.canonicalize().unwrap(),
14245 candidate.worktree.canonicalize().unwrap()
14246 );
14247 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14248 assert_eq!(
14249 git::worktree_holding(&repo, &candidate.branch)
14250 .await
14251 .unwrap(),
14252 Some(registered)
14253 );
14254 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14255
14256 for _ in 0..2 {
14257 fold_run(&mut runner.state, false, &crate::run::home())
14258 .await
14259 .unwrap();
14260 runner.state = RunState::load(&runner.state.id).unwrap();
14261 assert!(runner.state.candidates[0].folded);
14262 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14263 assert_eq!(
14264 git::worktree_holding(&repo, &candidate.branch)
14265 .await
14266 .unwrap(),
14267 None
14268 );
14269 assert!(!runner.state.candidates[1].folded);
14270 assert!(winner.worktree.exists());
14271 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14272 }
14273 }
14274
14275 #[tokio::test]
14276 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14277 ask_test_home();
14278 let tmp = tempfile::tempdir().unwrap();
14279 let repo = tmp.path().join("repo");
14280 std::fs::create_dir(&repo).unwrap();
14281 init_repo(&repo);
14282 let mut runner = runner_at(RunStatus::Implementing);
14283 runner.state.repo = repo.clone();
14284 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14285 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14286 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14287 let branch = format!("candidate-{label}");
14288 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14289 git::worktree_add_branch(&repo, &wt, &branch, "main")
14290 .await
14291 .unwrap();
14292 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14293 git::git(&wt, &["add", "README.md"]).await.unwrap();
14294 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14295 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14296 runner.state.candidates.push(Candidate {
14297 index,
14298 label,
14299 agent: "alpha".to_owned(),
14300 branch,
14301 worktree: wt,
14302 summary: "completed implementation".to_owned(),
14303 stat: String::new(),
14304 files: 0,
14305 commits: 0,
14306 empty: false,
14307 failed: None,
14308 verified_noop: None,
14309 duration_ms: 42,
14310 folded: false,
14311 });
14312 runner
14313 .state
14314 .candidate_preservation
14315 .insert(label, "awaiting persistence".to_owned());
14316 }
14317 let wt = runner.state.candidates[0].worktree.clone();
14318 let index_lock = git::git(
14319 &wt,
14320 &[
14321 "rev-parse",
14322 "--path-format=absolute",
14323 "--git-path",
14324 "index.lock",
14325 ],
14326 )
14327 .await
14328 .unwrap();
14329 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14330 runner.state.save().unwrap();
14331 assert!(runner.persist_candidates().await.is_err());
14332 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14333 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14334 assert_eq!(
14335 runner.state.candidates[1].commits, 1,
14336 "other results are still collected"
14337 );
14338 assert!(
14339 !runner.state.candidates[0].empty,
14340 "failed rescue is not no change"
14341 );
14342 assert_eq!(
14343 runner.state.candidates[0].summary,
14344 "completed implementation"
14345 );
14346 assert!(runner.after_implement().is_err());
14347 let home = tmp.path().join("fold-home");
14348 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14349 assert!(wt.join("new-file").exists());
14350 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14351 assert!(recovery.join("staged.patch").exists());
14352 assert!(recovery.join("unstaged.patch").exists());
14353 assert_eq!(
14354 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14355 "untracked edits\n"
14356 );
14357 runner.state.begin_pass();
14358 runner.state.end_pass_parked(ParkCause::Upgrade {
14359 from: "1.0".into(),
14360 to: Some("1.1".into()),
14361 });
14362 runner.state.save().unwrap();
14363 runner.state = RunState::load(&runner.state.id).unwrap();
14364 assert!(matches!(
14365 runner.state.passes.last().unwrap().end,
14366 Some(crate::run::PassEnd::Parked {
14367 cause: ParkCause::Upgrade { .. }
14368 })
14369 ));
14370 std::fs::remove_file(index_lock).unwrap();
14371 runner.implement().await.unwrap();
14373 assert!(runner.state.candidate_preservation.is_empty());
14374 assert!(
14375 runner
14376 .state
14377 .candidates
14378 .iter()
14379 .all(|c| c.commits == 1 && !c.empty)
14380 );
14381 assert!(git::is_clean(&wt).await.unwrap());
14382 assert!(
14383 crate::run::read_artifact(&runner.state, "cand-A.patch")
14384 .unwrap()
14385 .contains("later tracked edits")
14386 );
14387 }
14388
14389 #[tokio::test]
14390 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14391 ask_test_home();
14392 let tmp = tempfile::tempdir().unwrap();
14393 let repo = tmp.path().join("repo");
14394 std::fs::create_dir(&repo).unwrap();
14395 init_repo(&repo);
14396 let wt = tmp.path().join("candidate");
14397 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14398 .await
14399 .unwrap();
14400 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14401 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14402 assert!(wt.join("uncommitted").exists());
14403 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14404 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14405 let mut runner = runner_at(RunStatus::Implementing);
14406 runner.state.repo = repo.clone();
14407 runner.state.base_commit = "missing-base".to_owned();
14408 runner.state.candidates.push(Candidate {
14409 index: 0,
14410 label: 'A',
14411 agent: "alpha".into(),
14412 branch: "candidate".into(),
14413 worktree: wt.clone(),
14414 summary: "completed".into(),
14415 stat: String::new(),
14416 files: 0,
14417 commits: 0,
14418 empty: false,
14419 failed: None,
14420 verified_noop: None,
14421 duration_ms: 1,
14422 folded: false,
14423 });
14424 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14425 assert!(
14426 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14427 .await
14428 .is_err()
14429 );
14430 assert!(!runner.state.candidates[0].folded);
14431 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14432 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14433 runner
14434 .state
14435 .candidate_preservation
14436 .insert('A', "pending".into());
14437 assert!(runner.persist_candidates().await.is_err());
14438 assert!(!runner.state.candidates[0].empty);
14439 assert!(!runner.state.candidates[0].folded);
14440 assert!(wt.exists());
14441 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14442 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14443 std::fs::create_dir_all(&patch_path).unwrap();
14444 assert!(
14445 runner.persist_candidates().await.is_err(),
14446 "artifact failure stays pending"
14447 );
14448 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14449 assert!(!runner.state.candidates[0].empty);
14450 std::fs::remove_dir(patch_path).unwrap();
14451 runner.persist_candidates().await.unwrap();
14452 assert!(
14453 runner.state.candidates[0].empty,
14454 "only successful inspection confirms no change"
14455 );
14456 }
14457
14458 fn init_repo(dir: &Path) {
14459 let run = |args: &[&str]| {
14460 let out = std::process::Command::new("git")
14461 .args(args)
14462 .current_dir(dir)
14463 .quiet()
14464 .output()
14465 .expect("spawn git");
14466 assert!(
14467 out.status.success(),
14468 "git {args:?} failed: {}",
14469 String::from_utf8_lossy(&out.stderr)
14470 );
14471 };
14472 run(&["init", "-b", "main"]);
14473 run(&["config", "user.name", "magi test"]);
14474 run(&["config", "user.email", "magi@example.com"]);
14475 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14476 run(&["add", "-A"]);
14477 run(&["commit", "-m", "init"]);
14478 }
14479
14480 fn ask_test_home() {
14488 crate::run::pin_test_home();
14489 }
14490
14491 fn runner_at(status: RunStatus) -> Runner {
14494 let mut state = RunState::new(
14495 PathBuf::from("/nonexistent/repo"),
14496 "main".to_owned(),
14497 "deadbeef".to_owned(),
14498 "task".to_owned(),
14499 Config::default(),
14500 );
14501 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14504 state.id = format!(
14505 "20000101-000000-{:04x}",
14506 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14507 );
14508 state.status = status;
14509 Runner {
14510 state,
14511 roles: ResolvedRoles {
14512 implementers: Vec::new(),
14513 judges: Vec::new(),
14514 reviewers: Vec::new(),
14515 fixer: None,
14516 conductor: conductor(),
14517 implementer_roster: Vec::new(),
14518 judge_roster: Vec::new(),
14519 reviewer_roster: Vec::new(),
14520 },
14521 sem: Arc::new(Semaphore::new(1)),
14522 pause: Pause::new(),
14523 interrupt: Pause::new(),
14524 }
14525 }
14526
14527 #[test]
14528 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14529 crate::run::pin_test_home();
14530 let mut runner = runner_at(RunStatus::Gating);
14531 runner.state.save().unwrap();
14532 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14535 assert!(runner.claim_driver().is_err());
14536 assert!(runner.state.driver_exited);
14537 assert_eq!(runner.state.driver_pid, None);
14538 let disk = RunState::load(&runner.state.id).unwrap();
14539 assert!(disk.driver_exited);
14540 assert_eq!(disk.driver_pid, None);
14541 assert_eq!(disk.status, RunStatus::Gating);
14542 }
14543
14544 #[tokio::test]
14545 async fn an_early_graph_error_releases_its_driver() {
14546 crate::run::pin_test_home();
14547 let mut runner = runner_at(RunStatus::Blocked);
14548 runner.state.config.graph.selected_implementer = Some("selected".into());
14549 runner.state.save().unwrap();
14550 assert!(runner.execute().await.is_err());
14551 let disk = RunState::load(&runner.state.id).unwrap();
14552 assert!(disk.driver_exited);
14553 assert_eq!(disk.driver_pid, None);
14554 assert_eq!(disk.status, RunStatus::Blocked);
14555 }
14556
14557 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14558 crate::run::MergeRepair {
14559 question: "repair-question".into(),
14560 head: "old-head".into(),
14561 pr: "https://github.com/o/r/pull/42".into(),
14562 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14563 gate: vec![outcome(Some(0), false)],
14564 gate_head: Some("old-head".into()),
14565 gate_fixes: Vec::new(),
14566 pre_gate: Vec::new(),
14567 phase,
14568 failure: None,
14569 }
14570 }
14571
14572 #[tokio::test]
14573 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14574 crate::run::pin_test_home();
14575 let mut runner = runner_at(RunStatus::Landing);
14576 runner.state.merge_repairs.push(merge_repair_request(
14577 crate::run::MergeRepairPhase::Requested,
14578 ));
14579 runner.state.save().unwrap();
14580 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14581 assert!(
14582 runner
14583 .execute()
14584 .await
14585 .unwrap_err()
14586 .to_string()
14587 .contains("already running")
14588 );
14589 assert_eq!(
14590 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14591 crate::run::MergeRepairPhase::Requested
14592 );
14593 drop(claim);
14594 let mut disk = runner.state.clone();
14595 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14596 disk.save().unwrap();
14597 assert!(
14598 runner
14599 .execute()
14600 .await
14601 .unwrap_err()
14602 .to_string()
14603 .contains("decision changed")
14604 );
14605 assert_eq!(
14606 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14607 crate::run::MergeRepairPhase::Complete
14608 );
14609 }
14610
14611 #[test]
14612 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14613 crate::run::pin_test_home();
14614 let mut runner = runner_at(RunStatus::Landing);
14615 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14616 runner.state.reviews = request.reviews.clone();
14617 runner.state.gate = request.gate.clone();
14618 runner.state.gate_ran = true;
14619 runner.state.land_approval = Some(crate::run::LandApproval {
14620 question: "old".into(),
14621 head: "old-head".into(),
14622 });
14623 runner.state.merge_repairs.push(request);
14624 runner.begin_merge_repair_review(0).unwrap();
14625 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14626 assert!(!runner.state.gate_status().ok());
14627 assert!(runner.state.land_approval.is_none());
14628 let disk = RunState::load(&runner.state.id).unwrap();
14629 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14630 assert!(disk.merge_repairs[0].gate[0].ok());
14631 assert_eq!(
14632 disk.merge_repairs[0].phase,
14633 crate::run::MergeRepairPhase::Reviewing
14634 );
14635 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14636 assert!(disk.reviews.is_empty());
14637 }
14638
14639 #[tokio::test]
14640 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14641 crate::run::pin_test_home();
14642 for phase in [
14643 crate::run::MergeRepairPhase::Fixing,
14644 crate::run::MergeRepairPhase::Requested,
14645 ] {
14646 let mut runner = runner_at(RunStatus::Landing);
14647 runner.state.config.graph.land_rounds = 0;
14648 runner.state.merge_repairs.push(merge_repair_request(phase));
14649 runner.state.save().unwrap();
14650 assert!(!runner.repair_merge_review().await.unwrap());
14651 assert_eq!(runner.state.status, RunStatus::Blocked);
14652 assert_eq!(
14653 runner.state.merge_repairs[0].phase,
14654 crate::run::MergeRepairPhase::Failed
14655 );
14656 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14657 assert!(
14658 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14659 "interrupted"
14660 } else {
14661 "budget exhausted"
14662 })
14663 );
14664 assert!(runner.state.merge.is_none());
14665 }
14666 }
14667
14668 #[test]
14672 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14673 crate::run::pin_test_home();
14674 let mut runner = runner_at(RunStatus::Implementing);
14675 let interrupt = Pause::new();
14676 runner.watch_interrupt(interrupt.clone());
14677
14678 interrupt.park_because("task a1b2 asked to run first");
14679
14680 assert!(runner.park_here().expect("park_here"));
14681 assert!(runner.state.parked);
14682 let last = runner.state.events.last().expect("a park event");
14683 assert_eq!(last.node, "park");
14684 assert!(
14685 last.message.contains("task a1b2 asked to run first"),
14686 "expected the interrupt reason in {:?}",
14687 last.message
14688 );
14689 }
14690
14691 #[test]
14694 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14695 crate::run::pin_test_home();
14696 let mut runner = runner_at(RunStatus::Implementing);
14697 runner.state.begin_pass();
14698 let shutdown = Pause::new();
14699 runner.on_pause(shutdown.clone());
14700 let upgrade = ParkCause::Upgrade {
14701 from: "0.121.0".to_owned(),
14702 to: Some("0.121.1".to_owned()),
14703 };
14704 shutdown.park_for(upgrade.clone());
14705 shutdown.park_for(ParkCause::Operator);
14706
14707 assert!(runner.park_here().expect("park_here"));
14708 assert_eq!(
14709 runner.state.passes[0].end,
14710 Some(crate::run::PassEnd::Parked { cause: upgrade })
14711 );
14712 let last = runner.state.events.last().expect("a park event");
14713 assert!(last.message.contains("0.121.0"), "{}", last.message);
14714 }
14715
14716 #[test]
14724 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14725 crate::run::pin_test_home();
14726 let mut runner = runner_at(RunStatus::Implementing);
14727 let shutdown = Pause::new();
14728 runner.on_pause(shutdown.clone());
14729 let interrupt = Pause::new();
14730 runner.watch_interrupt(interrupt.clone());
14731
14732 assert!(!runner.park_here().expect("park_here"));
14734 assert!(!runner.state.parked);
14735
14736 interrupt.park_because("test");
14738 assert!(!shutdown.parked());
14739 assert!(runner.park_here().expect("park_here"));
14740 }
14741
14742 #[tokio::test]
14756 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14757 crate::run::pin_test_home();
14758 let mut runner = runner_at(RunStatus::Implementing);
14759 let interrupt = Pause::new();
14760 runner.watch_interrupt(interrupt.clone());
14761
14762 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14763 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14764
14765 let node = async move {
14769 started_tx.send(()).expect("send started");
14770 finish_rx.await.expect("recv finish");
14771 "node finished"
14772 };
14773
14774 let interrupter = async move {
14775 started_rx.await.expect("recv started");
14776 interrupt.park_because("higher-priority task waiting");
14778 tokio::task::yield_now().await;
14782 finish_tx.send(()).expect("send finish");
14783 };
14784
14785 let (node_result, ()) = tokio::join!(node, interrupter);
14786 assert_eq!(
14787 node_result, "node finished",
14788 "the in-flight call ran to completion"
14789 );
14790
14791 assert!(runner.park_here().expect("park_here"));
14794 assert!(runner.state.parked);
14795 }
14796
14797 #[test]
14803 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14804 crate::run::pin_test_home();
14805 let mut runner = runner_at(RunStatus::Judging);
14806 runner.state.config.agents = vec![conductor()];
14810 runner.state.candidates = vec![Candidate {
14811 index: 0,
14812 label: 'A',
14813 agent: "alpha".to_owned(),
14814 branch: "magi/x/A".to_owned(),
14815 worktree: PathBuf::from("/nonexistent/worktree"),
14816 summary: "did the thing".to_owned(),
14817 stat: "1 file changed".to_owned(),
14818 files: 1,
14819 commits: 1,
14820 empty: false,
14821 failed: None,
14822 verified_noop: None,
14823 duration_ms: 1234,
14824 folded: false,
14825 }];
14826 let run_id = runner.state.id.clone();
14827
14828 let interrupt = Pause::new();
14829 runner.watch_interrupt(interrupt.clone());
14830 interrupt.park_because("task c3d4 asked to run first");
14831 assert!(runner.park_here().expect("park_here"));
14832
14833 let resumed = Runner::resume(&run_id).expect("resume");
14834 assert_eq!(resumed.state.candidates.len(), 1);
14835 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14836 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14837 assert_eq!(resumed.state.status, runner.state.status);
14838 assert!(
14839 resumed.state.parked,
14840 "still parked until `execute` actually walks the graph again"
14841 );
14842 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14843 }
14844
14845 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14847 let mut q = ask::Question::new(
14848 run.to_owned(),
14849 "implement".to_owned(),
14850 "impl-A".to_owned(),
14851 "Which storage backend should the cache use?".to_owned(),
14852 String::new(),
14853 vec!["SQLite".to_owned(), "Redis".to_owned()],
14854 );
14855 store.put(&mut q).unwrap();
14856 q
14857 }
14858
14859 #[test]
14860 fn a_failed_runs_open_question_is_abandoned() {
14861 ask_test_home();
14862 let store = ask::Questions::open();
14863 let mut runner = runner_at(RunStatus::Failed);
14864 let run = runner.state.id.clone();
14865 let q = ask_open_question(&store, &run);
14866
14867 runner.settle_questions();
14868
14869 let back = store.get(&q.id).unwrap();
14870 assert!(
14871 !back.status.open(),
14872 "the seat that asked died with the run; nobody is left to read an answer"
14873 );
14874 assert!(
14875 back.detail.contains(&run) && back.detail.contains("failed"),
14876 "the reason names what the run became, not just that it is gone: {}",
14877 back.detail
14878 );
14879 }
14880
14881 #[test]
14882 fn a_merged_runs_open_question_is_abandoned_too() {
14883 ask_test_home();
14884 let store = ask::Questions::open();
14885 for status in [RunStatus::Merged, RunStatus::Ready] {
14888 let mut runner = runner_at(status);
14889 let run = runner.state.id.clone();
14890 let q = ask_open_question(&store, &run);
14891
14892 runner.settle_questions();
14893
14894 let back = store.get(&q.id).unwrap();
14895 assert!(
14896 !back.status.open(),
14897 "{status:?} run's question must not outlive the run"
14898 );
14899 }
14900 }
14901
14902 #[test]
14903 fn a_still_resumable_runs_open_question_is_left_alone() {
14904 ask_test_home();
14905 let store = ask::Questions::open();
14906 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14912 let mut runner = runner_at(status);
14913 let run = runner.state.id.clone();
14914 let q = ask_open_question(&store, &run);
14915
14916 runner.settle_questions();
14917
14918 let back = store.get(&q.id).unwrap();
14919 assert!(
14920 back.status.open(),
14921 "{status:?} is still alive; the question must still be waiting"
14922 );
14923 }
14924 }
14925
14926 #[test]
14927 fn settle_questions_never_touches_an_already_answered_question() {
14928 ask_test_home();
14929 let store = ask::Questions::open();
14930 let mut runner = runner_at(RunStatus::Failed);
14931 let run = runner.state.id.clone();
14932 let mut q = ask_open_question(&store, &run);
14933 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14934 .unwrap();
14935 store.put(&mut q).unwrap();
14936
14937 runner.settle_questions();
14942 runner.settle_questions();
14943
14944 let back = store.get(&q.id).unwrap();
14945 assert_eq!(
14946 back.status,
14947 ask::QuestionStatus::Answered,
14948 "a real answer is a decision on record, never overwritten by a sweep"
14949 );
14950 }
14951
14952 #[tokio::test]
14963 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14964 crate::run::pin_test_home();
14965 let tmp = tempfile::tempdir().expect("tempdir");
14966 let repo = tmp.path().join("repo");
14967 std::fs::create_dir_all(&repo).unwrap();
14968 init_repo(&repo);
14969
14970 let mut config = Config::default();
14971 config.graph.worktree_root = Some(tmp.path().join("wt"));
14972
14973 let mut state = RunState::new(
14974 repo.clone(),
14975 "main".to_owned(),
14976 "deadbeef".to_owned(),
14977 "task".to_owned(),
14978 config,
14979 );
14980 let root = state.worktree_root();
14981 let wt_a = root.join("cand-A");
14982 let wt_b = root.join("cand-B");
14983 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14984 .await
14985 .expect("worktree A");
14986 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14987 .await
14988 .expect("worktree B");
14989
14990 state.candidates = vec![
14991 Candidate {
14992 index: 0,
14993 label: 'A',
14994 agent: "alpha".to_owned(),
14995 branch: "magi/x/A".to_owned(),
14996 worktree: wt_a.clone(),
14997 summary: String::new(),
14998 stat: String::new(),
14999 files: 0,
15000 commits: 0,
15001 empty: false,
15002 failed: None,
15003 verified_noop: None,
15004 duration_ms: 0,
15005 folded: false,
15006 },
15007 Candidate {
15008 index: 1,
15009 label: 'B',
15010 agent: "beta".to_owned(),
15011 branch: "magi/x/B".to_owned(),
15012 worktree: wt_b.clone(),
15013 summary: String::new(),
15014 stat: String::new(),
15015 files: 0,
15016 commits: 0,
15017 empty: false,
15018 failed: None,
15019 verified_noop: None,
15020 duration_ms: 0,
15021 folded: false,
15022 },
15023 ];
15024 state.tally = Some(Tally {
15025 first_choice: BTreeMap::from([('A', 1)]),
15026 borda: BTreeMap::new(),
15027 winner: 'A',
15028 rankings: 1,
15029 unanimous_initial: true,
15030 deliberated: false,
15031 changed_votes: 0,
15032 unanimous_final: true,
15033 tie_break: None,
15034 judges: 1,
15035 present: 1,
15036 quorum: 1,
15037 met_quorum: true,
15038 uncontested: None,
15039 });
15040 state.status = RunStatus::Ready;
15041
15042 fold_run(&mut state, false, &crate::run::home())
15043 .await
15044 .expect("fold_run");
15045
15046 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15047 assert!(
15048 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15049 "the unmerged winner's branch survives"
15050 );
15051 assert!(
15052 !state.candidates[0].folded,
15053 "the winner is not marked folded"
15054 );
15055
15056 assert!(!wt_b.exists(), "the loser's worktree is removed");
15057 assert!(
15058 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15059 "the loser's branch is removed"
15060 );
15061 assert!(state.candidates[1].folded, "the loser is marked folded");
15062 }
15063
15064 #[tokio::test]
15065 #[cfg(unix)]
15066 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15067 use std::os::fd::AsRawFd as _;
15068 use std::time::Duration;
15069
15070 let tmp = tempfile::tempdir().unwrap();
15071 let repo = tmp.path().join("repo");
15072 std::fs::create_dir_all(&repo).unwrap();
15073 init_repo(&repo);
15074 let common = repo.join(".git");
15075 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15076 let home = tmp.path().join("home");
15077 let mut config = Config::default();
15078 config.coordination = crate::config::test_coordination();
15079 config.graph.worktree_root = Some(tmp.path().join("wt"));
15080 config.graph.timeout_resource_wait = 1;
15081 let mut state = RunState::new(
15082 repo.clone(),
15083 "main".into(),
15084 "deadbeef".into(),
15085 "task".into(),
15086 config,
15087 );
15088 let root = state.worktree_root();
15089 let candidate = root.join("cand-A");
15090 let reviewer = root.join("review-1");
15091 let branch = state.branch_for('A');
15092 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15093 .await
15094 .unwrap();
15095 git::worktree_add_detached(&repo, &reviewer, "main")
15096 .await
15097 .unwrap();
15098 git::acquire_worktree_config(&repo).await.unwrap();
15099 state.enabled_worktree_config = true;
15100 state.status = RunStatus::Ready;
15101 state.candidates.push(Candidate {
15102 index: 0,
15103 label: 'A',
15104 agent: "implement".into(),
15105 branch: branch.clone(),
15106 worktree: candidate.clone(),
15107 summary: String::new(),
15108 stat: String::new(),
15109 files: 0,
15110 commits: 0,
15111 empty: false,
15112 failed: None,
15113 verified_noop: None,
15114 duration_ms: 0,
15115 folded: false,
15116 });
15117 state.save_under(&home).unwrap();
15118 let config_before = std::fs::read(common.join("config")).unwrap();
15119 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15120 .await
15121 .unwrap();
15122 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15125 assert_eq!(
15126 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15127 0
15128 );
15129 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15130 assert!(
15131 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15132 .await
15133 .is_err()
15134 );
15135 assert!(candidate.exists());
15136 assert!(reviewer.exists());
15137 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15138 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15139 assert_eq!(
15140 git::git(&repo, &["worktree", "list", "--porcelain"])
15141 .await
15142 .unwrap(),
15143 worktrees_before
15144 );
15145 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15146 .await
15147 .unwrap()
15148 .unwrap_err();
15149 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15150 drop(fold);
15151 assert!(candidate.exists() && reviewer.exists());
15152 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15153 assert_eq!(
15154 state.resource_block.as_ref().unwrap().kind,
15155 "repository coordination"
15156 );
15157 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15158 drop(external);
15159 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15160 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15161 .await
15162 .unwrap()
15163 .unwrap();
15164 drop(fold);
15165 assert!(removed.contains(&branch));
15166 assert!(!root.exists());
15167 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15168 assert!(state.candidates[0].folded);
15169 assert!(!state.enabled_worktree_config);
15170 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15171 let enabled = git::git_raw(
15172 &repo,
15173 &["config", "--local", "--get", "extensions.worktreeConfig"],
15174 )
15175 .await
15176 .unwrap();
15177 assert!(!enabled.ok(), "the last config reference was released");
15178 }
15179
15180 #[tokio::test]
15183 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15184 let tmp = tempfile::tempdir().expect("tempdir");
15185 let repo = tmp.path().join("repo");
15186 std::fs::create_dir_all(&repo).unwrap();
15187 init_repo(&repo);
15188 let home = tmp.path().join("home");
15189
15190 let mut config = Config::default();
15191 config.graph.worktree_root = Some(tmp.path().join("wt"));
15192 let mut state = RunState::new(
15193 repo.clone(),
15194 "main".to_owned(),
15195 "deadbeef".to_owned(),
15196 "task".to_owned(),
15197 config,
15198 );
15199 git::git(&repo, &["branch", "magi/x/A", "main"])
15201 .await
15202 .expect("branch");
15203 state.candidates = vec![Candidate {
15204 index: 0,
15205 label: 'A',
15206 agent: "alpha".to_owned(),
15207 branch: "magi/x/A".to_owned(),
15208 worktree: state.worktree_root().join("cand-A"),
15209 summary: String::new(),
15210 stat: String::new(),
15211 files: 0,
15212 commits: 0,
15213 empty: false,
15214 failed: None,
15215 verified_noop: None,
15216 duration_ms: 0,
15217 folded: true,
15218 }];
15219 state.released_to = Some("20260901-000000-new1".to_owned());
15220 state.released_branches = vec!["magi/x/A".to_owned()];
15221
15222 fold_run(&mut state, true, &home).await.expect("fold_run");
15223 state.status = RunStatus::Merged;
15224 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15225 state.merge = Some(crate::run::MergeOutcome {
15226 mode: MergeMode::Pr,
15227 ok: true,
15228 detail: "confirmed merged".into(),
15229 empty: false,
15230 });
15231 fold_run(&mut state, true, &home)
15232 .await
15233 .expect("merged fold");
15234
15235 assert!(
15236 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15237 "the handed-over branch survives a fold"
15238 );
15239 }
15240
15241 #[tokio::test]
15245 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15246 let tmp = tempfile::tempdir().expect("tempdir");
15247 let repo = tmp.path().join("repo");
15248 std::fs::create_dir_all(&repo).unwrap();
15249 init_repo(&repo);
15250 let run = |args: &[&str]| {
15251 let out = std::process::Command::new("git")
15252 .quiet()
15253 .args(args)
15254 .current_dir(&repo)
15255 .output()
15256 .expect("spawn git");
15257 assert!(out.status.success(), "git {args:?}");
15258 };
15259 run(&["branch", "magi/x/A"]);
15260 run(&["checkout", "-q", "-b", "magi/x/B"]);
15261 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15262 run(&["add", "-A"]);
15263 run(&["commit", "-q", "-m", "work"]);
15264 run(&["checkout", "-q", "main"]);
15265
15266 let probe = RunState::new(
15270 repo.clone(),
15271 "main".to_owned(),
15272 "deadbeef".to_owned(),
15273 "task".to_owned(),
15274 Config::default(),
15275 );
15276 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15277 let bare = tmp.path().join("origin.git");
15278 let out = std::process::Command::new("git")
15279 .quiet()
15280 .args(["init", "-q", "--bare"])
15281 .arg(&bare)
15282 .output()
15283 .expect("spawn git");
15284 assert!(out.status.success(), "git init --bare");
15285 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15286 run(&["push", "-q", "origin", "main"]);
15287
15288 let mut state = RunState::new(
15289 repo.clone(),
15290 "main".to_owned(),
15291 "deadbeef".to_owned(),
15292 "task".to_owned(),
15293 Config::default(),
15294 );
15295 state.seeds = vec![refs::Seed {
15296 token: "magi/27b2/A".to_owned(),
15297 kind: refs::SeedKind::Unresolved,
15298 sha: String::new(),
15299 branch: true,
15300 detail: "no branch or commit named magi/27b2/A".to_owned(),
15301 }];
15302
15303 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15304 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15305 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15306 let detail = empty_candidate_detail(&state, "main");
15307 assert!(detail.starts_with("empty candidate"), "{detail}");
15308 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15309 assert!(detail.contains("--start-from"), "{detail}");
15310 }
15311
15312 #[tokio::test]
15321 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15322 let tmp = tempfile::tempdir().expect("tempdir");
15323 let repo = tmp.path().join("repo");
15324 std::fs::create_dir_all(&repo).unwrap();
15325 init_repo(&repo);
15326
15327 let mut config = Config::default();
15328 config.merge.mode = MergeMode::Local;
15329
15330 let mut state = RunState::new(
15331 repo.clone(),
15332 "main".to_owned(),
15333 "deadbeef".to_owned(),
15334 "task".to_owned(),
15335 config,
15336 );
15337 state.candidates = vec![Candidate {
15338 index: 0,
15339 label: 'A',
15340 agent: "alpha".to_owned(),
15341 branch: "does-not-exist".to_owned(),
15342 worktree: repo.clone(),
15343 summary: String::new(),
15344 stat: String::new(),
15345 files: 0,
15346 commits: 0,
15347 empty: false,
15348 failed: None,
15349 verified_noop: None,
15350 duration_ms: 0,
15351 folded: false,
15352 }];
15353 state.tally = Some(Tally {
15354 first_choice: BTreeMap::from([('A', 1)]),
15355 borda: BTreeMap::new(),
15356 winner: 'A',
15357 rankings: 1,
15358 unanimous_initial: true,
15359 deliberated: false,
15360 changed_votes: 0,
15361 unanimous_final: true,
15362 tie_break: None,
15363 judges: 0,
15364 present: 0,
15365 quorum: 0,
15366 met_quorum: true,
15367 uncontested: Some("only candidate A produced a change".to_owned()),
15368 });
15369 state.reviews = vec![ReviewRound {
15370 round: 1,
15371 head: "deadbeef".to_owned(),
15372 verified_head: None,
15373 verified_at: None,
15374 reviews: Vec::new(),
15375 e2e: Vec::new(),
15376 fix: None,
15377 blocking: 0,
15378 answered: 0,
15379 expected: 0,
15380 clean: true,
15381 verify_retried: false,
15382 e2e_deferred: false,
15383 e2e_defer_reason: None,
15384 progressed: false,
15385 vote_split: false,
15386 reconsideration: Vec::new(),
15387 verdict: None,
15388 }];
15389 state.gate = vec![CommandOutcome {
15390 command: "test".to_owned(),
15391 code: Some(0),
15392 output_tail: String::new(),
15393 duration_ms: 0,
15394 resource_blocked: false,
15395 }];
15396 state.gate_ran = true;
15397 state.status = RunStatus::Ready;
15402 state.merge = Some(MergeOutcome {
15403 mode: MergeMode::Local,
15404 ok: false,
15405 detail: "already concluded".to_owned(),
15406 empty: false,
15407 });
15408
15409 let mut runner = Runner {
15410 state,
15411 roles: ResolvedRoles {
15412 implementers: Vec::new(),
15413 judges: Vec::new(),
15414 reviewers: Vec::new(),
15415 fixer: None,
15416 conductor: conductor(),
15417 implementer_roster: Vec::new(),
15418 judge_roster: Vec::new(),
15419 reviewer_roster: Vec::new(),
15420 },
15421 sem: Arc::new(Semaphore::new(1)),
15422 pause: Pause::new(),
15423 interrupt: Pause::new(),
15424 };
15425
15426 runner.merge().await.expect("merge");
15427
15428 assert_eq!(
15429 runner.state.status,
15430 RunStatus::Ready,
15431 "a concluded run's status must not change on reentry"
15432 );
15433 assert_eq!(
15434 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15435 Some("already concluded"),
15436 "merge must not run again once the node already recorded an outcome"
15437 );
15438 }
15439
15440 #[tokio::test]
15449 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15450 let tmp = tempfile::tempdir().expect("tempdir");
15451 let repo = tmp.path().join("repo");
15452 std::fs::create_dir_all(&repo).unwrap();
15453 init_repo(&repo);
15454
15455 let mut config = Config::default();
15456 config.merge.mode = MergeMode::Local;
15457
15458 let mut state = RunState::new(
15459 repo.clone(),
15460 "main".to_owned(),
15461 "deadbeef".to_owned(),
15462 "task".to_owned(),
15463 config,
15464 );
15465 state.candidates = vec![Candidate {
15466 index: 0,
15467 label: 'A',
15468 agent: "alpha".to_owned(),
15469 branch: "does-not-exist".to_owned(),
15470 worktree: repo.clone(),
15471 summary: String::new(),
15472 stat: String::new(),
15473 files: 0,
15474 commits: 0,
15475 empty: false,
15476 failed: None,
15477 verified_noop: None,
15478 duration_ms: 0,
15479 folded: false,
15480 }];
15481 state.tally = Some(Tally {
15482 first_choice: BTreeMap::from([('A', 1)]),
15483 borda: BTreeMap::new(),
15484 winner: 'A',
15485 rankings: 1,
15486 unanimous_initial: true,
15487 deliberated: false,
15488 changed_votes: 0,
15489 unanimous_final: true,
15490 tie_break: None,
15491 judges: 0,
15492 present: 0,
15493 quorum: 0,
15494 met_quorum: true,
15495 uncontested: Some("only candidate A produced a change".to_owned()),
15496 });
15497 state.reviews = vec![ReviewRound {
15498 round: 1,
15499 head: "deadbeef".to_owned(),
15500 verified_head: None,
15501 verified_at: None,
15502 reviews: Vec::new(),
15503 e2e: Vec::new(),
15504 fix: None,
15505 blocking: 0,
15506 answered: 0,
15507 expected: 0,
15508 clean: true,
15509 verify_retried: false,
15510 e2e_deferred: false,
15511 e2e_defer_reason: None,
15512 progressed: false,
15513 vote_split: false,
15514 reconsideration: Vec::new(),
15515 verdict: None,
15516 }];
15517 state.gate = Vec::new();
15519 state.gate_ran = false;
15520 state.status = RunStatus::Gating;
15521
15522 let mut runner = Runner {
15523 state,
15524 roles: ResolvedRoles {
15525 implementers: Vec::new(),
15526 judges: Vec::new(),
15527 reviewers: Vec::new(),
15528 fixer: None,
15529 conductor: conductor(),
15530 implementer_roster: Vec::new(),
15531 judge_roster: Vec::new(),
15532 reviewer_roster: Vec::new(),
15533 },
15534 sem: Arc::new(Semaphore::new(1)),
15535 pause: Pause::new(),
15536 interrupt: Pause::new(),
15537 };
15538
15539 runner.merge().await.expect("merge");
15540
15541 assert!(
15542 runner.state.merge.is_none(),
15543 "an empty gate must never be read as a passing one: {:?}",
15544 runner.state.merge
15545 );
15546 }
15547
15548 #[tokio::test]
15555 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15556 let tmp = tempfile::tempdir().expect("tempdir");
15557 let repo = tmp.path().join("repo");
15558 std::fs::create_dir_all(&repo).unwrap();
15559 init_repo(&repo);
15560
15561 let config = Config::default();
15563
15564 let mut state = RunState::new(
15565 repo.clone(),
15566 "main".to_owned(),
15567 "deadbeef".to_owned(),
15568 "task".to_owned(),
15569 config,
15570 );
15571 state.candidates = vec![Candidate {
15572 index: 0,
15573 label: 'A',
15574 agent: "alpha".to_owned(),
15575 branch: "does-not-exist".to_owned(),
15576 worktree: repo.clone(),
15577 summary: String::new(),
15578 stat: String::new(),
15579 files: 0,
15580 commits: 0,
15581 empty: false,
15582 failed: None,
15583 verified_noop: None,
15584 duration_ms: 0,
15585 folded: false,
15586 }];
15587 state.tally = Some(Tally {
15588 first_choice: BTreeMap::from([('A', 1)]),
15589 borda: BTreeMap::new(),
15590 winner: 'A',
15591 rankings: 1,
15592 unanimous_initial: true,
15593 deliberated: false,
15594 changed_votes: 0,
15595 unanimous_final: true,
15596 tie_break: None,
15597 judges: 0,
15598 present: 0,
15599 quorum: 0,
15600 met_quorum: true,
15601 uncontested: Some("only candidate A produced a change".to_owned()),
15602 });
15603 state.reviews = vec![ReviewRound {
15604 round: 1,
15605 head: "deadbeef".to_owned(),
15606 verified_head: None,
15607 verified_at: None,
15608 reviews: Vec::new(),
15609 e2e: Vec::new(),
15610 fix: None,
15611 blocking: 0,
15612 answered: 0,
15613 expected: 0,
15614 clean: true,
15615 verify_retried: false,
15616 e2e_deferred: false,
15617 e2e_defer_reason: None,
15618 progressed: false,
15619 vote_split: false,
15620 reconsideration: Vec::new(),
15621 verdict: None,
15622 }];
15623
15624 let mut runner = Runner {
15625 state,
15626 roles: ResolvedRoles {
15627 implementers: Vec::new(),
15628 judges: Vec::new(),
15629 reviewers: Vec::new(),
15630 fixer: None,
15631 conductor: conductor(),
15632 implementer_roster: Vec::new(),
15633 judge_roster: Vec::new(),
15634 reviewer_roster: Vec::new(),
15635 },
15636 sem: Arc::new(Semaphore::new(1)),
15637 pause: Pause::new(),
15638 interrupt: Pause::new(),
15639 };
15640
15641 runner.gate().await.expect("gate");
15642 assert!(
15643 runner.state.gate_ran,
15644 "zero configured commands is still a real attempt, not an unrun gate"
15645 );
15646 assert!(runner.state.gate.is_empty());
15647 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15648 assert_ne!(
15649 runner.state.status,
15650 RunStatus::Blocked,
15651 "a gate with nothing to check must not read as failed"
15652 );
15653
15654 runner.merge().await.expect("merge");
15655 assert_eq!(
15656 runner.state.status,
15657 RunStatus::Ready,
15658 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15659 );
15660 }
15661
15662 #[tokio::test]
15673 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15674 crate::run::pin_test_home();
15675 let home = crate::run::home();
15676
15677 let tmp = tempfile::tempdir().expect("tempdir");
15678 let repo = tmp.path().join("repo");
15679 std::fs::create_dir_all(&repo).unwrap();
15680 init_repo(&repo);
15681 let cache_dir = tmp.path().join("target");
15684
15685 let mut config = Config::default();
15686 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15687 config.graph.timeout_verify = Some(2);
15690 config.graph.timeout_resource_wait = 2;
15691
15692 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15693 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15694 .expect("no io error acquiring directly")
15695 {
15696 crate::cache::AcquireOutcome::Acquired(g) => g,
15697 crate::cache::AcquireOutcome::Busy(b) => {
15698 panic!("expected the direct acquire to win the lease first: {b:?}")
15699 }
15700 };
15701
15702 let mut state = RunState::new(
15703 repo.clone(),
15704 "main".to_owned(),
15705 "deadbeef".to_owned(),
15706 "task".to_owned(),
15707 config,
15708 );
15709 state.candidates = vec![Candidate {
15710 index: 0,
15711 label: 'A',
15712 agent: "alpha".to_owned(),
15713 branch: "does-not-exist".to_owned(),
15714 worktree: repo.clone(),
15715 summary: String::new(),
15716 stat: String::new(),
15717 files: 0,
15718 commits: 0,
15719 empty: false,
15720 failed: None,
15721 verified_noop: None,
15722 duration_ms: 0,
15723 folded: false,
15724 }];
15725 state.tally = Some(Tally {
15726 first_choice: BTreeMap::from([('A', 1)]),
15727 borda: BTreeMap::new(),
15728 winner: 'A',
15729 rankings: 1,
15730 unanimous_initial: true,
15731 deliberated: false,
15732 changed_votes: 0,
15733 unanimous_final: true,
15734 tie_break: None,
15735 judges: 0,
15736 present: 0,
15737 quorum: 0,
15738 met_quorum: true,
15739 uncontested: Some("only candidate A produced a change".to_owned()),
15740 });
15741 state.reviews = vec![ReviewRound {
15742 round: 1,
15743 head: "deadbeef".to_owned(),
15744 verified_head: None,
15745 verified_at: None,
15746 reviews: Vec::new(),
15747 e2e: Vec::new(),
15748 fix: None,
15749 blocking: 0,
15750 answered: 0,
15751 expected: 0,
15752 clean: true,
15753 verify_retried: false,
15754 e2e_deferred: false,
15755 e2e_defer_reason: None,
15756 progressed: false,
15757 vote_split: false,
15758 reconsideration: Vec::new(),
15759 verdict: None,
15760 }];
15761
15762 let mut runner = Runner {
15763 state,
15764 roles: ResolvedRoles {
15765 implementers: Vec::new(),
15766 judges: Vec::new(),
15767 reviewers: Vec::new(),
15768 fixer: None,
15769 conductor: conductor(),
15770 implementer_roster: Vec::new(),
15771 judge_roster: Vec::new(),
15772 reviewer_roster: Vec::new(),
15773 },
15774 sem: Arc::new(Semaphore::new(1)),
15775 pause: Pause::new(),
15776 interrupt: Pause::new(),
15777 };
15778
15779 let started = std::time::Instant::now();
15780 runner.gate().await.expect("gate");
15781 assert!(
15782 started.elapsed() < Duration::from_secs(1),
15783 "a gate with nothing to run must never wait on a lease it never needed"
15784 );
15785 assert!(
15786 runner.state.gate_ran,
15787 "zero commands is still a real, immediate attempt"
15788 );
15789 assert!(runner.state.gate.is_empty());
15790 assert_ne!(
15791 runner.state.status,
15792 RunStatus::Blocked,
15793 "must not read as resource-blocked on a lease it never asked for"
15794 );
15795 }
15796
15797 #[cfg(unix)]
15798 #[tokio::test]
15799 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15800 crate::run::pin_test_home();
15801 let tmp = tempfile::tempdir().unwrap();
15802 let mut config = Config::default();
15803 config.disk.min_free_bytes = 0;
15804 let mut state = RunState::new(
15805 tmp.path().to_owned(),
15806 "main".into(),
15807 "base".into(),
15808 "verification evidence".into(),
15809 config,
15810 );
15811 let (outcomes, pids) = run_commands(
15812 &mut state,
15813 "gate",
15814 "gate",
15815 1,
15816 &["sh".into(), "-c".into()],
15817 &[
15818 "printf before-timeout; exec sleep 30".into(),
15819 "kill -TERM $$".into(),
15820 "exit 7".into(),
15821 "printf success".into(),
15822 ],
15823 tmp.path(),
15824 Duration::from_secs(1),
15825 )
15826 .await;
15827 wait_for_timed_out_children_to_die(&pids).await;
15828 assert_eq!(
15829 pids.len(),
15830 1,
15831 "unleased timeout retains its death-check PID"
15832 );
15833 assert_eq!(outcomes.len(), 4);
15834 assert_eq!(outcomes[0].code, None);
15835 assert!(outcomes[0].output_tail.contains("before-timeout"));
15836 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15837 assert_eq!(outcomes[1].code, None);
15838 assert!(outcomes[1].output_tail.contains("command terminated:"));
15839 assert!(!outcomes[1].output_tail.contains("timed out"));
15840 assert_eq!(outcomes[2].code, Some(7));
15841 assert!(!outcomes[2].output_tail.contains("timed out"));
15842 assert_eq!(outcomes[3].code, Some(0));
15843 assert_eq!(outcomes[3].output_tail, "success");
15844 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15845 }
15846
15847 #[tokio::test]
15857 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15858 crate::run::pin_test_home();
15859
15860 let tmp = tempfile::tempdir().expect("tempdir");
15861 let repo = tmp.path().join("repo");
15862 std::fs::create_dir_all(&repo).unwrap();
15863 init_repo(&repo);
15864
15865 let mut config = Config::default();
15866 config.verify.gate = vec![
15867 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15868 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15869 .to_owned(),
15870 ];
15871
15872 let mut state = RunState::new(
15873 repo.clone(),
15874 "main".to_owned(),
15875 "deadbeef".to_owned(),
15876 "task".to_owned(),
15877 config,
15878 );
15879 let run_id = state.id.clone();
15880 state.candidates = vec![Candidate {
15881 index: 0,
15882 label: 'A',
15883 agent: "alpha".to_owned(),
15884 branch: "does-not-exist".to_owned(),
15885 worktree: repo.clone(),
15886 summary: String::new(),
15887 stat: String::new(),
15888 files: 0,
15889 commits: 0,
15890 empty: false,
15891 failed: None,
15892 verified_noop: None,
15893 duration_ms: 0,
15894 folded: false,
15895 }];
15896 state.tally = Some(Tally {
15897 first_choice: BTreeMap::from([('A', 1)]),
15898 borda: BTreeMap::new(),
15899 winner: 'A',
15900 rankings: 1,
15901 unanimous_initial: true,
15902 deliberated: false,
15903 changed_votes: 0,
15904 unanimous_final: true,
15905 tie_break: None,
15906 judges: 0,
15907 present: 0,
15908 quorum: 0,
15909 met_quorum: true,
15910 uncontested: Some("only candidate A produced a change".to_owned()),
15911 });
15912 state.reviews = vec![ReviewRound {
15913 round: 1,
15914 head: "deadbeef".to_owned(),
15915 verified_head: None,
15916 verified_at: None,
15917 reviews: Vec::new(),
15918 e2e: Vec::new(),
15919 fix: None,
15920 blocking: 0,
15921 answered: 0,
15922 expected: 0,
15923 clean: true,
15924 verify_retried: false,
15925 e2e_deferred: false,
15926 e2e_defer_reason: None,
15927 progressed: false,
15928 vote_split: false,
15929 reconsideration: Vec::new(),
15930 verdict: None,
15931 }];
15932
15933 let mut runner = Runner {
15934 state,
15935 roles: ResolvedRoles {
15936 implementers: Vec::new(),
15937 judges: Vec::new(),
15938 reviewers: Vec::new(),
15939 fixer: None,
15940 conductor: conductor(),
15941 implementer_roster: Vec::new(),
15942 judge_roster: Vec::new(),
15943 reviewer_roster: Vec::new(),
15944 },
15945 sem: Arc::new(Semaphore::new(1)),
15946 pause: Pause::new(),
15947 interrupt: Pause::new(),
15948 };
15949
15950 let started_marker = repo.join("started.marker");
15951 let release_marker = repo.join("release.marker");
15952 let poller = tokio::spawn(async move {
15953 for _ in 0..100 {
15958 if started_marker.exists()
15959 && let Ok(s) = crate::run::RunState::load(&run_id)
15960 && let Some(a) = s.active.get("gate")
15961 {
15962 std::fs::write(&release_marker, b"go").expect("release marker");
15963 return Some(a.clone());
15964 }
15965 tokio::time::sleep(Duration::from_millis(50)).await;
15966 }
15967 None
15968 });
15969
15970 runner.gate().await.expect("gate");
15971 let captured = poller.await.expect("poller task");
15972 let captured = captured.expect(
15973 "the poller never saw a `gate` task entry in run.json while the command was \
15974 still blocked on its own release marker",
15975 );
15976
15977 assert_eq!(captured.task.as_deref(), Some("gate"));
15978 assert_eq!(captured.node, "gate");
15979 assert_eq!(captured.index, Some(1));
15980 assert_eq!(captured.total, Some(1));
15981 assert!(
15982 captured
15983 .command
15984 .as_deref()
15985 .is_some_and(|c| c.contains("started.marker")),
15986 "{captured:?}"
15987 );
15988
15989 assert!(
15990 runner.state.active.is_empty(),
15991 "the entry must be cleared once the command actually finished: {:?}",
15992 runner.state.active
15993 );
15994 assert!(runner.state.gate_ran);
15995 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15996 }
15997
15998 #[tokio::test]
16001 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16002 use crate::verdict::{Finding, ReviewVote, Severity};
16003 crate::run::pin_test_home();
16004 let tmp = tempfile::tempdir().expect("tempdir");
16005 let repo = tmp.path().join("repo");
16006 std::fs::create_dir_all(&repo).unwrap();
16007 init_repo(&repo);
16008
16009 for (severity, vote, expect) in [
16010 (Severity::Major, ReviewVote::Reject, true),
16011 (Severity::Minor, ReviewVote::Reject, false),
16012 (Severity::Major, ReviewVote::Approve, false),
16013 ] {
16014 let mut round = review_round(false, 1, 1, 1, false, true);
16015 round.reviews = vec![ReviewRecord {
16016 reviewer: 1,
16017 agent: "alpha".to_owned(),
16018 summary: String::new(),
16019 findings: vec![Finding {
16020 id: "R1-1-1".to_owned(),
16021 severity,
16022 file: None,
16023 line: None,
16024 title: "t".to_owned(),
16025 detail: String::new(),
16026 }],
16027 vote: Some(vote),
16028 failed: None,
16029 duration_ms: 0,
16030 attempts: 0,
16031 }];
16032 let mut state = RunState::new(
16033 repo.clone(),
16034 "main".to_owned(),
16035 "deadbeef".to_owned(),
16036 "task".to_owned(),
16037 Config::default(),
16038 );
16039 state.reviews = vec![round];
16040 let mut runner = Runner {
16041 state,
16042 roles: ResolvedRoles {
16043 implementers: Vec::new(),
16044 judges: Vec::new(),
16045 reviewers: Vec::new(),
16046 fixer: None,
16047 conductor: conductor(),
16048 implementer_roster: Vec::new(),
16049 judge_roster: Vec::new(),
16050 reviewer_roster: Vec::new(),
16051 },
16052 sem: Arc::new(Semaphore::new(1)),
16053 pause: Pause::new(),
16054 interrupt: Pause::new(),
16055 };
16056 let shell = runner.state.config.shell();
16057 runner
16058 .stop_reviewing("round budget spent", &shell, &repo)
16059 .await
16060 .expect("stop_reviewing");
16061 assert_eq!(runner.state.status, RunStatus::Gating);
16062 assert_eq!(
16063 runner.state.contested_handoff.is_some(),
16064 expect,
16065 "{severity:?} + {vote:?}"
16066 );
16067 }
16068 }
16069
16070 #[tokio::test]
16083 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16084 crate::run::pin_test_home();
16085 let home = crate::run::home();
16086
16087 let tmp = tempfile::tempdir().expect("tempdir");
16088 let repo = tmp.path().join("repo");
16089 std::fs::create_dir_all(&repo).unwrap();
16090 init_repo(&repo);
16091 let head = crate::git::rev_parse(&repo, "HEAD")
16092 .await
16093 .expect("rev-parse");
16094 let cache_dir = tmp.path().join("target");
16097
16098 let mut config = Config::default();
16099 config.verify.e2e = vec![format!(
16100 "CARGO_TARGET_DIR='{}' test -f README.md",
16101 cache_dir.display()
16102 )];
16103 config.graph.review_rounds = 1;
16104 config.graph.timeout_verify = Some(2);
16107 config.graph.timeout_resource_wait = 2;
16108
16109 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16110 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16111 .expect("no io error acquiring directly")
16112 {
16113 crate::cache::AcquireOutcome::Acquired(g) => g,
16114 crate::cache::AcquireOutcome::Busy(b) => {
16115 panic!("expected the direct acquire to win the lease first: {b:?}")
16116 }
16117 };
16118
16119 let mut state = RunState::new(
16120 repo.clone(),
16121 "main".to_owned(),
16122 head.clone(),
16123 "task".to_owned(),
16124 config,
16125 );
16126 state.candidates = vec![Candidate {
16127 index: 0,
16128 label: 'A',
16129 agent: "alpha".to_owned(),
16130 branch: "does-not-exist".to_owned(),
16131 worktree: repo.clone(),
16132 summary: String::new(),
16133 stat: String::new(),
16134 files: 0,
16135 commits: 0,
16136 empty: false,
16137 failed: None,
16138 verified_noop: None,
16139 duration_ms: 0,
16140 folded: false,
16141 }];
16142 state.tally = Some(Tally {
16143 first_choice: BTreeMap::from([('A', 1)]),
16144 borda: BTreeMap::new(),
16145 winner: 'A',
16146 rankings: 1,
16147 unanimous_initial: true,
16148 deliberated: false,
16149 changed_votes: 0,
16150 unanimous_final: true,
16151 tie_break: None,
16152 judges: 0,
16153 present: 0,
16154 quorum: 0,
16155 met_quorum: true,
16156 uncontested: Some("only candidate A produced a change".to_owned()),
16157 });
16158 state.reviews = vec![ReviewRound {
16162 round: 1,
16163 head: head.clone(),
16164 verified_head: None,
16165 verified_at: None,
16166 reviews: Vec::new(),
16167 e2e: Vec::new(),
16168 fix: None,
16169 blocking: 1,
16170 answered: 1,
16171 expected: 1,
16172 clean: false,
16173 verify_retried: false,
16174 e2e_deferred: true,
16175 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16176 progressed: false,
16177 vote_split: false,
16178 reconsideration: Vec::new(),
16179 verdict: None,
16180 }];
16181
16182 let mut runner = Runner {
16183 state,
16184 roles: ResolvedRoles {
16185 implementers: Vec::new(),
16186 judges: Vec::new(),
16187 reviewers: Vec::new(),
16188 fixer: None,
16189 conductor: conductor(),
16190 implementer_roster: Vec::new(),
16191 judge_roster: Vec::new(),
16192 reviewer_roster: Vec::new(),
16193 },
16194 sem: Arc::new(Semaphore::new(1)),
16195 pause: Pause::new(),
16196 interrupt: Pause::new(),
16197 };
16198
16199 let shell = runner.state.config.shell();
16200 runner
16201 .stop_reviewing("round budget spent", &shell, &repo)
16202 .await
16203 .expect("stop_reviewing");
16204
16205 let last = runner.state.reviews.last().expect("round record");
16206 assert_eq!(
16207 last.e2e_status(),
16208 E2eStatus::ResourceBlocked,
16209 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16210 failed: {last:?}"
16211 );
16212 assert_eq!(
16213 last.verified_head.as_deref(),
16214 Some(head.as_str()),
16215 "which commit this attempt targeted is known even though nothing finished checking \
16216 it"
16217 );
16218 let first_attempt_at = last
16219 .verified_at
16220 .expect("when this attempt ran is known too");
16221 assert_ne!(
16222 runner.state.status,
16223 RunStatus::Blocked,
16224 "contention is evidence about the machine, not the patch — it must not settle the \
16225 run as blocked: {:?}",
16226 runner.state.status
16227 );
16228 assert!(
16229 !runner
16230 .state
16231 .events
16232 .iter()
16233 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16234 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16235 runner.state.events
16236 );
16237
16238 runner
16243 .stop_reviewing("round budget spent", &shell, &repo)
16244 .await
16245 .expect("stop_reviewing retry");
16246 assert_eq!(
16247 runner.state.reviews.len(),
16248 1,
16249 "no new round was started: {:?}",
16250 runner.state.reviews
16251 );
16252 let last = runner.state.reviews.last().expect("round record");
16253 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16254 assert!(
16255 last.verified_at.expect("still known") > first_attempt_at,
16256 "a second reentry must be a fresh attempt, not a stale copy of the first"
16257 );
16258 assert_ne!(runner.state.status, RunStatus::Blocked);
16259
16260 held.release();
16261 }
16262
16263 #[tokio::test]
16275 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16276 crate::run::pin_test_home();
16277 let home = crate::run::home();
16278
16279 let tmp = tempfile::tempdir().expect("tempdir");
16280 let repo = tmp.path().join("repo");
16281 std::fs::create_dir_all(&repo).unwrap();
16282 init_repo(&repo);
16283 let head = crate::git::rev_parse(&repo, "HEAD")
16284 .await
16285 .expect("rev-parse");
16286 let cache_dir = tmp.path().join("target");
16287
16288 let mut config = Config::default();
16289 config.verify.e2e = vec![format!(
16290 "CARGO_TARGET_DIR='{}' test -f README.md",
16291 cache_dir.display()
16292 )];
16293 config.graph.review_rounds = 1;
16294 config.graph.timeout_verify = Some(2);
16295 config.graph.timeout_resource_wait = 2;
16296
16297 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16298 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16299 .expect("no io error acquiring directly")
16300 {
16301 crate::cache::AcquireOutcome::Acquired(g) => g,
16302 crate::cache::AcquireOutcome::Busy(b) => {
16303 panic!("expected the direct acquire to win the lease first: {b:?}")
16304 }
16305 };
16306
16307 let mut state = RunState::new(
16308 repo.clone(),
16309 "main".to_owned(),
16310 head.clone(),
16311 "task".to_owned(),
16312 config,
16313 );
16314 state.candidates = vec![Candidate {
16315 index: 0,
16316 label: 'A',
16317 agent: "alpha".to_owned(),
16318 branch: "does-not-exist".to_owned(),
16319 worktree: repo.clone(),
16320 summary: String::new(),
16321 stat: String::new(),
16322 files: 0,
16323 commits: 0,
16324 empty: false,
16325 failed: None,
16326 verified_noop: None,
16327 duration_ms: 0,
16328 folded: false,
16329 }];
16330 state.tally = Some(Tally {
16331 first_choice: BTreeMap::from([('A', 1)]),
16332 borda: BTreeMap::new(),
16333 winner: 'A',
16334 rankings: 1,
16335 unanimous_initial: true,
16336 deliberated: false,
16337 changed_votes: 0,
16338 unanimous_final: true,
16339 tie_break: None,
16340 judges: 0,
16341 present: 0,
16342 quorum: 0,
16343 met_quorum: true,
16344 uncontested: Some("only candidate A produced a change".to_owned()),
16345 });
16346 state.reviews = vec![ReviewRound {
16350 round: 1,
16351 head: head.clone(),
16352 verified_head: Some(head.clone()),
16353 verified_at: Some(jiff::Timestamp::now()),
16354 reviews: Vec::new(),
16355 e2e: vec![CommandOutcome {
16356 command: format!(
16357 "CARGO_TARGET_DIR='{}' test -f README.md",
16358 cache_dir.display()
16359 ),
16360 code: None,
16361 output_tail: "waiting for the shared build cache".to_owned(),
16362 duration_ms: 0,
16363 resource_blocked: true,
16364 }],
16365 fix: None,
16366 blocking: 1,
16367 answered: 1,
16368 expected: 1,
16369 clean: false,
16370 verify_retried: false,
16371 e2e_deferred: false,
16372 e2e_defer_reason: None,
16373 progressed: false,
16374 vote_split: false,
16375 reconsideration: Vec::new(),
16376 verdict: None,
16377 }];
16378
16379 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16380 let mut runner = Runner {
16381 state,
16382 roles: ResolvedRoles {
16383 implementers: Vec::new(),
16384 judges: Vec::new(),
16385 reviewers: Vec::new(),
16386 fixer: None,
16387 conductor: conductor(),
16388 implementer_roster: Vec::new(),
16389 judge_roster: Vec::new(),
16390 reviewer_roster: Vec::new(),
16391 },
16392 sem: Arc::new(Semaphore::new(1)),
16393 pause: Pause::new(),
16394 interrupt: Pause::new(),
16395 };
16396
16397 runner.review_loop().await.expect("review_loop");
16402
16403 assert_eq!(
16404 runner.state.reviews.len(),
16405 1,
16406 "no new round was started on top of the unresolved one: {:?}",
16407 runner.state.reviews
16408 );
16409 let last = &runner.state.reviews[0];
16410 assert_eq!(
16411 last.e2e_status(),
16412 E2eStatus::ResourceBlocked,
16413 "still contended: {last:?}"
16414 );
16415 assert!(
16416 last.verified_at.expect("still known") > first_attempt_at,
16417 "review_loop must have actually retried the check, not left it exactly as found"
16418 );
16419 assert_ne!(
16420 runner.state.status,
16421 RunStatus::Blocked,
16422 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16423 runner.state.status
16424 );
16425
16426 held.release();
16427 }
16428
16429 #[tokio::test]
16430 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16431 crate::run::pin_test_home();
16432 for status in [RunStatus::Blocked, RunStatus::Gating] {
16433 let mut runner = runner_at(status);
16434 runner.state.config.graph.land = true;
16435 runner.state.gate_ran = true;
16436 runner.state.merge = Some(MergeOutcome {
16437 mode: MergeMode::Pr,
16438 ok: false,
16439 detail: "the fixer produced no commit".into(),
16440 empty: false,
16441 });
16442 let url = "https://example.invalid/o/r/pull/1";
16443 runner.state.pr = Some(crate::run::PrRecord {
16444 url: url.into(),
16445 number: 1,
16446 state: "open".into(),
16447 checks: "red".into(),
16448 round: 1,
16449 rounds: 1,
16450 red_at_merge: Vec::new(),
16451 });
16452 runner.state.save().unwrap();
16453 runner.execute_graph().await.unwrap();
16457 assert_eq!(runner.state.status, RunStatus::Blocked);
16458 assert!(runner.state.land_sync_pending);
16459 assert!(runner.state.events.iter().any(|event|
16460 event.node == "land" && event.message == format!("watching {url}")));
16461 assert_eq!(
16462 runner.state.merge.as_ref().unwrap().detail,
16463 "the fixer produced no commit"
16464 );
16465 }
16466 }
16467
16468 #[tokio::test]
16469 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16470 crate::run::pin_test_home();
16471 let tmp = tempfile::tempdir().expect("tempdir");
16472 let repo = tmp.path().join("repo");
16473 std::fs::create_dir_all(&repo).unwrap();
16474 init_repo(&repo);
16475
16476 let mut config = Config::default();
16477 config.merge.mode = MergeMode::Pr;
16478 config.graph.land = true;
16479 config.graph.land_approval = false;
16480
16481 let mut state = RunState::new(
16482 repo.clone(),
16483 "main".to_owned(),
16484 "deadbeef".to_owned(),
16485 "task".to_owned(),
16486 config,
16487 );
16488 state.candidates = vec![Candidate {
16489 index: 0,
16490 label: 'A',
16491 agent: "alpha".to_owned(),
16492 branch: "does-not-exist".to_owned(),
16493 worktree: repo.clone(),
16494 summary: String::new(),
16495 stat: String::new(),
16496 files: 0,
16497 commits: 0,
16498 empty: false,
16499 failed: None,
16500 verified_noop: None,
16501 duration_ms: 0,
16502 folded: false,
16503 }];
16504 state.tally = Some(Tally {
16505 first_choice: BTreeMap::from([('A', 1)]),
16506 borda: BTreeMap::new(),
16507 winner: 'A',
16508 rankings: 1,
16509 unanimous_initial: true,
16510 deliberated: false,
16511 changed_votes: 0,
16512 unanimous_final: true,
16513 tie_break: None,
16514 judges: 0,
16515 present: 0,
16516 quorum: 0,
16517 met_quorum: true,
16518 uncontested: Some("only candidate A produced a change".to_owned()),
16519 });
16520 state.reviews = vec![ReviewRound {
16521 round: 1,
16522 head: "deadbeef".to_owned(),
16523 verified_head: None,
16524 verified_at: None,
16525 reviews: Vec::new(),
16526 e2e: Vec::new(),
16527 fix: None,
16528 blocking: 0,
16529 answered: 0,
16530 expected: 0,
16531 clean: true,
16532 verify_retried: false,
16533 e2e_deferred: false,
16534 e2e_defer_reason: None,
16535 progressed: false,
16536 vote_split: false,
16537 reconsideration: Vec::new(),
16538 verdict: None,
16539 }];
16540 state.gate = vec![CommandOutcome {
16541 command: "test".to_owned(),
16542 code: Some(0),
16543 output_tail: String::new(),
16544 duration_ms: 0,
16545 resource_blocked: false,
16546 }];
16547 state.gate_ran = true;
16548 state.status = RunStatus::Landing;
16552 state.merge = Some(MergeOutcome {
16553 mode: MergeMode::Pr,
16554 ok: true,
16555 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16556 empty: false,
16557 });
16558
16559 ask_test_home();
16563 state.save().expect("save landing run");
16566 let store = ask::Questions::open();
16567 let q = ask_open_question(&store, &state.id);
16568
16569 let mut runner = Runner {
16570 state,
16571 roles: ResolvedRoles {
16572 implementers: Vec::new(),
16573 judges: Vec::new(),
16574 reviewers: Vec::new(),
16575 fixer: None,
16576 conductor: conductor(),
16577 implementer_roster: Vec::new(),
16578 judge_roster: Vec::new(),
16579 reviewer_roster: Vec::new(),
16580 },
16581 sem: Arc::new(Semaphore::new(1)),
16582 pause: Pause::new(),
16583 interrupt: Pause::new(),
16584 };
16585
16586 runner.execute().await.expect("execute");
16591
16592 assert_eq!(
16593 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16594 Some("https://example.invalid/x/y/pull/1"),
16595 "reentry must not push again or open a second pull request over the \
16596 one `land` is already watching"
16597 );
16598 assert_ne!(
16599 runner.state.status,
16600 RunStatus::Landing,
16601 "land could not actually reach the fake pull request, so it must \
16602 have given up rather than left the run silently parked forever"
16603 );
16604 assert_eq!(runner.state.status, RunStatus::Blocked);
16608 assert!(
16609 store.get(&q.id).unwrap().status.open(),
16610 "Blocked is still alive; settle_questions must have been a no-op here"
16611 );
16612 }
16613
16614 fn state_with_round(round: ReviewRound) -> RunState {
16615 let mut s = RunState::new(
16616 PathBuf::from("/repo"),
16617 "main".to_owned(),
16618 "abc1234".to_owned(),
16619 "add retries".to_owned(),
16620 Config::default(),
16621 );
16622 s.reviews = vec![round];
16623 s
16624 }
16625
16626 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16627 crate::verdict::Finding {
16628 id: id.to_owned(),
16629 severity,
16630 file: None,
16631 line: None,
16632 title: title.to_owned(),
16633 detail: String::new(),
16634 }
16635 }
16636
16637 #[test]
16638 fn pr_body_names_open_findings_and_declined_ones() {
16639 let round = ReviewRound {
16640 round: 2,
16641 head: "deadbee".to_owned(),
16642 verified_head: None,
16643 verified_at: None,
16644 reviews: vec![ReviewRecord {
16645 attempts: 0,
16646 reviewer: 1,
16647 agent: "alpha".to_owned(),
16648 summary: String::new(),
16649 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16650 vote: None,
16651 failed: None,
16652 duration_ms: 0,
16653 }],
16654 e2e: vec![CommandOutcome {
16655 command: "cargo test".to_owned(),
16656 code: Some(0),
16657 output_tail: String::new(),
16658 duration_ms: 0,
16659 resource_blocked: false,
16660 }],
16661 verify_retried: false,
16662 e2e_deferred: false,
16663 e2e_defer_reason: None,
16664 fix: Some(FixRecord {
16665 agent: "alpha".to_owned(),
16666 addressed: Vec::new(),
16667 rejected: vec![crate::verdict::Rejection {
16668 id: "R1-1-1".to_owned(),
16669 why: "not reachable from any caller".to_owned(),
16670 }],
16671 notes: String::new(),
16672 committed: true,
16673 failed: None,
16674 duration_ms: 0,
16675 continuation: None,
16676 }),
16677 blocking: 0,
16678 answered: 1,
16679 expected: 1,
16680 clean: false,
16681 progressed: true,
16682 vote_split: false,
16683 reconsideration: Vec::new(),
16684 verdict: None,
16685 };
16686 let state = state_with_round(round);
16687 let body = pr_message(&state, 'A').body;
16688
16689 assert!(body.contains("add retries"), "the task must still be there");
16690 assert!(body.contains("R2-1-1"), "{body}");
16691 assert!(!body.contains("unused import"), "{body}");
16692 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16693 assert!(
16694 !body.contains("not reachable from any caller"),
16695 "the reason it was declined: {body}"
16696 );
16697 }
16698
16699 #[test]
16700 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16701 let round = ReviewRound {
16702 round: 1,
16703 head: "deadbee".to_owned(),
16704 verified_head: None,
16705 verified_at: None,
16706 reviews: vec![ReviewRecord {
16707 attempts: 0,
16708 reviewer: 1,
16709 agent: "alpha".to_owned(),
16710 summary: String::new(),
16711 findings: Vec::new(),
16712 vote: None,
16713 failed: None,
16714 duration_ms: 0,
16715 }],
16716 e2e: Vec::new(),
16717 verify_retried: false,
16718 e2e_deferred: false,
16719 e2e_defer_reason: None,
16720 fix: None,
16721 blocking: 0,
16722 answered: 1,
16723 expected: 1,
16724 clean: true,
16725 progressed: false,
16726 vote_split: false,
16727 reconsideration: Vec::new(),
16728 verdict: None,
16729 };
16730 let state = state_with_round(round);
16731 let body = pr_message(&state, 'A').body;
16732 assert!(!body.contains("Open review findings"), "{body}");
16733 assert!(!body.contains("Declined"), "{body}");
16734 }
16735
16736 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16737 let mut state = RunState::new(
16738 PathBuf::from("/repo"),
16739 "main".to_owned(),
16740 "abc1234".to_owned(),
16741 instruction.to_owned(),
16742 Config::default(),
16743 );
16744 state.candidates.push(Candidate {
16745 index: 0,
16746 label: 'A',
16747 agent: "alpha".to_owned(),
16748 branch: "magi/x/A".to_owned(),
16749 worktree: PathBuf::from("/wt"),
16750 summary: summary.to_owned(),
16751 stat: String::new(),
16752 files: 1,
16753 commits: 1,
16754 empty: false,
16755 failed: None,
16756 verified_noop: None,
16757 folded: false,
16758 duration_ms: 0,
16759 });
16760 state
16761 }
16762
16763 fn review_state(subjects: &[&str]) -> RunState {
16764 let mut state = state_with_summary(
16765 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16766 "",
16767 );
16768 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16769 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16770 state
16771 }
16772
16773 #[tokio::test]
16776 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16777 ask_test_home();
16778 let tmp = tempfile::tempdir().unwrap();
16779 let repo = tmp.path().join("repo");
16780 std::fs::create_dir_all(&repo).unwrap();
16781 init_repo(&repo);
16782 let origin = tmp.path().join("origin.git");
16783 let g = |dir: &Path, args: &[&str]| {
16784 let out = std::process::Command::new("git")
16785 .args(args)
16786 .current_dir(dir)
16787 .quiet()
16788 .output()
16789 .expect("spawn git");
16790 assert!(
16791 out.status.success(),
16792 "git {args:?}: {}",
16793 String::from_utf8_lossy(&out.stderr)
16794 );
16795 };
16796 g(
16797 tmp.path(),
16798 &[
16799 "clone",
16800 "--bare",
16801 "-q",
16802 repo.to_str().unwrap(),
16803 origin.to_str().unwrap(),
16804 ],
16805 );
16806 g(
16807 &repo,
16808 &["remote", "add", "origin", origin.to_str().unwrap()],
16809 );
16810 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16811
16812 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16814 g(&repo, &["add", "-A"]);
16815 g(
16816 &repo,
16817 &["commit", "-q", "-m", "chore(deps): update a crate"],
16818 );
16819 g(&repo, &["push", "-q", "origin", "main"]);
16820 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16821 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16822 g(&repo, &["add", "-A"]);
16823 g(
16824 &repo,
16825 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16826 );
16827 g(&repo, &["checkout", "-q", "main"]);
16828
16829 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16830 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16831 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16833 assert_eq!(fallback, c1);
16834
16835 let mut state = review_state(&[
16836 "chore(deps): update a crate",
16837 "fix(daemon): apply a chosen action",
16838 ]);
16839 state.repo = repo.clone();
16840 state.base_branch = "main".to_owned();
16841 state.base_commit = c1;
16842 refresh_reviewed_commits(&mut state, "feat/own").await;
16843 assert_eq!(
16844 state.reviewed_commits,
16845 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16846 );
16847 assert_eq!(
16848 review_title(&state).as_deref(),
16849 Some("fix(daemon): apply a chosen action")
16850 );
16851 assert_eq!(
16852 leaked_subjects(&state, "feat/own").await,
16853 Some(vec!["chore(deps): update a crate".to_owned()])
16854 );
16855 state.config.merge.remote = "nowhere".to_owned();
16858 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16859 }
16860
16861 struct SummaryFixture {
16865 _tmp: tempfile::TempDir,
16866 repo: PathBuf,
16867 reply: PathBuf,
16868 runner: Runner,
16869 }
16870
16871 fn summary_fixture() -> SummaryFixture {
16872 ask_test_home();
16873 let tmp = tempfile::tempdir().unwrap();
16874 let repo = tmp.path().join("repo");
16875 std::fs::create_dir_all(&repo).unwrap();
16876 init_repo(&repo);
16877 let origin = tmp.path().join("origin.git");
16878 let g = |args: &[&str]| {
16879 let out = std::process::Command::new("git")
16880 .args(args)
16881 .current_dir(&repo)
16882 .quiet()
16883 .output()
16884 .expect("spawn git");
16885 assert!(
16886 out.status.success(),
16887 "git {args:?}: {}",
16888 String::from_utf8_lossy(&out.stderr)
16889 );
16890 };
16891 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16892 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16893 g(&["push", "-q", "origin", "main"]);
16894 g(&["checkout", "-q", "-b", "feat/own"]);
16895 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16896 g(&["add", "-A"]);
16897 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16898 let reply = tmp.path().join("reply.txt");
16899 std::fs::write(
16900 &reply,
16901 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16902 )
16903 .unwrap();
16904 let mut runner = runner_at(RunStatus::Gating);
16905 let mut state = review_state(&["feat(own): add the own file"]);
16906 state.repo = repo.clone();
16907 state.base_branch = "main".to_owned();
16908 state.candidates[0].branch = "feat/own".to_owned();
16909 state.candidates[0].worktree = repo.clone();
16910 let mut author = spec("alpha");
16911 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16912 author
16913 .env
16914 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16915 state.config.agents = vec![author];
16916 runner.state = state;
16917 SummaryFixture {
16918 _tmp: tmp,
16919 repo,
16920 reply,
16921 runner,
16922 }
16923 }
16924
16925 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16926 let winner = f.runner.state.candidates[0].clone();
16927 f.runner
16928 .guarded_pr_message(&winner, None, true)
16929 .await
16930 .unwrap()
16931 .expect("an English description never parks")
16932 }
16933
16934 fn clean_round_with_minor_findings() -> ReviewRound {
16935 let mut round = passing_round(2, None);
16936 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16937 round.reviews = vec![ReviewRecord {
16938 attempts: 0,
16939 reviewer: 1,
16940 agent: "alpha".into(),
16941 summary: "内部レビュー".into(),
16942 findings: vec![
16943 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16944 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16945 ],
16946 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16947 failed: None,
16948 duration_ms: 0,
16949 }];
16950 round
16951 }
16952
16953 #[tokio::test]
16954 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16955 for review_only in [false, true] {
16956 let mut f = summary_fixture();
16957 if !review_only {
16958 f.runner.state.candidates[0].agent = "alpha".into();
16959 }
16960 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16961 assert!(f.runner.state.reviews[0].clean);
16962 assert!(f.runner.state.reviews[0].fix.is_none());
16963 assert!(f.runner.state.open_findings().is_empty());
16964 assert_eq!(
16965 crate::pr_outcome::required_finding_ids(&f.runner.state),
16966 ["R2-1-1", "R2-1-2"]
16967 );
16968 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16969 let evidence = crate::pr_outcome::summary_evidence(
16970 &f.runner.state,
16971 crate::pr_outcome::FinalHead::At(&head),
16972 );
16973 assert!(
16974 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16975 );
16976 let omitted = posted(&mut f).await;
16978 assert!(omitted.body.contains("No description"));
16979 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16980 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16981 assert!(f.runner.state.change_summary.is_none());
16982 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16983 let described = posted(&mut f).await;
16984 assert!(described.body.contains("improve documentation for callers"));
16985 assert!(described.body.contains("remaining edge case"));
16986 assert!(described.body.contains("## Open review findings"));
16987 assert!(!described.body.contains("未修正"));
16988 }
16989 }
16990
16991 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16992 let mut f = summary_fixture();
16993 if !review_only {
16994 f.runner.state.candidates[0].agent = "alpha".into();
16995 }
16996 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16997 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16998 fixed
16999 .fix
17000 .as_mut()
17001 .unwrap()
17002 .rejected
17003 .push(crate::verdict::Rejection {
17004 id: "R1-1-1".into(),
17005 why: "呼び出されないコードのため辞退".into(),
17006 });
17007 let mut clean = clean_round_with_minor_findings();
17008 clean.verified_head = Some(head.clone());
17009 f.runner.state.reviews = vec![fixed, clean];
17010 ok_gate(&mut f.runner.state, Some(&head));
17011 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17012 f.runner.state.config.agents.push(scripted("beta", reply));
17013 f.runner.state.config.roles.language_judge =
17014 Some(crate::config::AgentChoice::One("beta".into()));
17015 f
17016 }
17017
17018 #[tokio::test]
17019 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17020 for review_only in [false, true] {
17021 let mut f = rewrite_fixture_with_findings(review_only,
17022 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17023 let m = posted(&mut f).await;
17024 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17025 let expected =
17026 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17027 assert!(m.body.contains(&expected));
17028 assert!(m.body.contains("## Review rounds"));
17029 assert!(
17030 m.body
17031 .contains("Gate (`verify.gate`): passed on the final commit")
17032 );
17033 assert!(
17034 m.body
17035 .contains("Tests (`verify.e2e`): passed on the final commit")
17036 );
17037 let c = f.runner.state.change_summary.as_ref().unwrap();
17038 assert!(
17039 m.body
17040 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17041 );
17042 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17043 assert!(crate::pr_outcome::covers_required_findings(
17044 &f.runner.state,
17045 &m.body
17046 ));
17047 assert!(m.body.contains("documentation remains incomplete"));
17048 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17049 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17050 assert!(f.runner.state.github_text.is_none());
17051 }
17052 }
17053
17054 #[tokio::test]
17055 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17056 for review_only in [false, true] {
17057 for reply in [
17058 "TITLE: fix: public description\nA short English description.",
17059 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17060 ] {
17061 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17062 let winner = f.runner.state.candidates[0].clone();
17063 let posted = f
17064 .runner
17065 .guarded_pr_message(&winner, None, true)
17066 .await
17067 .unwrap();
17068 assert!(posted.is_none());
17069 assert!(f.runner.state.parked);
17070 let withheld = f.runner.state.github_text.as_ref().unwrap();
17071 assert!(
17072 withheld.rewrite.is_none(),
17073 "an incomplete rewrite must not be selectable"
17074 );
17075 let q = ask::Questions::open()
17076 .get(withheld.question.as_ref().unwrap())
17077 .unwrap();
17078 assert!(
17079 !q.choices
17080 .contains(&crate::github_text::USE_REWRITE.to_owned())
17081 );
17082 assert!(
17083 q.detail.contains("R1-1-1")
17084 && q.detail.contains("R2-1-1")
17085 && q.detail.contains("R2-1-2")
17086 );
17087 }
17088 }
17089 }
17090
17091 #[tokio::test]
17092 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17093 for review_only in [false, true] {
17094 let mut f = summary_fixture();
17095 if !review_only {
17096 f.runner.state.candidates[0].agent = "alpha".into();
17097 }
17098 f.runner.state.instruction = "内部タスクの説明".into();
17099 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17100 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17101 f.runner.state.reviews = vec![
17102 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17103 round_with_notes(2, Some("二回目の内部修正報告です。")),
17104 passing_round(3, Some(&head)),
17105 ];
17106 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17107 ok_gate(&mut f.runner.state, Some(&head));
17108 let captured = f.repo.join("public-prompt.txt");
17109 let author = &mut f.runner.state.config.agents[0];
17110 author.command = vec![
17111 "sh".into(),
17112 "-c".into(),
17113 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17114 ];
17115 author.env.insert(
17116 "PROMPT_FILE".into(),
17117 captured.to_string_lossy().into_owned(),
17118 );
17119 let m = posted(&mut f).await;
17120 assert!(m.body.contains("Adds `own.txt`"));
17121 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17122 assert!(m.body.contains("R1-1-1"));
17123 assert!(
17124 m.body
17125 .contains("Gate (`verify.gate`): passed on the final commit")
17126 );
17127 assert!(
17128 m.body
17129 .contains("Tests (`verify.e2e`): passed on the final commit")
17130 );
17131 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17132 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17133 let prompt = std::fs::read_to_string(captured).unwrap();
17134 assert!(prompt.contains(&format!(
17135 "git diff {} {head}",
17136 f.runner.state.change_summary.as_ref().unwrap().base
17137 )));
17138 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17139 assert_eq!(
17140 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17141 "二回目の内部修正報告です。"
17142 );
17143 }
17144 }
17145
17146 #[tokio::test]
17147 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17148 for review_only in [false, true] {
17149 let mut f = summary_fixture();
17150 if !review_only {
17151 f.runner.state.candidates[0].agent = "alpha".into();
17152 }
17153 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17154 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17155 std::fs::write(&f.reply, "no usable description").unwrap();
17156 let failed = posted(&mut f).await;
17157 assert!(
17158 failed
17159 .body
17160 .contains("No description of what this change does could be generated")
17161 );
17162 assert!(failed.body.is_ascii());
17163 assert!(f.runner.state.github_text.is_none());
17164 assert!(f.runner.state.change_summary.is_none());
17165 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17166 let evidence = crate::pr_outcome::summary_evidence(
17167 &f.runner.state,
17168 crate::pr_outcome::FinalHead::At(&head),
17169 );
17170 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17171 assert!(
17172 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17173 .exists()
17174 );
17175 f.runner.state =
17176 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17177 std::fs::write(
17178 &f.reply,
17179 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17180 )
17181 .unwrap();
17182 let retried = posted(&mut f).await;
17183 assert!(retried.body.contains("Adds the committed own file."));
17184 assert_eq!(
17185 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17186 "日本語の修正報告"
17187 );
17188 }
17189 }
17190
17191 #[tokio::test]
17192 async fn same_head_verification_and_findings_invalidate_public_prose() {
17193 let mut f = summary_fixture();
17194 posted(&mut f).await;
17195 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17196 let original = f
17197 .runner
17198 .state
17199 .change_summary
17200 .as_ref()
17201 .unwrap()
17202 .evidence
17203 .clone();
17204 ok_gate(&mut f.runner.state, Some(&head));
17205 std::fs::write(
17206 &f.reply,
17207 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17208 )
17209 .unwrap();
17210 let verified = posted(&mut f).await;
17211 assert!(verified.body.contains("now verified"));
17212 assert_ne!(
17213 f.runner.state.change_summary.as_ref().unwrap().evidence,
17214 original
17215 );
17216 let mut round = round_with_notes(1, Some("内部報告"));
17217 round.reviews.push(ReviewRecord {
17218 attempts: 0,
17219 reviewer: 1,
17220 agent: "alpha".into(),
17221 summary: "内部レビュー".into(),
17222 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17223 vote: None,
17224 failed: None,
17225 duration_ms: 0,
17226 });
17227 round
17228 .fix
17229 .as_mut()
17230 .unwrap()
17231 .rejected
17232 .push(crate::verdict::Rejection {
17233 id: "R1-1-1".into(),
17234 why: "呼び出し元から到達できない".into(),
17235 });
17236 f.runner.state.reviews = vec![round];
17237 let omitted = posted(&mut f).await;
17239 assert!(omitted.body.contains("No description"));
17240 assert!(omitted.body.contains("R1-1-1"));
17241 assert!(!omitted.body.contains("到達"));
17242 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17243 let described = posted(&mut f).await;
17244 assert!(described.body.contains("unreachable from callers"));
17245 assert!(described.body.contains("R1-1-1"));
17246 assert!(!described.body.contains("到達"));
17247 let evidence = f
17248 .runner
17249 .state
17250 .change_summary
17251 .as_ref()
17252 .unwrap()
17253 .evidence
17254 .clone();
17255 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17256 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17257 assert!(posted(&mut f).await.body.contains("rationale updated"));
17258 assert_ne!(
17259 f.runner.state.change_summary.as_ref().unwrap().evidence,
17260 evidence
17261 );
17262 f.runner
17264 .state
17265 .change_summary
17266 .as_mut()
17267 .unwrap()
17268 .evidence
17269 .clear();
17270 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17271 assert!(posted(&mut f).await.body.contains("current specification"));
17272 }
17273
17274 #[tokio::test]
17275 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17276 let mut f = summary_fixture();
17277 let m = posted(&mut f).await;
17278 assert_eq!(m.title, "feat(own): add the own file");
17279 assert!(
17280 m.body
17281 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17282 "{}",
17283 m.body
17284 );
17285 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17286 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17287 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17288 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17289 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17290 let kept = f.runner.state.change_summary.as_ref().unwrap();
17291 assert_eq!(
17292 (kept.head.as_str(), kept.base.as_str()),
17293 (head.as_str(), base.as_str())
17294 );
17295 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17296 }
17297
17298 #[tokio::test]
17299 async fn review_only_summary_receives_the_configured_github_language() {
17300 let mut f = summary_fixture();
17301 f.runner.state.config.graph.github_language = "ja".into();
17302 let captured = f.repo.join("summary-prompt.txt");
17303 let author = &mut f.runner.state.config.agents[0];
17304 author.command = vec![
17305 "sh".into(),
17306 "-c".into(),
17307 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17308 ];
17309 author.env.insert(
17310 "PROMPT_FILE".into(),
17311 captured.to_string_lossy().into_owned(),
17312 );
17313 std::fs::write(
17314 &f.reply,
17315 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17316 )
17317 .unwrap();
17318 let m = posted(&mut f).await;
17319 let prompt = std::fs::read_to_string(captured).unwrap();
17320 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17321 assert!(!prompt.contains("in English"));
17322 assert_eq!(m.title, "feat(own): ファイルを追加");
17323 assert!(m.body.contains("own.txt を追加します。"));
17324 }
17325
17326 #[tokio::test]
17327 async fn review_fixes_come_after_the_whole_change_summary() {
17328 let mut f = summary_fixture();
17329 f.runner.state.reviews = vec![
17330 round_with_notes(1, Some("handled the PATH case")),
17331 round_with_notes(2, None),
17332 ];
17333 let m = posted(&mut f).await;
17334 let summary = m.body.find("## Summary").expect("summary");
17335 let fixes = m.body.find("## Review rounds").expect("review rounds");
17336 assert!(summary < fixes, "{}", m.body);
17337 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17338 assert!(m.body.contains("## Final verification"), "{}", m.body);
17339 }
17340
17341 #[tokio::test]
17342 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17343 let mut f = summary_fixture();
17344 let first = posted(&mut f).await;
17345 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17346 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17348 let again = posted(&mut f).await;
17349 assert_eq!((again.title, again.body), (first.title, first.body));
17350
17351 let g = |args: &[&str]| {
17352 let out = std::process::Command::new("git")
17353 .args(args)
17354 .current_dir(&f.repo)
17355 .quiet()
17356 .output()
17357 .expect("spawn git");
17358 assert!(out.status.success(), "git {args:?}");
17359 };
17360 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17361 g(&["add", "-A"]);
17362 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17363 let second = posted(&mut f).await;
17364 let kept = f.runner.state.change_summary.clone().unwrap();
17365 assert_ne!(kept.head, first_head);
17366 assert!(second.body.contains("Not read yet."), "{}", second.body);
17367 assert!(!second.body.contains("so the first summary is read"));
17368 assert_eq!(second.title, "feat(own): other");
17369 }
17370
17371 #[tokio::test]
17372 async fn an_unavailable_summary_is_stated_and_never_invented() {
17373 let mut f = summary_fixture();
17374 std::fs::write(&f.reply, "no title line here").unwrap();
17375 let m = posted(&mut f).await;
17376 assert!(
17377 m.body
17378 .contains("No description of what this change does could be generated"),
17379 "{}",
17380 m.body
17381 );
17382 assert!(m.body.contains("internal reports remain in local records"));
17383 assert!(f.runner.state.change_summary.is_none());
17384 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17385 let winner = f.runner.state.candidates[0].clone();
17387 let preview = f
17388 .runner
17389 .guarded_pr_message(&winner, None, false)
17390 .await
17391 .unwrap()
17392 .unwrap();
17393 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17394 }
17395
17396 #[test]
17397 fn pr_message_review_single_commit_uses_its_subject() {
17398 let state = review_state(&["feat(nats): per-role user"]);
17399 let m = pr_message(&state, 'A');
17400 assert_eq!(m.title, "feat(nats): per-role user");
17401 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17402 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17403 }
17404
17405 #[test]
17406 fn pr_message_review_multi_commit_takes_the_oldest() {
17407 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17408 let m = pr_message(&state, 'A');
17409 assert_eq!(m.title, "feat: the change");
17410 for s in ["feat: the change", "fix: typo", "fix: again"] {
17411 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17412 }
17413 }
17414
17415 #[test]
17416 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17417 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17418 let mut state = review_state(&[first, "fix: later fixup"]);
17419 state.candidates[0].branch = "機能/ログイン".to_owned();
17420 let m = pr_message(&state, 'A');
17421 assert!(
17422 m.title.starts_with("chore: land candidate A of run"),
17423 "{}",
17424 m.title
17425 );
17426 }
17427 }
17428
17429 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17430 BranchFacts {
17431 commits: commits
17432 .iter()
17433 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17434 .collect(),
17435 stat: stat.to_owned(),
17436 }
17437 }
17438
17439 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17440 let mut r = review_round(true, 0, 1, 1, true, true);
17441 r.round = round;
17442 r.fix = notes.map(|n| FixRecord {
17443 agent: "fixer".to_owned(),
17444 addressed: Vec::new(),
17445 rejected: Vec::new(),
17446 notes: n.to_owned(),
17447 committed: true,
17448 failed: None,
17449 duration_ms: 0,
17450 continuation: None,
17451 });
17452 r
17453 }
17454
17455 #[test]
17456 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17457 let state = review_state(&["ignored"]);
17458 let f = facts(
17459 &[
17460 (
17461 "fix(login): resolve PATH on macOS",
17462 "Login shells skip rc files.",
17463 ),
17464 ("fix: address review", ""),
17465 ],
17466 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17467 );
17468 let m = pr_message_with(&state, 'A', Some(&f));
17469 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17470 assert!(
17471 m.body
17472 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17473 "{}",
17474 m.body
17475 );
17476 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17477 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17478 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17479 for banned in [
17480 "Review the work already",
17481 "no task statement",
17482 "Original task",
17483 ] {
17484 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17485 }
17486 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17487 }
17488
17489 #[test]
17490 fn pr_message_review_truncates_a_huge_first_commit_body() {
17491 let state = review_state(&["ignored"]);
17492 let f = facts(
17493 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17494 "s",
17495 );
17496 let m = pr_message_with(&state, 'A', Some(&f));
17497 assert!(m.body.len() < 30_000, "{}", m.body.len());
17498 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17499 assert!(
17500 m.body.contains("- ... 1 more commit(s)"),
17501 "{}",
17502 m.body.len()
17503 );
17504 assert!(m.body.ends_with("magi:candidate-a\n"));
17505 }
17506
17507 #[test]
17508 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17509 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17510 assert_eq!(m.title, "feat: x");
17511 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17512 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17513 assert!(!m.body.contains("no task statement"), "{}", m.body);
17514 }
17515
17516 #[test]
17517 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17518 let mut state = review_state(&["日本語の件名"]);
17519 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17520 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17521 state.candidates[0].branch = "機能/ログイン".to_owned();
17522 assert!(
17523 pr_message(&state, 'A')
17524 .title
17525 .starts_with("chore: land candidate A")
17526 );
17527 }
17528
17529 #[test]
17530 fn pr_message_review_fixes_survive_a_clean_final_round() {
17531 let mut state = review_state(&["feat: x"]);
17532 state.reviews = vec![
17533 round_with_notes(1, Some("handled the PATH case")),
17534 round_with_notes(2, None),
17535 ];
17536 let body = pr_message(&state, 'A').body;
17537 assert!(
17538 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17539 "{body}"
17540 );
17541 assert!(body.contains("### Round 1"), "{body}");
17542
17543 state.reviews = vec![
17544 round_with_notes(1, Some("first fix")),
17545 round_with_notes(2, Some("")),
17546 round_with_notes(3, Some("second fix")),
17547 round_with_notes(4, None),
17548 ];
17549 let body = pr_message(&state, 'A').body;
17550 assert!(
17551 body.contains("### Round 1") && !body.contains("first fix"),
17552 "{body}"
17553 );
17554 assert!(
17555 body.contains("### Round 3") && !body.contains("second fix"),
17556 "{body}"
17557 );
17558 assert!(body.contains("### Round 2"), "{body}");
17559 }
17560
17561 const FINAL: &str = "abcdef1234567890";
17562
17563 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17564 let mut r = round_with_notes(round, None);
17565 r.verified_head = verified.map(str::to_owned);
17566 r
17567 }
17568
17569 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17570 state.gate = vec![CommandOutcome {
17571 command: "gate".to_owned(),
17572 code: Some(0),
17573 output_tail: String::new(),
17574 duration_ms: 0,
17575 resource_blocked: false,
17576 }];
17577 state.gate_ran = true;
17578 state.gate_head = head.map(str::to_owned);
17579 }
17580
17581 fn body_at(state: &RunState) -> String {
17582 pr_message_raw(
17583 state,
17584 'A',
17585 None,
17586 crate::pr_outcome::FinalHead::At(FINAL),
17587 ChangeNote::NotAsked,
17588 )
17589 .body
17590 }
17591
17592 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17593
17594 #[test]
17595 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17596 let mut state = review_state(&["feat: x"]);
17597 state.reviews = vec![
17598 round_with_notes(1, Some(SANDBOXED)),
17599 passing_round(2, Some(FINAL)),
17600 ];
17601 ok_gate(&mut state, Some(FINAL));
17602 let body = body_at(&state);
17603 assert!(!body.contains("Handled the PATH case."), "{body}");
17604 assert!(!body.contains("could not commit"), "{body}");
17605 assert!(!body.contains("could not run"), "{body}");
17606 assert!(
17607 body.contains("The fix is committed on the branch"),
17608 "{body}"
17609 );
17610 assert!(!body.contains("sandbox"), "{body}");
17611 assert!(
17612 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17613 "{body}"
17614 );
17615 assert!(
17616 body.contains("Gate (`verify.gate`): passed on the final commit"),
17617 "{body}"
17618 );
17619 assert!(
17621 state.reviews[0]
17622 .fix
17623 .as_ref()
17624 .unwrap()
17625 .notes
17626 .contains("could not commit")
17627 );
17628 }
17629
17630 #[test]
17631 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17632 let mut state = review_state(&["feat: x"]);
17633 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17634 r1.fix.as_mut().unwrap().committed = false;
17635 let mut r2 = passing_round(2, Some(FINAL));
17636 r2.e2e[0].code = Some(1);
17637 state.reviews = vec![r1, r2];
17638 let body = body_at(&state);
17639 assert!(!body.contains("could not commit"), "{body}");
17640 assert!(!body.contains("could not run"), "{body}");
17641 assert!(body.contains("This round produced no commit"), "{body}");
17642 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17643 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17644 assert!(!body.contains("passed"), "{body}");
17645 }
17646
17647 #[test]
17648 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17649 let mut state = review_state(&["feat: x"]);
17650 state.reviews = vec![
17651 round_with_notes(1, Some(SANDBOXED)),
17652 passing_round(2, Some("0123456789abcdef")),
17653 ];
17654 ok_gate(&mut state, Some("0123456789abcdef"));
17655 let body = body_at(&state);
17656 assert!(!body.contains("HTTP tests could not run"), "{body}");
17657 assert!(!body.contains("could not commit"), "{body}");
17659 assert!(
17660 body.contains("passed on an earlier commit (`0123456`)"),
17661 "{body}"
17662 );
17663 assert!(!body.contains("passed on the final commit"), "{body}");
17664
17665 ok_gate(&mut state, None);
17667 let body = body_at(&state);
17668 assert!(
17669 body.contains("passed on a commit that was not recorded"),
17670 "{body}"
17671 );
17672 }
17673
17674 #[test]
17675 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17676 let state = state_with_summary(
17677 "add retries to the client",
17678 "TITLE: feat: retries\n\nDid it.",
17679 );
17680 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17681 assert_eq!(m.title, "feat: retries");
17682 assert!(
17683 m.body.contains("<summary>Original task</summary>"),
17684 "{}",
17685 m.body
17686 );
17687 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17688 assert!(
17689 m.body
17690 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17691 "{}",
17692 m.body
17693 );
17694 }
17695
17696 #[test]
17697 fn pr_message_review_bounds_a_long_english_subject() {
17698 let long = format!("feat: {}", "word ".repeat(100));
17699 let m = pr_message(&review_state(&[&long]), 'A');
17700 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17701 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17702 }
17703
17704 #[test]
17705 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17706 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17707 let m = pr_message(&state, 'A');
17708 assert_eq!(m.title, "add retries");
17709 assert!(m.body.contains("<summary>Original task</summary>"));
17710 assert!(!m.body.contains("Commits under review"));
17711 assert_eq!(landing_subject_source(&state), state.instruction);
17712 }
17713
17714 #[test]
17715 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17716 let state = review_state(&["feat: the change", "fix: typo"]);
17717 let source = landing_subject_source(&state);
17718 assert_eq!(land::merge_subject("", &source), "feat: the change");
17719 assert_eq!(
17720 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17721 "feat: the change"
17722 );
17723 assert_eq!(
17725 land::merge_subject("feat: renamed by hand", &source),
17726 "feat: renamed by hand"
17727 );
17728 let blank = review_state(&["日本語"]);
17729 assert!(
17730 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17731 );
17732 }
17733
17734 #[test]
17735 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17736 let state = review_state(&["feat: the change"]);
17737 let source = landing_subject_source(&state);
17738 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17739 assert_eq!(
17740 land::merge_subject(landing_title(&state, old), &source),
17741 "feat: the change"
17742 );
17743 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17744 let task = state_with_summary("add retries", "");
17745 assert_eq!(landing_title(&task, old), old);
17746 }
17747
17748 #[test]
17749 fn pr_message_describes_the_change_not_the_task() {
17750 let state = state_with_summary(
17751 "今回やってほしいこと: results projector を直す",
17752 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17753 );
17754 let m = pr_message(&state, 'A');
17755 assert_eq!(m.title, "fix(web): batch the runs list reads");
17756 assert!(
17757 m.body.starts_with("## Summary\n\n- reads run.json once"),
17758 "{}",
17759 m.body
17760 );
17761 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17762 let task_at = m.body.find("今回やってほしいこと").unwrap();
17763 let details_at = m.body.find("<details>").unwrap();
17764 assert!(
17765 details_at < task_at,
17766 "the task lives inside <details>: {}",
17767 m.body
17768 );
17769 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17770 assert!(m.body.contains("magi:candidate-a"));
17771 }
17772
17773 #[test]
17774 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17775 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17776 let m = pr_message(&state, 'A');
17777 assert_eq!(m.title, "add retries");
17778 assert!(
17779 m.body.contains("## Summary\n\n- did some things"),
17780 "{}",
17781 m.body
17782 );
17783
17784 let none = RunState::new(
17785 PathBuf::from("/repo"),
17786 "main".to_owned(),
17787 "abc1234".to_owned(),
17788 "add retries".to_owned(),
17789 Config::default(),
17790 );
17791 let m = pr_message(&none, 'A');
17792 assert_eq!(m.title, "add retries");
17793 assert!(!m.body.contains("## Summary"), "{}", m.body);
17794 }
17795
17796 #[test]
17797 fn pr_message_refuses_the_candidate_commit_subject() {
17798 for bad in [
17799 "TITLE: magi: candidate A (uncommitted work)",
17800 "TITLE: chore: stuff (uncommitted work)",
17801 "TITLE: ",
17802 ] {
17803 let state = state_with_summary("add retries", bad);
17804 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17805 }
17806 }
17807
17808 #[test]
17809 fn pr_message_bounds_a_very_long_task_and_title() {
17810 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17811 let state = state_with_summary(&long, "- nothing");
17812 let m = pr_message(&state, 'A');
17813 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17814 assert!(!m.title.contains('\n'));
17815
17816 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17817 let m = pr_message(&state, 'A');
17818 assert!(m.title.starts_with("feat: "));
17819 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17820 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17821 }
17822
17823 fn long_title_of(instruction: &str) -> String {
17824 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17825 }
17826
17827 #[test]
17828 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17829 let first = "Make the landing path keep a readable title for long tasks";
17830 let line = format!(
17831 "{first}. {}",
17832 "And then keep going with more words ".repeat(20)
17833 );
17834 let t = long_title_of(&line);
17835 assert_eq!(t, first);
17836 assert!(!t.starts_with("chore: land"));
17837 }
17838
17839 #[test]
17840 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17841 let line = "word ".repeat(200);
17842 let t = long_title_of(&line);
17843 assert!(t.ends_with("word..."), "{t}");
17844 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17845 }
17846
17847 #[test]
17848 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17849 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17850 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17851 }
17852 }
17853
17854 #[test]
17855 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17856 let mut state = state_with_summary("再試行を追加する", "- nothing");
17857 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17858 state.config.graph.github_language = "ja".into();
17859 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17860 }
17861
17862 #[test]
17863 fn pr_message_title_limit_is_exact() {
17864 let at = "a".repeat(PR_TITLE_MAX);
17865 assert_eq!(long_title_of(&at), at);
17866 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17867 assert!(over.ends_with("..."), "{over}");
17868 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17869 }
17870
17871 #[test]
17872 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17873 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17874 let t = long_title_of(&line);
17875 assert!(t.ends_with("..."), "{t}");
17876 assert!(t.is_ascii(), "{t}");
17877 }
17878
17879 #[test]
17880 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17881 let line = format!(
17882 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17883 "plus more filler words ".repeat(20)
17884 );
17885 let t = long_title_of(&line);
17886 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17887 }
17888
17889 #[test]
17890 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17891 let line = format!("\n\n# {}", "title words ".repeat(40));
17892 let state = state_with_summary(&line, "- nothing");
17893 let m = pr_message(&state, 'A');
17894 assert!(m.title.starts_with("title words"), "{}", m.title);
17895 assert_eq!(
17896 land::merge_subject(&m.title, &landing_subject_source(&state)),
17897 m.title
17898 );
17899 assert_eq!(
17901 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17902 "feat: renamed by hand"
17903 );
17904 }
17905
17906 #[tokio::test]
17907 async fn github_text_rewrite_is_bounded_and_falls_back() {
17908 crate::run::pin_test_home();
17909 for (reply, accepted) in [
17910 (
17911 "TITLE: fix: retries\nAdd retries for failed requests.",
17912 true,
17913 ),
17914 (
17915 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17916 false,
17917 ),
17918 (
17919 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17920 true,
17921 ),
17922 (
17923 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17924 true,
17925 ),
17926 (
17927 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17928 true,
17929 ),
17930 (
17931 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17932 false,
17933 ),
17934 ("TITLE: fix: retries\n", false),
17935 ] {
17936 let dir = tempfile::tempdir().unwrap();
17937 let mut runner = runner_at(RunStatus::Gating);
17938 runner.state = state_with_summary(
17939 "add retries",
17940 "TITLE: fix: retries\n日本語の説明を書きます。",
17941 );
17942 runner.state.repo = dir.path().to_owned();
17943 runner.state.candidates[0].worktree = dir.path().to_owned();
17944 let mut author = spec("alpha");
17945 author.command = vec![
17946 "sh".into(),
17947 "-c".into(),
17948 "printf '%s' \"$REWRITE_REPLY\"".into(),
17949 ];
17950 author.env.insert("REWRITE_REPLY".into(), reply.into());
17951 runner.state.config.agents = vec![author];
17952 seed_public_summary(&mut runner);
17953 let winner = runner.state.candidates[0].clone();
17954 let first = runner
17955 .guarded_pr_message(&winner, None, true)
17956 .await
17957 .unwrap();
17958 let message = if accepted {
17959 first.expect("a passing rewrite never asks")
17960 } else {
17961 assert!(first.is_none());
17963 assert!(runner.state.parked);
17964 let store = ask::Questions::open();
17965 let asked: Vec<_> = store
17966 .list()
17967 .into_iter()
17968 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17969 .collect();
17970 assert_eq!(asked.len(), 1);
17971 assert!(!asked[0].detail.contains("secret"));
17972 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17973 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17974 assert_eq!(
17975 asked[0]
17976 .choices
17977 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17978 rewrite.is_some()
17979 );
17980 store
17983 .update(&asked[0].id, |q| {
17984 q.abandon("test");
17985 Ok(())
17986 })
17987 .unwrap();
17988 let again = runner
17989 .guarded_pr_message(&winner, None, true)
17990 .await
17991 .unwrap()
17992 .expect("an abandoned question falls back");
17993 let n = store
17994 .list()
17995 .into_iter()
17996 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17997 .count();
17998 assert_eq!(n, 1, "asked once per rejected text");
17999 again
18000 };
18001 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18002 assert!(!message.body.contains("secret"));
18003 assert!(!message.body.contains("prefix"));
18004 assert!(!message.title.contains("secret"));
18005 assert_eq!(
18006 message.body.contains("Add retries for failed requests."),
18007 accepted
18008 );
18009 assert_eq!(
18010 runner.state.seats["github-text-rewrite"].turns, 1,
18011 "only one rewrite invocation"
18012 );
18013 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18014 if accepted {
18015 assert!(runner.state.github_text.is_none());
18016 assert!(!runner.state.parked);
18017 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18018 assert!(message.body.contains("## Final verification"));
18019 assert!(message.body.contains("Evidence: final diff"));
18020 assert!(
18021 message
18022 .body
18023 .contains(&format!("magi:run/{}", runner.state.id))
18024 );
18025 }
18026 if !accepted {
18027 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18028 assert!(
18029 message
18030 .body
18031 .contains(&format!("magi:run/{}", runner.state.id))
18032 );
18033 }
18034 }
18035 }
18036
18037 fn scripted(id: &str, reply: &str) -> AgentSpec {
18038 let mut a = spec(id);
18039 a.command = vec![
18040 "sh".into(),
18041 "-c".into(),
18042 "printf '%s' \"$REWRITE_REPLY\"".into(),
18043 ];
18044 a.env.insert("REWRITE_REPLY".into(), reply.into());
18045 a
18046 }
18047
18048 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18049 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18050
18051 fn seed_public_summary(runner: &mut Runner) {
18053 let repo = &runner.state.repo;
18054 let g = |args: &[&str]| {
18055 let out = std::process::Command::new("git")
18056 .args(args)
18057 .current_dir(repo)
18058 .quiet()
18059 .output()
18060 .unwrap();
18061 assert!(out.status.success(), "git {args:?}");
18062 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18063 };
18064 g(&["init", "-q", "-b", "main"]);
18065 g(&[
18066 "-c",
18067 "user.name=Test",
18068 "-c",
18069 "user.email=test@example.invalid",
18070 "commit",
18071 "-q",
18072 "--allow-empty",
18073 "-m",
18074 "feat: fixture",
18075 ]);
18076 let head = g(&["rev-parse", "HEAD"]);
18077 g(&["update-ref", "refs/remotes/origin/main", &head]);
18078 let evidence = crate::pr_outcome::summary_evidence(
18079 &runner.state,
18080 crate::pr_outcome::FinalHead::At(&head),
18081 );
18082 let generated = &runner.state.candidates[0].summary;
18083 runner.state.change_summary = Some(crate::run::ChangeSummary {
18084 base: head.clone(),
18085 head,
18086 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18087 title: summary_title(generated).unwrap(),
18088 body: summary_without_title(generated),
18089 });
18090 }
18091
18092 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18093 let mut runner = runner_at(RunStatus::Gating);
18094 runner.state = state_with_summary("add retries", JA_SUMMARY);
18095 if review_only {
18096 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18097 }
18098 runner.state.repo = dir.to_owned();
18099 runner.state.candidates[0].worktree = dir.to_owned();
18100 runner.state.config.agents = agents;
18101 seed_public_summary(&mut runner);
18102 runner
18103 }
18104
18105 fn gate_questions(runner: &Runner) -> usize {
18106 ask::Questions::open()
18107 .list()
18108 .into_iter()
18109 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18110 .count()
18111 }
18112
18113 #[tokio::test]
18114 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18115 crate::run::pin_test_home();
18116 let dir = tempfile::tempdir().unwrap();
18117 let mut runner = rewrite_runner(
18118 dir.path(),
18119 true,
18120 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18121 );
18122 runner.state.config.roles.language_judge =
18123 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18124 let winner = runner.state.candidates[0].clone();
18125 let msg = runner
18126 .guarded_pr_message(&winner, None, true)
18127 .await
18128 .unwrap()
18129 .expect("an English rewrite posts without asking");
18130 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18131 assert!(msg.body.contains("<details>kept</details>"));
18132 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18133 assert_eq!(gate_questions(&runner), 0);
18134 assert!(runner.state.github_text.is_none());
18135 assert!(!runner.state.seats.contains_key("impl-A"));
18136 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18137 }
18138
18139 #[tokio::test]
18140 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18141 crate::run::pin_test_home();
18142 let dir = tempfile::tempdir().unwrap();
18143 let mut runner = rewrite_runner(
18144 dir.path(),
18145 false,
18146 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18147 );
18148 runner.state.config.roles.language_judge =
18149 Some(crate::config::AgentChoice::One("beta".into()));
18150 runner.state.seats.insert(
18151 "impl-A".into(),
18152 SeatState::new("impl-A", "alpha", runner.state.seed),
18153 );
18154 let winner = runner.state.candidates[0].clone();
18155 let msg = runner
18156 .guarded_pr_message(&winner, None, true)
18157 .await
18158 .unwrap()
18159 .expect("rewritten");
18160 assert!(msg.body.contains("<details>kept</details>"));
18161 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18162 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18163 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18164 }
18165
18166 #[tokio::test]
18167 async fn rewrite_reuses_a_resumable_implementer_session() {
18168 crate::run::pin_test_home();
18169 let dir = tempfile::tempdir().unwrap();
18170 let mut runner = rewrite_runner(
18171 dir.path(),
18172 false,
18173 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18174 );
18175 runner.state.config.roles.language_judge =
18176 Some(crate::config::AgentChoice::One("beta".into()));
18177 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18178 seat.turns = 1;
18179 runner.state.seats.insert("impl-A".into(), seat);
18180 let winner = runner.state.candidates[0].clone();
18181 runner
18182 .guarded_pr_message(&winner, None, true)
18183 .await
18184 .unwrap()
18185 .expect("rewritten");
18186 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18187 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18188 }
18189
18190 #[tokio::test]
18191 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18192 crate::run::pin_test_home();
18193 let dir = tempfile::tempdir().unwrap();
18194 let mut broken = spec("bad");
18195 broken.command = vec!["false".into()];
18196 let mut runner = rewrite_runner(
18197 dir.path(),
18198 true,
18199 vec![
18200 broken,
18201 scripted("noformat", "Add retries."),
18202 scripted("good", EN_REWRITE),
18203 ],
18204 );
18205 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18206 "bad".into(),
18207 "noformat".into(),
18208 "good".into(),
18209 ]));
18210 let winner = runner.state.candidates[0].clone();
18211 let msg = runner
18212 .guarded_pr_message(&winner, None, true)
18213 .await
18214 .unwrap()
18215 .expect("the third agent rewrites");
18216 assert!(msg.body.contains("`src/net.rs`"));
18217 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18218 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18219 }
18220
18221 #[tokio::test]
18222 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18223 crate::run::pin_test_home();
18224 for (agents, role, saved) in [
18226 (
18227 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18228 Some("beta"),
18229 true,
18230 ),
18231 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18232 (Vec::new(), None, false),
18233 ] {
18234 let dir = tempfile::tempdir().unwrap();
18235 let mut runner = rewrite_runner(dir.path(), true, agents);
18236 runner.state.config.roles.language_judge =
18237 role.map(|r| crate::config::AgentChoice::One(r.into()));
18238 let winner = runner.state.candidates[0].clone();
18239 let first = runner
18240 .guarded_pr_message(&winner, None, true)
18241 .await
18242 .unwrap();
18243 assert!(first.is_none(), "withheld text parks the run");
18244 assert!(runner.state.parked);
18245 assert_eq!(gate_questions(&runner), 1);
18246 let g = runner.state.github_text.as_ref().unwrap();
18247 assert_eq!(g.rewrite.is_some(), saved);
18248 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18249 }
18250 }
18251
18252 #[tokio::test]
18253 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18254 crate::run::pin_test_home();
18255 let dir = tempfile::tempdir().unwrap();
18256 let mut runner = runner_at(RunStatus::Gating);
18257 runner.state =
18258 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18259 runner.state.repo = dir.path().to_owned();
18260 runner.state.candidates[0].worktree = dir.path().to_owned();
18261 let mut author = spec("alpha");
18262 author.command = vec![
18263 "sh".into(),
18264 "-c".into(),
18265 "printf '%s' \"$REWRITE_REPLY\"".into(),
18266 ];
18267 author.env.insert(
18268 "REWRITE_REPLY".into(),
18269 format!(
18270 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18271 "English sentence. ".repeat(600)
18272 ),
18273 );
18274 runner.state.config.agents = vec![author];
18275 seed_public_summary(&mut runner);
18276 let winner = runner.state.candidates[0].clone();
18277 assert!(
18278 runner
18279 .guarded_pr_message(&winner, None, true)
18280 .await
18281 .unwrap()
18282 .is_none()
18283 );
18284 let record = runner.state.github_text.clone().unwrap();
18285 let rewrite = record.rewrite.clone().unwrap();
18286 assert_eq!(rewrite.failed, ["body-language"]);
18287 assert!(
18288 rewrite
18289 .body
18290 .ends_with(&format!("magi:run/{}", runner.state.id))
18291 );
18292 let store = ask::Questions::open();
18293 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18294 assert!(
18295 q.choices
18296 .contains(&crate::github_text::USE_REWRITE.to_owned())
18297 );
18298 for label in [
18299 "ORIGINAL",
18300 "FALLBACK",
18301 "REWRITE",
18302 "Rules still failed: body-language",
18303 "truncated",
18304 ] {
18305 assert!(q.detail.contains(label), "{label}");
18306 }
18307 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18308 &runner.state,
18309 &crate::github_text::artifact_name(&record.fingerprint),
18310 ))
18311 .unwrap();
18312 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18313 runner.state =
18315 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18316 store
18317 .update(&q.id, |q| {
18318 q.say("まだ日本語")?;
18319 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18320 })
18321 .unwrap();
18322 assert!(
18323 runner
18324 .guarded_pr_message(&winner, None, true)
18325 .await
18326 .unwrap()
18327 .is_none()
18328 );
18329 let retry = runner.state.github_text.as_ref().unwrap();
18330 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18331 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18332 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18333 assert!(
18334 q.posting_comparison
18335 .as_ref()
18336 .unwrap()
18337 .matches_detail(&q.detail)
18338 );
18339 store
18340 .update(&q.id, |q| {
18341 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18342 })
18343 .unwrap();
18344 let message = runner
18345 .guarded_pr_message(&winner, None, true)
18346 .await
18347 .unwrap()
18348 .unwrap();
18349 assert_eq!(message.title, rewrite.title);
18350 assert_eq!(message.body, rewrite.body);
18351 assert_eq!(
18352 runner.state.github_text.as_ref().unwrap().choice,
18353 Some(GithubTextChoice::Rewrite)
18354 );
18355 let again = runner
18356 .guarded_pr_message(&winner, None, true)
18357 .await
18358 .unwrap()
18359 .unwrap();
18360 assert_eq!(again.title, message.title);
18361 assert_eq!(again.body, message.body);
18362 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18363 assert!(
18364 !runner
18365 .state
18366 .events
18367 .iter()
18368 .any(|e| e.message.contains(&rewrite.title)
18369 || e.message.contains("English sentence."))
18370 );
18371 }
18372
18373 #[tokio::test]
18374 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18375 crate::run::pin_test_home();
18376 let dir = tempfile::tempdir().unwrap();
18377 let mut runner = runner_at(RunStatus::Gating);
18378 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18379 runner.state.repo = dir.path().to_owned();
18380 runner.state.candidates[0].worktree = dir.path().to_owned();
18381 seed_public_summary(&mut runner);
18382 let winner = runner.state.candidates[0].clone();
18383 assert!(
18384 runner
18385 .guarded_pr_message(&winner, None, true)
18386 .await
18387 .unwrap()
18388 .is_none()
18389 );
18390 let record = runner.state.github_text.as_ref().unwrap();
18391 assert!(record.rewrite.is_none());
18392 let mut old = serde_json::to_value(record).unwrap();
18394 old.as_object_mut().unwrap().remove("rewrite");
18395 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18396 assert!(old.rewrite.is_none());
18397 let id = record.question.clone().unwrap();
18398 ask::Questions::open()
18399 .update(&id, |q| {
18400 q.choices.push(crate::github_text::USE_REWRITE.into());
18401 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18402 })
18403 .unwrap();
18404 let message = runner
18405 .guarded_pr_message(&winner, None, true)
18406 .await
18407 .unwrap()
18408 .unwrap();
18409 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18410 assert_eq!(
18411 runner.state.github_text.as_ref().unwrap().choice,
18412 Some(GithubTextChoice::Fallback)
18413 );
18414 }
18415
18416 #[tokio::test]
18417 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18418 crate::run::pin_test_home();
18419 let dir = tempfile::tempdir().unwrap();
18420 let mut runner = runner_at(RunStatus::Gating);
18421 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18422 runner.state.repo = dir.path().to_owned();
18423 runner.state.candidates[0].worktree = dir.path().to_owned();
18424 seed_public_summary(&mut runner);
18425 let winner = runner.state.candidates[0].clone();
18426 assert!(
18427 runner
18428 .guarded_pr_message(&winner, None, true)
18429 .await
18430 .unwrap()
18431 .is_none()
18432 );
18433 let store = ask::Questions::open();
18434 let find = |run: &str| {
18435 store
18436 .list()
18437 .into_iter()
18438 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18439 .unwrap()
18440 };
18441 let run = runner.state.id.clone();
18442 let q = find(&run);
18443 let fp = runner
18444 .state
18445 .github_text
18446 .as_ref()
18447 .unwrap()
18448 .fingerprint
18449 .clone();
18450 let name = crate::github_text::artifact_name(&fp);
18451 let path = crate::run::artifact_path(&runner.state, &name);
18452 assert!(q.detail.contains(&path.display().to_string()));
18453 assert!(q.detail.contains("title-language: title"));
18454 let saved = std::fs::read_to_string(&path).expect("artifact written");
18455 assert!(saved.contains("日本語のタイトル"));
18456 assert!(q.cwd.is_none());
18457 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18458 let say = |id: &str, text: &str, choice: &str| {
18460 store
18461 .update(id, |q| {
18462 q.thread.push(ask::Turn {
18463 who: ask::Who::Operator,
18464 body: text.to_owned(),
18465 at: jiff::Timestamp::now(),
18466 note: None,
18467 });
18468 q.answer(ask::Answer::Choice(choice.to_owned()))
18469 })
18470 .unwrap();
18471 };
18472 say(&q.id, "まだ日本語", "use my text");
18473 assert!(
18474 runner
18475 .guarded_pr_message(&winner, None, true)
18476 .await
18477 .unwrap()
18478 .is_none()
18479 );
18480 let second = find(&run);
18481 assert_ne!(second.id, q.id);
18482 assert!(second.detail.contains("did not pass"));
18483 assert!(second.detail.contains("日本語のタイトル"));
18486 assert!(!second.detail.contains("まだ日本語"));
18487 assert!(second.detail.contains(&path.display().to_string()));
18488 say(&second.id, "fix: retry failed requests", "use my text");
18489 let message = runner
18490 .guarded_pr_message(&winner, None, true)
18491 .await
18492 .unwrap()
18493 .expect("a vetted replacement is posted");
18494 assert_eq!(message.title, "fix: retry failed requests");
18495 assert_eq!(
18496 runner
18497 .state
18498 .github_text
18499 .as_ref()
18500 .unwrap()
18501 .chosen_title
18502 .as_deref(),
18503 Some("fix: retry failed requests")
18504 );
18505 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18506 let again = runner
18508 .guarded_pr_message(&winner, None, true)
18509 .await
18510 .unwrap()
18511 .unwrap();
18512 assert_eq!(again.title, message.title);
18513 assert_eq!(
18514 store
18515 .list()
18516 .into_iter()
18517 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18518 .count(),
18519 2
18520 );
18521 }
18522
18523 #[test]
18524 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18525 let mut state = state_with_summary(
18529 "add retries",
18530 "TITLE: fix(web): batch reads\n- reads run.json once",
18531 );
18532 state.config.graph.language = "ja".to_owned();
18533 let m = pr_message(&state, 'A');
18534 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18535 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18536
18537 let task = "今回やってほしいこと: results projector を直す";
18540 let mut state = state_with_summary(task, "- no title line");
18541 state.config.graph.language = "ja".to_owned();
18542 let m = pr_message(&state, 'A');
18543 assert_eq!(
18544 m.title,
18545 format!("chore: land candidate A of run {}", state.id)
18546 );
18547 assert!(
18548 m.body.contains(&format!(
18549 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18550 )),
18551 "{}",
18552 m.body
18553 );
18554 }
18555
18556 #[test]
18557 fn pr_message_scrubs_home_paths_and_addresses() {
18558 let state = state_with_summary(
18559 "fix it in /Users/someone/src/x",
18560 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18561 );
18562 let m = pr_message(&state, 'A');
18563 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18564 assert!(!m.body.contains(leak), "{}", m.body);
18565 }
18566 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18567 }
18568
18569 #[test]
18570 fn pr_message_survives_a_task_that_closes_details() {
18571 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18572 let m = pr_message(&state, 'A');
18573 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18574 }
18575
18576 #[test]
18577 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18578 let cmd = manual_merge_command(
18579 MergeStyle::Squash,
18580 Path::new("/repo"),
18581 "b",
18582 "fix: \"quoted\" $(x) `y`\n\nbody",
18583 );
18584 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18585 }
18586
18587 #[test]
18588 fn manual_merge_command_matches_the_configured_style() {
18589 let repo = Path::new("/repo");
18590 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18591
18592 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18593 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18594
18595 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18596 assert_eq!(
18597 squash,
18598 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18599 \"Merge magi run 0832 (candidate A)\""
18600 );
18601
18602 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18603 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18604 }
18605
18606 #[test]
18607 fn a_nudge_gets_a_quarter_of_the_budget() {
18608 assert_eq!(retry_budget(secs(1200), true), secs(300));
18610 assert_eq!(retry_budget(secs(3600), true), secs(900));
18611 }
18612
18613 #[test]
18614 fn a_resent_prompt_keeps_the_whole_budget() {
18615 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18618 assert_eq!(retry_budget(secs(60), false), secs(60));
18619 }
18620
18621 #[test]
18622 fn the_floor_never_exceeds_the_original_budget() {
18623 assert_eq!(retry_budget(secs(60), true), secs(60));
18627 assert_eq!(retry_budget(secs(480), true), secs(120));
18628 assert_eq!(retry_budget(secs(0), true), secs(0));
18629 }
18630
18631 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18632 agent::CommandEvidence {
18633 id: "item1".to_owned(),
18634 description: "cargo test".to_owned(),
18635 exit_code,
18636 result_summary: String::new(),
18637 source: "codex".to_owned(),
18638 }
18639 }
18640
18641 #[test]
18642 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18643 assert!(!has_unconfirmed_command(&[]));
18647 }
18648
18649 #[test]
18650 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18651 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18655 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18656 assert!(!has_unconfirmed_command(&[
18657 evidence(Some(0)),
18658 evidence(Some(101))
18659 ]));
18660 }
18661
18662 #[test]
18663 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18664 assert!(has_unconfirmed_command(&[
18665 evidence(Some(0)),
18666 evidence(None)
18667 ]));
18668 }
18669
18670 #[test]
18671 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18672 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18673 assert_eq!(
18674 verified_noop_claim(true, &[], text).as_deref(),
18675 Some("already fixed by b32cfc4, on main.")
18676 );
18677 }
18678
18679 #[test]
18680 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18681 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18684 assert!(verified_noop_claim(false, &[], text).is_none());
18685 }
18686
18687 #[test]
18688 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18689 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18690 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18691 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18693 }
18694
18695 #[test]
18696 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18697 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18698 }
18699
18700 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18703 runner.state.candidates = shape
18704 .iter()
18705 .enumerate()
18706 .map(|(i, &(empty, verified))| Candidate {
18707 index: i,
18708 label: (b'A' + i as u8) as char,
18709 agent: "sonnet".to_owned(),
18710 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18711 worktree: PathBuf::from(format!("/wt/{i}")),
18712 summary: String::new(),
18713 stat: String::new(),
18714 files: 0,
18715 commits: 0,
18716 empty,
18717 failed: None,
18718 verified_noop: verified.map(str::to_owned),
18719 duration_ms: 0,
18720 folded: false,
18721 })
18722 .collect();
18723 }
18724
18725 #[test]
18726 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18727 ask_test_home();
18728 let mut runner = runner_at(RunStatus::Implementing);
18729 set_candidates(
18730 &mut runner,
18731 &[
18732 (true, Some("already on main at b32cfc4")),
18733 (true, Some("same fix, see the existing test")),
18734 ],
18735 );
18736
18737 runner
18738 .after_implement()
18739 .expect("a verified no-op is not an error");
18740
18741 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18742 }
18743
18744 #[test]
18745 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18746 ask_test_home();
18747 let mut runner = runner_at(RunStatus::Implementing);
18748 set_candidates(
18752 &mut runner,
18753 &[(true, Some("already on main at b32cfc4")), (true, None)],
18754 );
18755
18756 let err = runner
18757 .after_implement()
18758 .expect_err("an unverified empty candidate must still fail the run");
18759
18760 assert!(
18761 err.to_string().contains("no candidate produced a change"),
18762 "{err}"
18763 );
18764 assert_eq!(runner.state.status, RunStatus::Failed);
18765 }
18766
18767 #[test]
18768 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18769 ask_test_home();
18770 let mut runner = runner_at(RunStatus::Implementing);
18771 set_candidates(&mut runner, &[(true, None), (true, None)]);
18772
18773 let err = runner
18774 .after_implement()
18775 .expect_err("no candidate declared anything; this is an ordinary failure");
18776
18777 assert!(
18778 err.to_string().contains("no candidate produced a change"),
18779 "{err}"
18780 );
18781 assert_eq!(runner.state.status, RunStatus::Failed);
18782 }
18783}
18784
18785#[derive(Debug)]
18788struct GraftRestoreFailed(String);
18789
18790impl std::fmt::Display for GraftRestoreFailed {
18791 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18792 write!(
18793 f,
18794 "could not restore the winner to its pre-graft commit: {}",
18795 self.0
18796 )
18797 }
18798}
18799
18800impl std::error::Error for GraftRestoreFailed {}
18801
18802#[cfg(test)]
18806mod resource_wait_tests {
18807 use super::*;
18808 use std::collections::BTreeMap;
18809
18810 fn sh(id: &str, script: &str) -> AgentSpec {
18811 AgentSpec {
18812 id: id.to_owned(),
18813 kind: crate::config::AgentKind::Command,
18814 model: None,
18815 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18816 extra_args: Vec::new(),
18817 env: BTreeMap::new(),
18818 prompt_delivery: None,
18819 }
18820 }
18821
18822 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18823 SeatJob {
18824 seat: SeatState::new("impl-A", &spec.id, 1),
18825 spec,
18826 cwd: dir.to_path_buf(),
18827 prompt: "do it".to_owned(),
18828 timeout,
18829 allow_write: true,
18830 sessions: false,
18831 artifacts: dir.join("art"),
18832 stem: "impl-A".to_owned(),
18833 handover: None,
18834 }
18835 }
18836
18837 fn state(resource_wait_secs: u64) -> RunState {
18838 let mut config = Config::default();
18839 config.graph.timeout_resource_wait = resource_wait_secs;
18840 RunState::new(
18841 PathBuf::from("/repo"),
18842 "main".to_owned(),
18843 "abc1234".to_owned(),
18844 "task".to_owned(),
18845 config,
18846 )
18847 }
18848
18849 fn hold(cache: &Path) -> crate::cache::Guard {
18850 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18851 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18852 crate::cache::AcquireOutcome::Acquired(g) => g,
18853 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18854 }
18855 }
18856
18857 async fn run(
18858 jobs: Vec<SeatJob>,
18859 sem: Arc<Semaphore>,
18860 cache: Option<&Path>,
18861 st: &mut RunState,
18862 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18863 let prompts = Prompts::default();
18864 let ctx = WaveCtx {
18865 run: "r",
18866 node: "implement",
18867 prompts: &prompts,
18868 cache,
18869 round: None,
18870 carry_seats: false,
18871 pauses: None,
18872 };
18873 wave(jobs, sem, &ctx, st, 0).await
18874 }
18875
18876 #[tokio::test]
18877 async fn verification_cache_wait_preserves_the_command_budget() {
18878 crate::run::pin_test_home();
18879 let tmp = tempfile::tempdir().unwrap();
18880 let cache = tmp.path().join("target");
18881 let guard = hold(&cache);
18882 let release = tokio::spawn(async move {
18883 tokio::time::sleep(Duration::from_millis(100)).await;
18884 guard.release();
18885 });
18886 let mut st = state(10);
18887 let budget = Duration::from_millis(50);
18888 let wait = repository_wait(&st, None);
18889 let (outcomes, _) = with_cache_lease(
18890 &mut st,
18891 Some(&cache),
18892 "gate",
18893 "gate",
18894 tmp.path(),
18895 "abc1234",
18896 budget,
18897 "test gate",
18898 &wait,
18899 |_, available| async move {
18900 assert_eq!(
18901 available, budget,
18902 "contention must not shorten verification"
18903 );
18904 (Vec::new(), false, Vec::new())
18905 },
18906 )
18907 .await
18908 .unwrap();
18909 release.await.unwrap();
18910 assert!(outcomes.is_empty(), "{outcomes:?}");
18911 assert!(
18912 st.events
18913 .iter()
18914 .any(|event| event.message.contains("waiting for"))
18915 );
18916 assert!(st.resource_wait.is_none());
18917 hold(&cache).release();
18919 }
18920
18921 #[tokio::test]
18922 async fn verification_cache_wait_expires_without_starting_commands() {
18923 crate::run::pin_test_home();
18924 let tmp = tempfile::tempdir().unwrap();
18925 let cache = tmp.path().join("target");
18926 let _held = hold(&cache);
18927 let mut st = state(0);
18928 let wait = repository_wait(&st, None);
18929 let (outcomes, _) = with_cache_lease(
18930 &mut st,
18931 Some(&cache),
18932 "gate",
18933 "gate",
18934 tmp.path(),
18935 "abc1234",
18936 Duration::from_secs(60),
18937 "test gate",
18938 &wait,
18939 |_, _| async { panic!("commands must not start without the cache lease") },
18940 )
18941 .await
18942 .unwrap();
18943 assert_eq!(outcomes.len(), 1);
18944 assert!(outcomes[0].resource_blocked);
18945 assert!(st.resource_wait.is_none());
18946 }
18947
18948 #[tokio::test]
18949 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18950 crate::run::pin_test_home();
18951 let tmp = tempfile::tempdir().unwrap();
18952 let cache = tmp.path().join("target-a");
18953 let guard = hold(&cache);
18954 tokio::spawn(async move {
18956 tokio::time::sleep(Duration::from_millis(1500)).await;
18957 guard.release();
18958 });
18959 let mut st = state(60);
18960 let results = run(
18961 vec![job(
18962 sh("alpha", "echo ok"),
18963 tmp.path(),
18964 Duration::from_secs(1),
18965 )],
18966 Arc::new(Semaphore::new(1)),
18967 Some(&cache),
18968 &mut st,
18969 )
18970 .await;
18971 assert!(
18972 matches!(results[0].2, AgentOutcome::Ok(_)),
18973 "the wait must not be charged to the agent"
18974 );
18975 assert!(st.resource_block.is_none());
18976 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18977 assert!(st.resource_wait.is_none() && st.active.is_empty());
18978 }
18979
18980 #[tokio::test]
18981 #[cfg(unix)]
18982 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18983 use std::os::fd::AsRawFd as _;
18984 crate::run::pin_test_home();
18985 let tmp = tempfile::tempdir().unwrap();
18986 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18987 let common = tmp.path().join(".git");
18988 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18989 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18990 assert_eq!(
18991 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18992 0
18993 );
18994 let mut st = state(1);
18995 st.config.coordination = crate::config::test_coordination();
18996 let mut reader = job(
18997 sh("reader", "echo must-not-run"),
18998 tmp.path(),
18999 Duration::from_secs(30),
19000 );
19001 reader.allow_write = false;
19002 let results = run(
19003 vec![reader.clone()],
19004 Arc::new(Semaphore::new(1)),
19005 None,
19006 &mut st,
19007 )
19008 .await;
19009 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19010 assert_eq!(results[0].1.turns, 0);
19011 assert!(FailClass::of(&results[0].2).is_none());
19012 assert_eq!(
19013 st.resource_block.as_ref().unwrap().kind,
19014 "repository coordination"
19015 );
19016 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19017 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19018 drop(external);
19019 let editor = crate::repo_lock::SeatLease::for_checkout(
19020 tmp.path(),
19021 true,
19022 crate::config::test_coordination().as_ref(),
19023 )
19024 .await
19025 .unwrap();
19026 let pause = Pause::new();
19027 let interrupt = Pause::new();
19028 let prompts = Prompts::default();
19029 let ctx = WaveCtx {
19030 run: "r",
19031 node: "review",
19032 prompts: &prompts,
19033 cache: None,
19034 round: None,
19035 carry_seats: false,
19036 pauses: Some((&pause, &interrupt)),
19037 };
19038 let (results, ()) = tokio::join!(
19039 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19040 async {
19041 tokio::time::sleep(Duration::from_millis(50)).await;
19042 pause.park();
19043 }
19044 );
19045 assert!(
19046 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19047 );
19048 assert_eq!(results[0].1.turns, 0);
19049 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19050 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19051 drop(editor);
19052 }
19053
19054 #[tokio::test]
19055 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19056 crate::run::pin_test_home();
19057 let tmp = tempfile::tempdir().unwrap();
19058 let cache = tmp.path().join("target-b");
19059 let _held = hold(&cache);
19060 let mut st = state(1);
19061 let results = run(
19062 vec![job(
19063 sh("alpha", "echo ok"),
19064 tmp.path(),
19065 Duration::from_secs(1),
19066 )],
19067 Arc::new(Semaphore::new(1)),
19068 Some(&cache),
19069 &mut st,
19070 )
19071 .await;
19072 let (_, seat, out) = &results[0];
19073 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19074 assert!(wave_blocked(&results));
19075 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19076 assert_eq!(seat.turns, 0, "nothing was asked");
19077 assert!(
19078 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19079 "no agent was spawned"
19080 );
19081 assert!(FailClass::of(out).is_none(), "not a failure class");
19082 let block = st.resource_block.as_ref().expect("recorded for the resume");
19083 assert_eq!(block.kind, "shared build cache");
19084 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19085 assert!(st.active.is_empty());
19086 }
19087
19088 #[tokio::test]
19089 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19090 crate::run::pin_test_home();
19091 let tmp = tempfile::tempdir().unwrap();
19092 let mut st = state(1);
19093 let results = run(
19094 vec![job(
19095 sh("alpha", "echo ok"),
19096 tmp.path(),
19097 Duration::from_secs(1),
19098 )],
19099 Arc::new(Semaphore::new(0)),
19100 None,
19101 &mut st,
19102 )
19103 .await;
19104 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19105 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19106 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19107 }
19108
19109 #[tokio::test]
19110 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19111 crate::run::pin_test_home();
19112 let tmp = tempfile::tempdir().unwrap();
19113 let mut st = state(60);
19114 let results = run(
19115 vec![job(
19116 sh("alpha", "sleep 30"),
19117 tmp.path(),
19118 Duration::from_secs(1),
19119 )],
19120 Arc::new(Semaphore::new(1)),
19121 None,
19122 &mut st,
19123 )
19124 .await;
19125 let out = &results[0].2;
19126 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19127 let class = FailClass::of(out).expect("a failure");
19128 assert_eq!(class, FailClass::Timeout);
19129 assert!(should_hand_over(None, &class), "existing handover policy");
19130 assert!(st.resource_block.is_none());
19131 }
19132}