1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 for id in &takeover.earlier {
1033 if let Ok(old) = RunState::load_under(id, &takeover.home)
1034 && old.repo == state.repo
1035 && (old.pr_head.as_deref() == Some(branch)
1036 || old.winner().is_some_and(|w| w.branch == branch))
1037 {
1038 for hold in old.branch_owner_holds {
1039 if !state.branch_owner_holds.contains(&hold) {
1040 state.event(
1041 "approval-policy",
1042 format!(
1043 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044 hold.control.key(),
1045 hold.file,
1046 hold.head
1047 ),
1048 );
1049 state.branch_owner_holds.push(hold);
1050 }
1051 }
1052 }
1053 }
1054 let reviewed_head = git::rev_parse(repo, branch).await?;
1055 state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057 let worktree = state.worktree_root().join("under-review");
1060 if let Some(parent) = worktree.parent() {
1061 tokio::fs::create_dir_all(parent).await.ok();
1062 }
1063 let path = worktree.to_string_lossy().to_string();
1064 git::git(repo, &["worktree", "add", &path, branch])
1065 .await
1066 .with_context(|| {
1067 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068 })?;
1069
1070 let setup = match ensure_setup_config(&mut state, repo).await {
1071 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072 Err(e) => Err(e),
1073 };
1074 if let Err(e) = setup {
1075 git::worktree_remove_clean(repo, &worktree).await.ok();
1076 return Err(e);
1077 }
1078 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079 .await
1080 .unwrap_or(0);
1081 if commits == 0 {
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084 }
1085 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086 .await
1087 .map(|f| f.len())
1088 .unwrap_or(0);
1089 if files == 0
1090 && let (Ok(head_tree), Ok(base_tree)) = (
1091 git::tree_of(&worktree, "HEAD").await,
1092 git::tree_of(&worktree, &base_commit).await,
1093 )
1094 && head_tree == base_tree
1095 {
1096 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097 git::worktree_remove_clean(repo, &worktree).await.ok();
1098 bail!(
1099 "`{branch}` at {} has a tree identical to base {}; this usually means \
1100 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101 against `{}/{branch}`) rather than an empty change",
1102 short(&head),
1103 short(&base_commit),
1104 state.config.merge.remote
1105 );
1106 }
1107 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108 .await
1109 .unwrap_or_default();
1110
1111 state.candidates.push(Candidate {
1112 index: 0,
1113 label: 'A',
1114 agent: EXISTING_BRANCH.to_owned(),
1117 branch: branch.to_owned(),
1118 worktree,
1119 summary: String::new(),
1120 stat,
1121 files,
1122 commits,
1123 empty: false,
1124 failed: None,
1125 verified_noop: None,
1126 duration_ms: 0,
1127 folded: false,
1128 });
1129 state.tally = Some(Tally {
1130 first_choice: BTreeMap::from([('A', 0)]),
1131 borda: BTreeMap::new(),
1132 winner: 'A',
1133 rankings: 0,
1134 unanimous_initial: false,
1135 deliberated: false,
1136 changed_votes: 0,
1137 unanimous_final: false,
1138 tie_break: None,
1139 judges: 0,
1143 present: 0,
1144 quorum: 0,
1145 met_quorum: true,
1146 uncontested: Some("review-only run: nothing competed".to_owned()),
1147 });
1148 state.status = RunStatus::Reviewing;
1149 state.prep_complete = Some(true);
1150 state.event(
1151 "start",
1152 format!(
1153 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154 state.id
1155 ),
1156 );
1157 state.save()?;
1158 Ok(Self {
1159 state,
1160 roles,
1161 sem: Arc::new(Semaphore::new(max_parallel)),
1162 pause: Pause::new(),
1163 interrupt: Pause::new(),
1164 })
1165 }
1166
1167 pub fn resume(id: &str) -> Result<Self> {
1169 let state = RunState::load(id)?;
1170 crate::pr_head::ensure_owner(&state)?;
1171 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172 bail!(
1173 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174 );
1175 }
1176 if let Some(to) = &state.released_to {
1177 bail!(
1178 "run {} cannot be resumed: its worktree was released to run {}",
1179 state.short(),
1180 crate::run::short_of(to)
1181 );
1182 }
1183 let roles = state.config.resolve_roles()?;
1184 if let Some(id) = &state.config.graph.selected_implementer
1185 && state.candidates.iter().any(|c| c.agent != *id)
1186 {
1187 bail!(
1188 "replacement implementer pin does not match the saved candidates; task must stay held"
1189 );
1190 }
1191 let max_parallel = state.config.graph.max_parallel.max(1);
1192 Ok(Self {
1193 state,
1194 roles,
1195 sem: Arc::new(Semaphore::new(max_parallel)),
1196 pause: Pause::new(),
1197 interrupt: Pause::new(),
1198 })
1199 }
1200
1201 pub async fn execute(&mut self) -> Result<()> {
1208 if let Some(selection) = &self.state.pr_recovery
1209 && (selection.source_proof.is_some() || selection.source_run.is_some())
1210 {
1211 Box::pin(selection.verify_source(&self.state.repo)).await?;
1212 }
1213 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216 || self.state.land_approval.is_some()
1217 || self.state.merge_decision.is_some()
1218 || !self.state.merge_repairs.is_empty()
1219 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220 {
1221 let claim = FixClaim::acquire(&self.state.dir())?;
1222 let disk = RunState::load(&self.state.id)?;
1223 let checkpoint = |s: &RunState| {
1224 (
1225 s.merge_repairs.len(),
1226 s.merge_repairs
1227 .last()
1228 .map(|r| (r.question.clone(), r.phase)),
1229 s.land_approval
1230 .as_ref()
1231 .map(|a| (a.question.clone(), a.head.clone())),
1232 )
1233 };
1234 if checkpoint(&disk) != checkpoint(&self.state) {
1235 bail!(
1236 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237 );
1238 }
1239 Some(claim)
1240 } else {
1241 None
1242 };
1243 let mut execution = self.claim_driver()?;
1244 let runner = &mut *execution.runner;
1245 let wait = runner.repository_wait();
1246 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249 let result = match result {
1250 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251 record_repository_block(&mut runner.state, &e);
1252 runner.park_here()?;
1253 Ok(())
1254 }
1255 other => other,
1256 };
1257 execution.driver.finish(&mut runner.state)?;
1258 let ended = if result.is_err() {
1259 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260 } else {
1261 crate::notices::run_ended(&runner.state)
1262 };
1263 if let Some(notice) = ended {
1264 crate::notices::raise(notice);
1265 }
1266 result
1267 }
1268
1269 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271 }
1272
1273 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274 let wait = self.repository_wait();
1275 coordinate_operation(&mut self.state, cwd, &wait).await
1276 }
1277
1278 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1281 Ok(owner) => owner,
1282 Err(error) => {
1283 let detail = format!("worktree maintenance or ownership update: {error:#}");
1284 self.state.resource_block = Some(crate::run::ResourceBlock {
1288 node: "start".into(),
1289 kind: "worktree maintenance".into(),
1290 detail: detail.clone(),
1291 at: Timestamp::now(),
1292 });
1293 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1294 }
1295 };
1296 let disk = RunState::load(&self.state.id)?;
1297 crate::pr_head::ensure_owner(&disk)?;
1298 if disk.liveness(false) == crate::run::Liveness::Live {
1299 bail!("run already has a live driver; wait for its execution to end");
1300 }
1301 if let Some(to) = &disk.released_to {
1302 bail!(
1303 "run {} cannot continue: its worktree was released to run {}",
1304 self.state.short(),
1305 crate::run::short_of(to)
1306 );
1307 }
1308 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1309 let mut execution = DriverExecution {
1310 runner: self,
1311 driver,
1312 };
1313 let _owner = owner;
1316 let state = &mut execution.runner.state;
1317 state.parked = false;
1322 state.resource_wait = None;
1323 state.resource_block = None;
1324 state.clear_active();
1331 let pid = std::process::id();
1346 state.driver_pid = Some(pid);
1347 state.driver_started_at = crate::proc::process_started_at(pid);
1348 state.driver_exited = false;
1349 state.begin_pass();
1350 execution.driver.bind(state);
1351 state.save()?;
1352 execution.driver.record(state)?;
1353 Ok(execution)
1354 }
1355
1356 async fn execute_graph(&mut self) -> Result<()> {
1357 self.state.configure_recovery_resume()?;
1358 self.persist_candidates().await?;
1361 if self
1362 .state
1363 .merge_decision
1364 .as_ref()
1365 .is_some_and(|decision| !decision.closed)
1366 && self.state.status != RunStatus::Landing
1367 {
1368 if crate::merge_decision::resumable_hold(&self.state) {
1369 self.state.merge = Some(MergeOutcome {
1370 mode: MergeMode::Pr,
1371 ok: false,
1372 detail: self
1373 .state
1374 .merge_decision
1375 .as_ref()
1376 .unwrap()
1377 .target
1378 .pr
1379 .clone(),
1380 empty: false,
1381 });
1382 self.run_land().await?;
1383 self.settle_questions();
1384 return Ok(());
1385 }
1386 bail!(
1387 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1388 );
1389 }
1390 if self.state.config.graph.selected_implementer.is_some()
1391 && self.state.status == RunStatus::Blocked
1392 && self.state.tally.is_none()
1393 && self.state.candidate_preservation.is_empty()
1394 && self.state.interrupted_implementations.is_empty()
1395 {
1396 bail!(
1397 "selected implementation is held; explicitly release the owning task to retry it"
1398 );
1399 }
1400
1401 if self.state.merge_repairs.last().is_some_and(|r| {
1402 matches!(
1403 r.phase,
1404 crate::run::MergeRepairPhase::Requested
1405 | crate::run::MergeRepairPhase::Fixing
1406 | crate::run::MergeRepairPhase::Reviewing
1407 )
1408 }) {
1409 if self.repair_merge_review().await? {
1410 self.run_land().await?;
1411 }
1412 self.settle_questions();
1413 return Ok(());
1414 }
1415 if self.state.pr_publication_pending {
1418 self.merge().await?;
1419 if self.resource_halt()? {
1420 return Ok(());
1421 }
1422 self.state.save()?;
1423 self.settle_questions();
1424 return Ok(());
1425 }
1426 if self.state.status == RunStatus::Stalled {
1439 if self.recover_stall().await? {
1440 self.finish_after_tally().await?;
1441 } else {
1442 self.state.save()?;
1444 }
1445 return Ok(());
1446 }
1447 if self.state.status == RunStatus::Landing
1457 || crate::handover::landing_checkpoint(&self.state)
1458 {
1459 self.state.land_sync_pending = true;
1460 self.state.save()?;
1461 self.run_land().await?;
1462 self.settle_questions();
1467 return Ok(());
1468 }
1469 if self.state.github_text.is_some()
1474 && self.state.merge.is_none()
1475 && self.state.status == RunStatus::Gating
1476 {
1477 self.merge().await?;
1478 if self.resource_halt()? {
1479 return Ok(());
1480 }
1481 self.state.save()?;
1482 return Ok(());
1483 }
1484 if let Err(e) = self.prep().await {
1485 if e.is::<crate::repo_lock::ResourceBlocked>() {
1486 return Err(e);
1487 }
1488 self.state.status = RunStatus::Blocked;
1489 self.state
1490 .prep_failure
1491 .get_or_insert_with(|| format!("{e:#}"));
1492 self.state
1493 .event("prep", format!("preparation stopped: {e:#}"));
1494 self.state.save()?;
1495 return Err(e);
1496 }
1497 if self.park_here()? {
1498 return Ok(());
1499 }
1500 self.advise().await?;
1501 if self.park_here()? {
1502 return Ok(());
1503 }
1504 self.implement().await?;
1505 if self.park_here()? {
1506 return Ok(());
1507 }
1508 if self.state.status == RunStatus::VerifiedNoop {
1512 return Ok(());
1513 }
1514 self.judge().await?;
1515 if self.park_here()? {
1516 return Ok(());
1517 }
1518 self.deliberate().await?;
1519 if self.park_here()? {
1520 return Ok(());
1521 }
1522 self.vote().await?;
1523 if self.park_here()? {
1524 return Ok(());
1525 }
1526 self.tally()?;
1527 if self.state.status == RunStatus::Stalled {
1532 self.state.save()?;
1536 return Ok(());
1537 }
1538 self.finish_after_tally().await?;
1539 Ok(())
1540 }
1541
1542 fn stop_for_resource(&mut self) -> Result<()> {
1547 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1549 {
1550 self.state.status = RunStatus::Blocked;
1551 }
1552 self.state.save()
1553 }
1554
1555 fn resource_halt(&mut self) -> Result<bool> {
1557 if self.state.resource_block.is_none() {
1558 return Ok(false);
1559 }
1560 self.stop_for_resource()?;
1561 Ok(true)
1562 }
1563
1564 fn park_here(&mut self) -> Result<bool> {
1571 let parking = self.pause.parked() || self.interrupt.parked();
1572 if !parking && self.state.resource_block.is_some() {
1573 self.stop_for_resource()?;
1574 return Ok(true);
1575 }
1576 if !self.pause.parked() && !self.interrupt.parked() {
1581 return Ok(false);
1582 }
1583 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1584 let why = match cause.as_ref().map(ParkCause::label) {
1585 Some(reason) => format!(
1586 "parked after `{}` ({reason}) — resume to carry on from here",
1587 self.state.status.as_str()
1588 ),
1589 None => format!(
1590 "parked after `{}` — resume to carry on from here",
1591 self.state.status.as_str()
1592 ),
1593 };
1594 self.state.event("park", why);
1595 self.state
1596 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1597 self.state.parked = true;
1598 self.state.save()?;
1599 Ok(true)
1600 }
1601
1602 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1604 if self.state.prep_failure.is_none() {
1605 return Ok(());
1606 }
1607 let failed_at = self
1608 .state
1609 .events
1610 .iter()
1611 .rev()
1612 .find(|event| event.node == "prep")
1613 .map_or(self.state.updated_at, |event| event.at);
1614 anyhow::ensure!(
1615 approval_at > failed_at,
1616 "preparation retry requires a new explicit release after the failure"
1617 );
1618 self.state.prep_failure = None;
1619 self.state.prep_complete = Some(false);
1620 self.state
1621 .event("prep-retry", "operator approved retry of preparation");
1622 Ok(())
1623 }
1624
1625 pub fn on_pause(&mut self, pause: Pause) {
1627 self.pause = pause;
1628 }
1629
1630 pub fn watch_interrupt(&mut self, pause: Pause) {
1636 self.interrupt = pause;
1637 }
1638
1639 fn settle_questions(&mut self) {
1659 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1660 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1661 }
1662 }
1663
1664 async fn finish_after_tally(&mut self) -> Result<()> {
1667 self.graft().await?;
1670 if self.resource_halt()? {
1671 return Ok(());
1672 }
1673 self.fold_losers().await?;
1674 self.sync_to_base().await?;
1679 if self.resource_halt()? {
1680 return Ok(());
1681 }
1682 if self.state.status == RunStatus::AlreadyInBase {
1683 return Ok(());
1684 }
1685 self.review_loop().await?;
1686 if self.resource_halt()? {
1687 return Ok(());
1688 }
1689 self.sync_to_base().await?;
1690 if self.resource_halt()? {
1691 return Ok(());
1692 }
1693 if self.state.status == RunStatus::AlreadyInBase {
1694 return Ok(());
1695 }
1696 self.gate().await?;
1697 self.merge().await?;
1698 if self.resource_halt()? {
1699 return Ok(());
1700 }
1701 self.state.save()?;
1702 Ok(())
1703 }
1704
1705 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1712 let result = match ensure_setup_config(&mut self.state, repo).await {
1713 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1714 Err(e) => Err(e),
1715 };
1716 if let Err(e) = result {
1717 git::worktree_remove_clean(repo, wt).await.ok();
1718 if in_prep {
1719 self.rollback_prep(repo).await;
1720 }
1721 self.state.status = RunStatus::Blocked;
1722 self.state.event(
1723 "setup",
1724 format!("worktree setup failed in {}: {e:#}", wt.display()),
1725 );
1726 self.state.save()?;
1727 return Err(e);
1728 }
1729 Ok(())
1730 }
1731
1732 async fn rollback_prep(&mut self, repo: &Path) {
1734 let root = self.state.worktree_root();
1735 for c in std::mem::take(&mut self.state.candidates) {
1736 if matches!(
1737 git::worktree_remove_clean(repo, &c.worktree).await,
1738 Ok(true)
1739 ) {
1740 git::branch_delete(repo, &c.branch).await.ok();
1741 } else {
1742 self.state.candidates.push(c);
1743 }
1744 }
1745 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1746 for j in 1..=seats {
1747 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1748 .await
1749 .ok();
1750 }
1751 for k in 1..=self.state.config.graph.advisors {
1752 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1753 .await
1754 .ok();
1755 }
1756 }
1757
1758 async fn prep(&mut self) -> Result<()> {
1759 if let Some(reason) = &self.state.prep_failure {
1760 bail!(
1761 "candidate preparation previously failed: {reason}; correct the source before retrying"
1762 );
1763 }
1764 if self
1765 .state
1766 .prep_complete
1767 .unwrap_or(!self.state.candidates.is_empty())
1768 {
1769 return Ok(());
1770 }
1771 let repo = self.state.repo.clone();
1772 let _coordination = self.coordinate(&repo).await?;
1775 let base = self.state.base_commit.clone();
1776 let plan = self.state.sources.plan(&repo, &base).await?;
1777 self.rollback_prep(&repo).await;
1778 if !self.state.candidates.is_empty() {
1779 bail!(
1780 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1781 );
1782 }
1783 self.state.prep_complete = Some(false);
1784 self.state.status = RunStatus::Prep;
1785 self.state.save()?;
1786 let repo = self.state.repo.clone();
1787 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1788 let root = self.state.worktree_root();
1789 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1790
1791 let hooks_dir = self.state.dir().join("hooks");
1794 if self.state.config.blind.commit_msg_hook {
1795 std::fs::create_dir_all(&hooks_dir)
1796 .with_context(|| format!("create {}", hooks_dir.display()))?;
1797 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1798 let path = hooks_dir.join("commit-msg");
1799 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1800 make_executable(&path)?;
1801 git::acquire_worktree_config(&repo).await?;
1809 self.state.enabled_worktree_config = true;
1810 }
1811
1812 ensure_setup_config(&mut self.state, &repo).await?;
1815
1816 for (index, (spec, label)) in self
1817 .roles
1818 .implementers
1819 .clone()
1820 .into_iter()
1821 .zip(labels)
1822 .enumerate()
1823 {
1824 let branch = self.state.branch_for(label);
1825 let worktree = root.join(format!("cand-{label}"));
1826 self.state.event(
1830 "prep-intent",
1831 serde_json::json!({
1832 "branch": branch,
1833 "worktree": worktree,
1834 "start": start,
1835 "picks": plan.picks,
1836 })
1837 .to_string(),
1838 );
1839 self.state.save()?;
1840 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1841 if self.state.config.blind.commit_msg_hook {
1842 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1843 }
1844 git::local_exclude(&worktree, "/.magi/").await?;
1845 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1846 git::worktree_remove_clean(&repo, &worktree).await.ok();
1849 git::branch_delete(&repo, &branch).await.ok();
1850 self.rollback_prep(&repo).await;
1851 self.state.status = RunStatus::Blocked;
1852 self.state
1853 .event("prep", format!("worktree setup failed: {e:#}"));
1854 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1855 self.state.save()?;
1856 return Err(e);
1857 }
1858 self.state.candidates.push(Candidate {
1859 index,
1860 label,
1861 agent: spec.id.clone(),
1862 branch,
1863 worktree,
1864 summary: String::new(),
1865 stat: String::new(),
1866 files: 0,
1867 commits: 0,
1868 empty: false,
1869 failed: None,
1870 verified_noop: None,
1871 duration_ms: 0,
1872 folded: false,
1873 });
1874 self.state.save()?;
1875 for pick in &plan.picks {
1876 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1877 {
1878 self.state.status = RunStatus::Blocked;
1879 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1880 self.state
1881 .event("prep", format!("cannot apply referenced commit: {e}"));
1882 self.state.save()?;
1883 return Err(e);
1884 }
1885 }
1886 }
1887
1888 for j in 1..=self.roles.judges.len() {
1889 let wt = root.join(format!("judge-{j}"));
1890 if !wt.exists() {
1891 git::worktree_add_detached(&repo, &wt, &base).await?;
1892 self.setup_seat_worktree(&repo, &wt, true).await?;
1893 }
1894 }
1895
1896 if self.state.config.graph.advise {
1904 for k in 1..=self.state.config.graph.advisors {
1905 let wt = root.join(format!("advisor-{k}"));
1906 if !wt.exists() {
1907 git::worktree_add_detached(&repo, &wt, &base).await?;
1908 self.setup_seat_worktree(&repo, &wt, true).await?;
1909 }
1910 }
1911 }
1912
1913 let authors: Vec<&str> = self
1918 .roles
1919 .implementers
1920 .iter()
1921 .map(|a| a.id.as_str())
1922 .collect();
1923 let overlap: Vec<String> = self
1924 .roles
1925 .judges
1926 .iter()
1927 .enumerate()
1928 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1929 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1930 .collect();
1931 if !overlap.is_empty() {
1932 let note = format!(
1933 "{} also authored a candidate; blind, but the panel is less \
1934 independent than {} distinct agents would be",
1935 overlap.join(", "),
1936 self.roles.judges.len()
1937 );
1938 self.state.event("prep", note);
1939 }
1940
1941 self.state.event(
1942 "prep",
1943 format!(
1944 "{} candidates, {} judges, base {} ({})",
1945 self.state.candidates.len(),
1946 self.roles.judges.len(),
1947 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1948 self.state.base_branch
1949 ),
1950 );
1951 self.state.prep_complete = Some(true);
1952 self.state.status = RunStatus::Implementing;
1953 self.state.save()?;
1954 Ok(())
1955 }
1956
1957 async fn advise(&mut self) -> Result<()> {
1990 let implement_untouched = self
1991 .state
1992 .candidates
1993 .iter()
1994 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1995 if !self.state.config.graph.advise || self.state.advise_attempted {
1996 return Ok(());
1997 }
1998 if !implement_untouched {
1999 self.state.event(
2000 "advise",
2001 "skipping the design-deliberation stage: at least one \
2002 candidate already shows implementation progress, so this \
2003 run is past the point the stage exists to run before"
2004 .to_owned(),
2005 );
2006 self.state.advise_attempted = true;
2007 self.state.save()?;
2008 return Ok(());
2009 }
2010 let run_id = self.state.id.clone();
2011 let prompts = self.state.config.prompts.clone();
2012 let instruction = self.state.instruction.clone();
2013 let language = self.state.config.graph.language.clone();
2014 let root = self.state.worktree_root();
2015 let n = self.state.config.graph.advisors;
2016 let where_recorded = self.state.dir().join("run.json");
2017
2018 let seats = match self.state.config.advisors() {
2019 Ok(seats) if !seats.is_empty() => seats,
2020 Ok(_) => {
2021 self.state.event(
2022 "advise",
2023 format!(
2024 "[graph] advisors is 0; skipping the design-deliberation \
2025 stage and continuing without a synthesis brief (see {})",
2026 where_recorded.display()
2027 ),
2028 );
2029 self.state.advise_attempted = true;
2030 self.state.save()?;
2031 return Ok(());
2032 }
2033 Err(e) => {
2034 self.state.event(
2035 "advise",
2036 format!(
2037 "could not resolve advisor seats ({e:#}); continuing \
2038 without a design-deliberation brief (see {})",
2039 where_recorded.display()
2040 ),
2041 );
2042 self.state.advise_attempted = true;
2043 self.state.save()?;
2044 return Ok(());
2045 }
2046 };
2047
2048 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2049 let artifacts = agent::artifacts_dir(&self.state.dir());
2050 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2051
2052 let mut jobs = Vec::new();
2053 for (i, spec) in seats.iter().cloned().enumerate() {
2054 let seat_key = format!("advisor-{}", i + 1);
2055 let seat = self.seat(&seat_key, &spec.id);
2056 jobs.push(SeatJob {
2057 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2058 spec,
2059 seat,
2060 cwd: worktrees[i % worktrees.len()].clone(),
2061 timeout,
2062 allow_write: false,
2063 sessions: false,
2064 artifacts: artifacts.clone(),
2065 stem: seat_key,
2066 handover: None,
2067 });
2068 }
2069
2070 self.state.event(
2071 "advise",
2072 format!(
2073 "{} advisor seat(s) sketching a design in parallel",
2074 jobs.len()
2075 ),
2076 );
2077 let mut quota_losses = Vec::new();
2078 let cache = self.state.config.cache_dir();
2079 let ctx = WaveCtx {
2080 carry_seats: false,
2081 pauses: Some((&self.pause, &self.interrupt)),
2082 run: &run_id,
2083 node: "advise",
2084 prompts: &prompts,
2085 cache: cache.as_deref(),
2086 round: None,
2087 };
2088 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2089 let results = ask_json_wave::<Proposal>(
2090 jobs,
2091 Arc::clone(&self.sem),
2092 self.state.config.graph.retries,
2093 &advisor_roster,
2094 &ctx,
2095 &mut quota_losses,
2096 &mut self.state,
2097 &|p: &Proposal| p.validate(),
2098 )
2099 .await;
2100 if self.state.resource_block.is_some() {
2101 return self.stop_for_resource();
2102 }
2103 self.state.quota.extend(quota_losses);
2104
2105 let mut records = Vec::with_capacity(results.len());
2106 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2107 let agent_id = seat.agent.clone();
2108 self.state.seats.insert(seat.key.clone(), seat);
2109 match res {
2110 Ok((proposal, out)) => {
2111 self.state
2112 .event("advise", format!("advisor-{} proposed a design", i + 1));
2113 records.push(advise::AdvisorRecord::proposed(
2114 i + 1,
2115 agent_id,
2116 proposal,
2117 out.duration_ms,
2118 ));
2119 }
2120 Err(e) => {
2121 self.state.event(
2122 "advise",
2123 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2124 );
2125 records.push(advise::AdvisorRecord::failed(
2126 i + 1,
2127 agent_id,
2128 e.to_string(),
2129 ));
2130 }
2131 }
2132 }
2133
2134 let mut advice = advise::Advice {
2135 records,
2136 synthesis: None,
2137 };
2138 if advice.proposals().is_empty() {
2139 self.state.event(
2140 "advise",
2141 "no advisor produced a usable proposal; continuing without a \
2142 synthesis brief"
2143 .to_owned(),
2144 );
2145 } else {
2146 match self
2147 .synthesize_brief(
2148 &advice,
2149 &instruction,
2150 &language,
2151 &worktrees[0],
2152 &artifacts,
2153 &run_id,
2154 &prompts,
2155 cache.as_deref(),
2156 )
2157 .await
2158 {
2159 Ok(Some(text)) => {
2160 self.state.event(
2161 "advise",
2162 "synthesized a design brief for the implementer".to_owned(),
2163 );
2164 advice.synthesis = Some(text);
2165 }
2166 Ok(None) => {
2167 self.state.event(
2168 "advise",
2169 "the synthesis seat produced nothing usable; continuing \
2170 without a design brief"
2171 .to_owned(),
2172 );
2173 }
2174 Err(e) => {
2175 if e.is::<crate::repo_lock::ResourceBlocked>() {
2176 return Err(e);
2177 }
2178 self.state.event(
2179 "advise",
2180 format!("could not synthesize a design brief: {e:#}"),
2181 );
2182 }
2183 }
2184 }
2185 advise::apply_reflection(&mut advice);
2186
2187 self.state.advice = Some(advice);
2188 self.state.advise_attempted = true;
2189 self.state.save()?;
2190 Ok(())
2191 }
2192
2193 #[allow(clippy::too_many_arguments)]
2205 async fn synthesize_brief(
2206 &mut self,
2207 advice: &advise::Advice,
2208 instruction: &str,
2209 language: &str,
2210 cwd: &Path,
2211 artifacts: &Path,
2212 run_id: &str,
2213 prompts: &Prompts,
2214 cache: Option<&Path>,
2215 ) -> Result<Option<String>> {
2216 let chain = agent::pick_chain(
2217 &self.state.config.agents,
2218 self.state.config.roles.synthesizer.as_ref(),
2219 &agent::installed,
2220 "synthesizer",
2221 )?;
2222 let proposals = advice.proposals();
2223 let mut prompt = prompt::with_overlay(
2224 prompt::synthesize_brief(instruction, &proposals, language),
2225 prompts.overlay("advise"),
2226 );
2227 if cache.is_some() {
2228 prompt.push('\n');
2233 prompt.push_str(&prompt::build_cache_note("advise", false));
2234 }
2235 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2236 let mut last = None;
2241 for (n, spec) in chain.iter().enumerate() {
2242 if n > 0 {
2243 self.state
2244 .event("advise", format!("synthesis falling back to {}", spec.id));
2245 }
2246 let mut seat = self.seat("advise-synthesis", &spec.id);
2247 let wait = self.repository_wait();
2248 let outcome = wait
2249 .scope(agent::invoke(
2250 spec,
2251 &mut seat,
2252 &Invocation {
2253 scope: crate::agent::InvocationScope::Repository,
2254 coordination: self.state.config.coordination.as_ref(),
2255 fetched_base: None,
2256 cwd,
2257 prompt: &prompt,
2258 timeout,
2259 allow_write: false,
2260 unsandboxed: false,
2261 sessions: false,
2262 artifacts,
2263 stem: &if n == 0 {
2264 "advise-synthesis".to_owned()
2265 } else {
2266 format!("advise-synthesis-{}", spec.id)
2267 },
2268 run: run_id,
2269 node: "advise",
2270 cache_dir: None,
2271 attachments: &[],
2272 writable: &[],
2273 },
2274 ))
2275 .await;
2276 if outcome.is_ok() {
2277 self.state.seats.insert(seat.key.clone(), seat);
2278 }
2279 let advance = agent::chain_advances(&outcome);
2280 last = Some(outcome);
2281 if !advance {
2282 break;
2283 }
2284 }
2285 let out = last.expect("a chain holds at least one agent")?;
2288 if !out.usable() {
2289 return Ok(None);
2290 }
2291 let text =
2292 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2293 Ok((!text.trim().is_empty()).then_some(text))
2294 }
2295
2296 async fn implement(&mut self) -> Result<()> {
2299 self.persist_candidates().await?;
2300 let run_id = self.state.id.clone();
2305 let prompts = self.state.config.prompts.clone();
2306 let todo: Vec<usize> = self
2307 .state
2308 .candidates
2309 .iter()
2310 .enumerate()
2311 .filter(|(_, c)| {
2312 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2313 && c.failed.is_none()
2314 && !c.empty
2315 })
2316 .map(|(i, _)| i)
2317 .collect();
2318 if todo.is_empty() {
2319 return self.after_implement();
2320 }
2321 self.state.status = RunStatus::Implementing;
2322
2323 let language = self.state.config.graph.language.clone();
2324 let github_language = self.state.config.graph.github_language.clone();
2325 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2326 let sessions = self.state.config.graph.sessions;
2327 let artifacts = agent::artifacts_dir(&self.state.dir());
2328 let brief = self
2332 .state
2333 .advice
2334 .as_ref()
2335 .and_then(|a| a.synthesis.as_deref())
2336 .map(str::to_owned);
2337 let attachments = self.state.attachments.clone();
2338
2339 let mut jobs = Vec::new();
2340 for &i in &todo {
2341 let (index, label, worktree) = {
2342 let c = &self.state.candidates[i];
2343 (c.index, c.label, c.worktree.clone())
2344 };
2345 let spec = self.roles.implementers[index].clone();
2346 let seat_key = format!("impl-{label}");
2347 let seat = self.seat(&seat_key, &spec.id);
2348 let instruction = seeded_instruction(&self.state);
2349 jobs.push(SeatJob {
2350 spec,
2351 seat,
2352 prompt: prompt::implement_in(
2353 &instruction,
2354 &worktree.to_string_lossy(),
2355 &language,
2356 &github_language,
2357 brief.as_deref(),
2358 &attachments,
2359 ),
2360 cwd: worktree,
2361 timeout,
2362 allow_write: true,
2363 sessions,
2364 artifacts: artifacts.clone(),
2365 stem: format!("impl-{label}"),
2366 handover: None,
2367 });
2368 }
2369
2370 self.state.event(
2371 "implement",
2372 format!("{} candidates in parallel", jobs.len()),
2373 );
2374 let mut sent = jobs.clone();
2380 let cache = self.state.config.cache_dir();
2381 let ctx = WaveCtx {
2382 carry_seats: false,
2383 pauses: Some((&self.pause, &self.interrupt)),
2384 run: &run_id,
2385 node: "implement",
2386 prompts: &prompts,
2387 cache: cache.as_deref(),
2388 round: None,
2389 };
2390 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2391 if !wave_blocked(&results) {
2392 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2393 .await;
2394 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2395 .await?;
2396 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2397 .await;
2398 }
2399 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2400
2401 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2402 && results
2403 .iter()
2404 .any(|(_, _, out)| FailClass::of(out).is_some());
2405 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2406 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2407 if self
2411 .state
2412 .candidate_preservation
2413 .contains_key(&self.state.candidates[i].label)
2414 {
2415 self.state.seats.insert(seat.key.clone(), seat);
2416 }
2417 continue;
2418 }
2419 let seat_key = seat.key.clone();
2420 let agent = seat.agent.clone();
2429 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2430 self.state.seats.insert(seat.key.clone(), seat);
2431 let label = self.state.candidates[i].label;
2432 self.state.interrupted_implementations.remove(&label);
2433
2434 let (summary, duration, failed, verified_claim) = match out {
2435 AgentOutcome::Ok(o) => {
2436 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2437 let failed = (!o.usable()).then(|| {
2438 if o.timed_out {
2439 "agent timed out".to_owned()
2440 } else {
2441 format!(
2442 "agent exited with {}",
2443 crate::notices::exit_status(o.exit_code)
2444 )
2445 }
2446 });
2447 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2448 (text, o.duration_ms, failed, verified_claim)
2449 }
2450 AgentOutcome::Dropped(o) => {
2456 let why = o
2457 .dropped
2458 .as_ref()
2459 .map(|d| d.why.as_str())
2460 .unwrap_or("the CLI ended the stream without delivering its answer");
2461 (
2462 String::new(),
2463 o.duration_ms,
2464 Some(format!("the CLI dropped the stream ({why})")),
2465 None,
2466 )
2467 }
2468 AgentOutcome::Quota(o) => {
2469 self.state.quota.push(QuotaLoss {
2470 seat: seat_key,
2471 node: "implement".to_owned(),
2472 at: Timestamp::now(),
2473 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2474 });
2475 (
2476 String::new(),
2477 o.duration_ms,
2478 Some("rate limited (quota); produced no change".to_owned()),
2479 None,
2480 )
2481 }
2482 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2483 (String::new(), 0, Some(e), None)
2484 }
2485 };
2486
2487 let c = &mut self.state.candidates[i];
2490 if !exhausted_the_fallback_chain {
2491 c.agent = agent;
2492 }
2493 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2494 c.duration_ms = duration;
2495 c.failed = failed;
2496 c.verified_noop = verified_claim;
2497 self.state
2498 .candidate_preservation
2499 .insert(label, "awaiting persistence".to_owned());
2500 }
2501 self.state.save()?;
2502 self.persist_candidates().await?;
2503
2504 if resource_blocked {
2505 return self.stop_for_resource();
2506 }
2507 if selected_failed {
2508 self.state.status = RunStatus::Blocked;
2509 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2510 self.state.save()?;
2511 bail!(
2512 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2513 );
2514 }
2515 self.after_implement()
2516 }
2517
2518 async fn persist_candidates(&mut self) -> Result<()> {
2520 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2521 for label in labels {
2522 let i = self
2523 .state
2524 .candidates
2525 .iter()
2526 .position(|c| c.label == label)
2527 .context("pending candidate is missing")?;
2528 let c = self.state.candidates[i].clone();
2529 let interrupted = self.state.interrupted_implementations.contains(&label);
2530 let result = async {
2531 let wait = crate::repo_lock::WaitPolicy::new(
2532 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2533 || false,
2534 );
2535 let _coordination =
2536 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2537 if let Err(e) = git::snapshot_uncommitted(
2540 &c.worktree,
2541 &self
2542 .state
2543 .dir()
2544 .join("artifacts")
2545 .join(format!("candidate-{label}-recovery")),
2546 )
2547 .await
2548 {
2549 self.state.event(
2550 "preserve",
2551 format!("candidate {label} snapshot incomplete: {e:#}"),
2552 );
2553 }
2554 let rescued = git::rescue_commit(
2555 &c.worktree,
2556 &format!("magi: candidate {label} (uncommitted work)"),
2557 )
2558 .await?;
2559 self.state.note_withheld("implement", &rescued.withheld);
2560 let base = &self.state.base_commit;
2561 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2562 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2563 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2564 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2565 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2566 if c.empty && commits > 0 && !patch.trim().is_empty()
2567 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2568 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2569 || self.state.tally.is_some())
2570 {
2571 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2575 }
2576 let c = &mut self.state.candidates[i];
2577 c.commits = commits;
2578 c.stat = stat;
2579 c.files = files;
2580 let no_changes = commits == 0 || patch.trim().is_empty();
2581 c.empty = no_changes && !interrupted;
2582 if interrupted && no_changes {
2583 c.failed = None;
2584 c.verified_noop = None;
2585 }
2586 if !c.empty {
2587 c.failed = None;
2588 c.verified_noop = None;
2589 }
2590 self.state.event(
2591 "preserve",
2592 format!("candidate {label}: {files} files, {commits} commits"),
2593 );
2594 Ok::<_, anyhow::Error>(())
2595 }
2596 .await;
2597 match result {
2598 Ok(()) => {
2599 self.state.candidate_preservation.remove(&label);
2600 if !interrupted || self.state.candidates[i].files > 0 {
2601 self.state.interrupted_implementations.remove(&label);
2602 }
2603 }
2604 Err(e) => {
2605 let reason = format!("candidate {label} preservation failed: {e:#}");
2606 self.state.event("preserve", &reason);
2607 self.state.candidate_preservation.insert(label, reason);
2608 }
2609 }
2610 if self.state.candidate_preservation.is_empty()
2611 && self.state.status == RunStatus::Blocked
2612 && self.state.candidates.iter().all(|c| c.failed.is_none())
2613 {
2614 self.state.status = RunStatus::Implementing;
2618 self.state.event(
2619 "preserve",
2620 "candidate persistence recovered; implementation can continue",
2621 );
2622 }
2623 self.state.save()?;
2624 }
2625 if !self.state.candidate_preservation.is_empty() {
2626 self.state.status = RunStatus::Blocked;
2627 self.state.save()?;
2628 bail!(
2629 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2630 );
2631 }
2632 Ok(())
2633 }
2634
2635 async fn resume_undelivered(
2663 &mut self,
2664 results: &mut [(usize, SeatState, AgentOutcome)],
2665 sent: &[SeatJob],
2666 prompts: &Prompts,
2667 run_id: &str,
2668 ) {
2669 for (wi, seat, out) in results.iter_mut() {
2670 let Some(dropped) = (match &*out {
2671 AgentOutcome::Dropped(o) => o.dropped.clone(),
2672 _ => None,
2673 }) else {
2674 continue;
2675 };
2676 let Some(job) = sent.get(*wi) else { continue };
2677 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2679 self.state.event(
2680 "implement",
2681 format!(
2682 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2683 work is in the tree",
2684 seat.key, dropped.output_tokens, dropped.why
2685 ),
2686 );
2687 continue;
2688 }
2689 if !has_context(&job.spec, seat, job.sessions) {
2697 self.state.event(
2698 "implement",
2699 format!(
2700 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2701 is no session left to resume",
2702 seat.key, dropped.output_tokens, dropped.why
2703 ),
2704 );
2705 continue;
2706 }
2707 self.state.event(
2708 "implement",
2709 format!(
2710 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2711 conversation",
2712 seat.key, dropped.output_tokens, dropped.why
2713 ),
2714 );
2715 let mut retry = job.clone();
2716 retry.seat = seat.clone();
2717 retry.prompt = prompt::resume_after_drop(&dropped.why);
2718 retry.timeout = retry_budget(job.timeout, true);
2719 retry.stem = format!("{}-resume", job.stem);
2720 let cache = self.state.config.cache_dir();
2721 let ctx = WaveCtx {
2722 carry_seats: false,
2723 pauses: Some((&self.pause, &self.interrupt)),
2724 run: run_id,
2725 node: "implement",
2726 prompts,
2727 cache: cache.as_deref(),
2728 round: None,
2729 };
2730 let (resumed_seat, resumed) =
2731 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2732 *seat = resumed_seat;
2733 *out = resumed;
2734 }
2735 }
2736
2737 async fn resume_seat_handovers(
2802 &mut self,
2803 results: &mut [(usize, SeatState, AgentOutcome)],
2804 sent: &mut [SeatJob],
2805 prompts: &Prompts,
2806 run_id: &str,
2807 ) -> Result<()> {
2808 let instruction = seeded_instruction(&self.state);
2809 let language = self.state.config.graph.language.clone();
2810 let github_language = self.state.config.graph.github_language.clone();
2811 let brief = self
2812 .state
2813 .advice
2814 .as_ref()
2815 .and_then(|a| a.synthesis.as_deref())
2816 .map(str::to_owned);
2817 let attachments = self.state.attachments.clone();
2818 for (wi, seat, out) in results.iter_mut() {
2819 let others: BTreeSet<String> = sent
2822 .iter()
2823 .enumerate()
2824 .filter(|(j, _)| j != wi)
2825 .map(|(_, j)| j.spec.id.clone())
2826 .collect();
2827 let Some(job) = sent.get_mut(*wi) else {
2828 continue;
2829 };
2830 let start = self
2835 .roles
2836 .implementer_roster
2837 .iter()
2838 .position(|s| s.id == job.spec.id)
2839 .unwrap_or(0);
2840 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2841 let mut fallback_attempt = 0usize;
2842 let mut prev: Option<FailClass> = None;
2843 while let Some(cur) = FailClass::of(&*out) {
2844 if !should_hand_over(prev.as_ref(), &cur) {
2845 break;
2846 }
2847 let Some(next) =
2848 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2849 .cloned()
2850 else {
2851 break;
2852 };
2853 tried.insert(next.id.clone());
2854 fallback_attempt += 1;
2855
2856 let coordination = self.coordinate(&job.cwd).await?;
2857 if let Ok(r) = git::rescue_commit(
2858 &job.cwd,
2859 &format!(
2860 "magi: candidate {} (uncommitted work before {} fallback)",
2861 seat.key,
2862 if cur == FailClass::Quota {
2863 "quota"
2864 } else {
2865 "handover"
2866 }
2867 ),
2868 )
2869 .await
2870 {
2871 self.state.note_withheld("implement", &r.withheld);
2872 }
2873 drop(coordination);
2874
2875 record_handover(
2876 &mut self.state,
2877 "implement",
2878 &seat.key,
2879 &seat.agent,
2880 &next.id,
2881 &cur,
2882 &fail_reason(&*out),
2883 );
2884 prev = Some(cur.clone());
2885
2886 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2887 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2888 job.spec = next.clone();
2896 let mut retry = job.clone();
2897 retry.seat = new_seat;
2898 retry.prompt = prompt::implement_in(
2899 &instruction,
2900 &job.cwd.to_string_lossy(),
2901 &language,
2902 &github_language,
2903 brief.as_deref(),
2904 &attachments,
2905 );
2906 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2907 let cache = self.state.config.cache_dir();
2908 let ctx = WaveCtx {
2909 carry_seats: false,
2910 pauses: Some((&self.pause, &self.interrupt)),
2911 run: run_id,
2912 node: "implement",
2913 prompts,
2914 cache: cache.as_deref(),
2915 round: None,
2916 };
2917 let (fallback_seat, fallback_out) = run_one(
2918 retry,
2919 Arc::clone(&self.sem),
2920 &ctx,
2921 &mut self.state,
2922 fallback_attempt,
2923 )
2924 .await;
2925 *seat = fallback_seat;
2926 *out = fallback_out;
2927 }
2928 }
2929 Ok(())
2930 }
2931
2932 async fn resume_unconfirmed_commands(
2956 &mut self,
2957 results: &mut [(usize, SeatState, AgentOutcome)],
2958 sent: &[SeatJob],
2959 prompts: &Prompts,
2960 run_id: &str,
2961 ) {
2962 for (wi, seat, out) in results.iter_mut() {
2963 let AgentOutcome::Ok(o) = &*out else {
2964 continue;
2965 };
2966 if !has_unconfirmed_command(&o.commands) {
2967 continue;
2968 }
2969 let Some(job) = sent.get(*wi) else { continue };
2970 if !has_context(&job.spec, seat, job.sessions) {
2971 self.state.event(
2972 "implement",
2973 format!(
2974 "{}: the reply named a command whose own CLI never confirmed the exit \
2975 status of, but there is no session left to resume",
2976 seat.key
2977 ),
2978 );
2979 continue;
2980 }
2981 self.state.event(
2982 "implement",
2983 format!(
2984 "{}: the reply named a command whose own CLI never confirmed the exit \
2985 status of; resuming the conversation",
2986 seat.key
2987 ),
2988 );
2989 let mut retry = job.clone();
2990 retry.seat = seat.clone();
2991 retry.prompt = prompt::resume_incomplete(
2992 "a command in your last reply had no confirmed exit status",
2993 );
2994 retry.timeout = retry_budget(job.timeout, true);
2995 retry.stem = format!("{}-confirm", job.stem);
2996 let cache = self.state.config.cache_dir();
2997 let ctx = WaveCtx {
2998 carry_seats: false,
2999 pauses: Some((&self.pause, &self.interrupt)),
3000 run: run_id,
3001 node: "implement",
3002 prompts,
3003 cache: cache.as_deref(),
3004 round: None,
3005 };
3006 let (resumed_seat, resumed) =
3007 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3008 *seat = resumed_seat;
3009 *out = resumed;
3010 }
3011 }
3012
3013 async fn continue_fix_report(
3034 &mut self,
3035 mut seat: SeatState,
3036 parse_err: String,
3037 job: &SeatJob,
3038 prompts: &Prompts,
3039 run_id: &str,
3040 round: usize,
3041 ) -> (
3042 SeatState,
3043 Option<FixReport>,
3044 Option<String>,
3045 ContinuationRecord,
3046 ) {
3047 let mut last_err = parse_err;
3048 let mut cumulative_wait_ms = 0u64;
3049 let mut attempts = 0usize;
3050 loop {
3051 if !has_context(&job.spec, &seat, job.sessions) {
3052 self.state.event(
3053 "fix",
3054 format!(
3055 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3056 session left to resume into"
3057 ),
3058 );
3059 let outcome = if attempts == 0 {
3060 ContinuationOutcome::NoSession
3061 } else {
3062 ContinuationOutcome::Exhausted
3063 };
3064 return (
3065 seat,
3066 None,
3067 Some(format!("unparsable fix report: {last_err}")),
3068 ContinuationRecord {
3069 attempts,
3070 cumulative_wait_ms,
3071 outcome,
3072 },
3073 );
3074 }
3075 if attempts >= MAX_FIX_CONTINUATIONS {
3076 self.state.event(
3077 "fix",
3078 format!(
3079 "round {round}: fixer's reply still had no adoption report after \
3080 {attempts} continuation(s) ({last_err}); giving up"
3081 ),
3082 );
3083 return (
3084 seat,
3085 None,
3086 Some(format!(
3087 "unparsable fix report after {attempts} continuation(s): {last_err}"
3088 )),
3089 ContinuationRecord {
3090 attempts,
3091 cumulative_wait_ms,
3092 outcome: ContinuationOutcome::Exhausted,
3093 },
3094 );
3095 }
3096 attempts += 1;
3097 self.state.event(
3098 "fix",
3099 format!(
3100 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3101 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3102 ),
3103 );
3104 let mut retry = job.clone();
3105 retry.seat = seat.clone();
3106 retry.prompt = prompt::resume_incomplete(&last_err);
3107 retry.timeout = retry_budget(job.timeout, true);
3108 retry.stem = format!("{}-continue{attempts}", job.stem);
3109 let cache = self.state.config.cache_dir();
3110 let ctx = WaveCtx {
3111 carry_seats: false,
3112 pauses: Some((&self.pause, &self.interrupt)),
3113 run: run_id,
3114 node: "fix",
3115 prompts,
3116 cache: cache.as_deref(),
3117 round: Some(round),
3118 };
3119 let (resumed_seat, resumed_out) = run_one(
3120 retry,
3121 Arc::clone(&self.sem),
3122 &ctx,
3123 &mut self.state,
3124 attempts,
3125 )
3126 .await;
3127 seat = resumed_seat;
3128 match resumed_out {
3129 AgentOutcome::Ok(o) => {
3130 cumulative_wait_ms += o.duration_ms;
3131 match verdict::extract_json::<FixReport>(&o.text) {
3132 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3133 self.state.event(
3134 "fix",
3135 format!(
3136 "round {round}: fixer's adoption report recovered after \
3137 {attempts} continuation(s)"
3138 ),
3139 );
3140 return (
3141 seat,
3142 Some(report),
3143 None,
3144 ContinuationRecord {
3145 attempts,
3146 cumulative_wait_ms,
3147 outcome: ContinuationOutcome::Resumed,
3148 },
3149 );
3150 }
3151 Ok(_) => {
3159 last_err = "the reply parsed, but it reported a command whose own CLI \
3160 never confirmed an exit status"
3161 .to_owned();
3162 }
3163 Err(e) => last_err = e.to_string(),
3164 }
3165 }
3166 AgentOutcome::Quota(o) => {
3167 cumulative_wait_ms += o.duration_ms;
3168 self.state.quota.push(QuotaLoss {
3169 seat: seat.key.clone(),
3170 node: "fix".to_owned(),
3171 at: Timestamp::now(),
3172 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3173 });
3174 self.state.event(
3175 "fix",
3176 format!(
3177 "round {round}: continuation rate limited (quota); not retrying now"
3178 ),
3179 );
3180 return (
3181 seat,
3182 None,
3183 Some("rate limited (quota) while recovering the fix report".to_owned()),
3184 ContinuationRecord {
3185 attempts,
3186 cumulative_wait_ms,
3187 outcome: ContinuationOutcome::QuotaLost,
3188 },
3189 );
3190 }
3191 AgentOutcome::Dropped(o) => {
3192 cumulative_wait_ms += o.duration_ms;
3193 let why = o
3194 .dropped
3195 .as_ref()
3196 .map(|d| d.why.as_str())
3197 .unwrap_or("the CLI ended the stream without delivering its answer");
3198 last_err = format!("the CLI dropped the stream ({why})");
3199 }
3200 AgentOutcome::Failed(e) => last_err = e,
3201 AgentOutcome::ResourceBlocked(e) => {
3204 return (
3205 seat,
3206 None,
3207 Some(format!("the continuation was not started: {e}")),
3208 ContinuationRecord {
3209 attempts: attempts.saturating_sub(1),
3210 cumulative_wait_ms,
3211 outcome: ContinuationOutcome::NoSession,
3212 },
3213 );
3214 }
3215 }
3216 }
3217 }
3218
3219 fn after_implement(&mut self) -> Result<()> {
3220 if !self.state.candidate_preservation.is_empty() {
3221 bail!("candidate preservation incomplete; judging is unsafe");
3222 }
3223 if self.state.leaks.is_empty() {
3225 let cfg = self.state.config.blind.clone();
3226 let mut leaks = Vec::new();
3227 for c in &self.state.candidates {
3228 let Some(patch) =
3229 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3230 else {
3231 continue;
3232 };
3233 leaks.extend(blind::scan(
3234 &format!("candidate {} patch", c.label),
3235 &patch,
3236 &cfg.vendor_tokens,
3237 ));
3238 }
3239 if !leaks.is_empty() {
3240 let summary = leaks
3241 .iter()
3242 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3243 .collect::<Vec<_>>()
3244 .join(", ");
3245 match cfg.on_leak {
3246 LeakPolicy::Fail => {
3247 self.state.status = RunStatus::Failed;
3248 self.state
3249 .event("blind", format!("vendor text in a patch: {summary}"));
3250 self.state.leaks = leaks;
3251 self.state.save()?;
3252 self.settle_questions();
3253 bail!(
3254 "blind.on_leak = \"fail\" and vendor text reached a \
3255 judged patch: {summary}"
3256 );
3257 }
3258 LeakPolicy::Redact => self.state.event(
3259 "blind",
3260 format!("redacting vendor text for judging: {summary}"),
3261 ),
3262 LeakPolicy::Warn => self.state.event(
3263 "blind",
3264 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3265 ),
3266 }
3267 self.state.leaks = leaks;
3268 }
3269 }
3270
3271 if self.state.viable().is_empty() {
3272 if self.state.all_candidates_verified_noop() {
3273 self.state.status = RunStatus::VerifiedNoop;
3284 self.state.save()?;
3285 self.settle_questions();
3286 return Ok(());
3287 }
3288 self.state.status = RunStatus::Failed;
3289 self.state.save()?;
3290 self.settle_questions();
3291 bail!("no candidate produced a change; nothing to judge");
3292 }
3293 self.state.status = RunStatus::Judging;
3294 self.state.save()?;
3295 Ok(())
3296 }
3297
3298 async fn judge(&mut self) -> Result<()> {
3301 let run_id = self.state.id.clone();
3306 let prompts = self.state.config.prompts.clone();
3307 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3308 return Ok(());
3309 }
3310 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3311 if viable.len() == 1 {
3312 self.state.judge_skipped = true;
3319 self.state.event(
3320 "judge",
3321 format!(
3322 "only candidate {} produced a change; judging skipped",
3323 viable[0].label
3324 ),
3325 );
3326 self.state.save()?;
3327 return Ok(());
3328 }
3329 self.state.status = RunStatus::Judging;
3330
3331 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3332 let language = self.state.config.graph.language.clone();
3333 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3334 let sessions = self.state.config.graph.sessions;
3335 let artifacts = agent::artifacts_dir(&self.state.dir());
3336 let root = self.state.worktree_root();
3337 let base_short = short(&self.state.base_commit);
3338
3339 let mut jobs = Vec::new();
3340 let mut orders = Vec::new();
3341 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3342 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3343 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3344 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3345 let seat_key = format!("judge-{}", j + 1);
3346 let seat = self.seat(&seat_key, &spec.id);
3347 jobs.push(SeatJob {
3348 prompt: prompt::judge(
3349 &self.state.instruction,
3350 &views,
3351 self.roles.judges.len(),
3352 &base_short,
3353 &language,
3354 ),
3355 spec,
3356 seat,
3357 cwd: root.join(format!("judge-{}", j + 1)),
3358 timeout,
3359 allow_write: false,
3360 sessions,
3361 artifacts: artifacts.clone(),
3362 stem: format!("judge-{}", j + 1),
3363 handover: None,
3364 });
3365 }
3366
3367 self.state.event(
3368 "judge",
3369 format!(
3370 "{} judges ranking {} candidates blind",
3371 jobs.len(),
3372 viable.len()
3373 ),
3374 );
3375 let labels_for_check = labels.clone();
3376 let mut quota_losses = Vec::new();
3377 let cache = self.state.config.cache_dir();
3378 let ctx = WaveCtx {
3379 carry_seats: false,
3380 pauses: Some((&self.pause, &self.interrupt)),
3381 run: &run_id,
3382 node: "judge",
3383 prompts: &prompts,
3384 cache: cache.as_deref(),
3385 round: None,
3386 };
3387 let results = ask_json_wave::<Ranking>(
3388 jobs,
3389 Arc::clone(&self.sem),
3390 self.state.config.graph.retries,
3391 &self.roles.judge_roster,
3392 &ctx,
3393 &mut quota_losses,
3394 &mut self.state,
3395 &move |r: &Ranking| r.validate(&labels_for_check),
3396 )
3397 .await;
3398 if self.state.resource_block.is_some() {
3399 return self.stop_for_resource();
3400 }
3401 self.state.quota.extend(quota_losses);
3402
3403 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3404 let agent_id = seat.agent.clone();
3405 self.state.seats.insert(seat.key.clone(), seat);
3406 let mut record = Judgement {
3407 judge: j + 1,
3408 seat: format!("judge-{}", j + 1),
3409 agent: agent_id,
3410 ranking: Vec::new(),
3411 reasons: BTreeMap::new(),
3412 confidence: None,
3413 strengths: BTreeMap::new(),
3414 order: orders[j].clone(),
3415 failed: None,
3416 duration_ms: 0,
3417 };
3418 match res {
3419 Ok((ranking, out)) => {
3420 record.ranking = ranking.normalized();
3421 record.reasons = ranking.reasons;
3422 record.strengths = ranking.strengths;
3423 record.confidence = ranking.confidence;
3424 record.duration_ms = out.duration_ms;
3425 self.state.event(
3426 "judge",
3427 format!(
3428 "judge {} ranked {}",
3429 j + 1,
3430 record.ranking.iter().collect::<String>()
3431 ),
3432 );
3433 }
3434 Err(e) => {
3435 record.failed = Some(e.to_string());
3436 self.state
3437 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3438 }
3439 }
3440 self.state.judgements.push(record);
3441 self.state.save()?;
3442 }
3443 Ok(())
3444 }
3445
3446 async fn deliberate(&mut self) -> Result<()> {
3449 let run_id = self.state.id.clone();
3454 let prompts = self.state.config.prompts.clone();
3455 if !self.state.deliberation.is_empty() {
3456 return Ok(());
3457 }
3458 let tops: Vec<char> = self
3459 .state
3460 .judgements
3461 .iter()
3462 .filter_map(|j| j.ranking.first().copied())
3463 .collect();
3464 let rounds = self.state.config.graph.deliberate_rounds;
3465 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3466 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3467 self.state.event(
3468 "deliberate",
3469 format!("judges agreed on {} outright; no deliberation", tops[0]),
3470 );
3471 }
3472 self.state.status = RunStatus::Voting;
3473 self.state.save()?;
3474 return Ok(());
3475 }
3476
3477 self.state.status = RunStatus::Deliberating;
3478 self.state.event(
3479 "deliberate",
3480 format!(
3481 "split: first choices were {} — opening {rounds} round(s)",
3482 tops.iter().collect::<String>()
3483 ),
3484 );
3485
3486 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3487 let language = self.state.config.graph.language.clone();
3488 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3489 let sessions = self.state.config.graph.sessions;
3490 let artifacts = agent::artifacts_dir(&self.state.dir());
3491 let root = self.state.worktree_root();
3492 let base_short = short(&self.state.base_commit);
3493
3494 for round in 1..=rounds {
3498 let mut turns: Vec<DeliberationTurn> = Vec::new();
3499 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3500 if self.state.judgements[j].failed.is_some() {
3501 continue;
3502 }
3503 let seat_key = format!("judge-{}", j + 1);
3504 let spec = self.occupant(&seat_key, spec);
3505 let mut seat = self.seat(&seat_key, &spec.id);
3506 let transcript = self.transcript(&turns, j);
3507 let build = |context: Option<&str>| {
3508 prompt::deliberate(
3509 &self.state.instruction,
3510 context,
3511 &transcript,
3512 round,
3513 rounds,
3514 &language,
3515 )
3516 };
3517 let block = self.candidate_block(&viable, &base_short);
3518 let full = build(Some(&block));
3519 let text = if has_context(&spec, &seat, sessions) {
3520 build(None)
3521 } else {
3522 full.clone()
3523 };
3524 let job = SeatJob {
3525 spec,
3526 seat: seat.clone(),
3527 prompt: text,
3528 cwd: root.join(format!("judge-{}", j + 1)),
3529 timeout,
3530 allow_write: false,
3531 sessions,
3532 artifacts: artifacts.clone(),
3533 stem: format!("delib-{round}-judge-{}", j + 1),
3534 handover: Some(full),
3535 };
3536 let cache = self.state.config.cache_dir();
3537 let ctx = WaveCtx {
3538 carry_seats: false,
3539 pauses: Some((&self.pause, &self.interrupt)),
3540 run: &run_id,
3541 node: "deliberate",
3542 prompts: &prompts,
3543 cache: cache.as_deref(),
3544 round: None,
3545 };
3546 let mut losses = Vec::new();
3550 let mut results = ask_wave_with::<String>(
3551 vec![job],
3552 Arc::clone(&self.sem),
3553 0,
3554 &self.roles.judge_roster,
3555 &ctx,
3556 &mut losses,
3557 &mut self.state,
3558 &|text: &str| {
3559 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3560 },
3561 )
3562 .await;
3563 self.state.quota.extend(losses);
3564 let (updated, res, _) = results.pop().expect("one job in, one result out");
3565 seat = updated;
3566 let agent_id = seat.agent.clone();
3567 self.state.seats.insert(seat.key.clone(), seat);
3568 let body = match res {
3569 Ok((body, _)) => body,
3570 Err(e) => {
3573 self.state
3574 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3575 continue;
3576 }
3577 };
3578 let tentative = verdict::extract_json::<Position>(&body)
3579 .ok()
3580 .and_then(|p| p.tentative)
3581 .and_then(|s| s.trim().chars().next())
3582 .map(|c| c.to_ascii_uppercase());
3583 self.state.event(
3584 "deliberate",
3585 format!(
3586 "round {round}: judge {} now favours {}",
3587 j + 1,
3588 tentative.map_or("—".to_owned(), |c| c.to_string())
3589 ),
3590 );
3591 turns.push(DeliberationTurn {
3592 judge: j + 1,
3593 agent: agent_id,
3594 body: blind::sanitize_prose(&body, &self.state.config.blind),
3595 tentative,
3596 });
3597 }
3598 self.state
3599 .deliberation
3600 .push(DeliberationRound { round, turns });
3601 self.state.save()?;
3602 }
3603
3604 self.state.status = RunStatus::Voting;
3605 self.state.save()?;
3606 Ok(())
3607 }
3608
3609 async fn vote(&mut self) -> Result<()> {
3612 let run_id = self.state.id.clone();
3617 let prompts = self.state.config.prompts.clone();
3618 if !self.state.votes.is_empty() {
3619 return Ok(());
3620 }
3621 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3622 if viable.len() == 1 {
3623 return Ok(());
3624 }
3625 self.state.status = RunStatus::Voting;
3626
3627 let language = self.state.config.graph.language.clone();
3628 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3629 let sessions = self.state.config.graph.sessions;
3630 let artifacts = agent::artifacts_dir(&self.state.dir());
3631 let root = self.state.worktree_root();
3632 let base_short = short(&self.state.base_commit);
3633 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3634
3635 let mut jobs = Vec::new();
3636 let mut seats_at = Vec::new();
3637 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3638 if self
3639 .state
3640 .judgements
3641 .get(j)
3642 .is_some_and(|r| r.failed.is_some())
3643 {
3644 continue;
3645 }
3646 let seat_key = format!("judge-{}", j + 1);
3647 let spec = self.occupant(&seat_key, spec);
3648 let seat = self.seat(&seat_key, &spec.id);
3649 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3650 let text = if has_context(&spec, &seat, sessions) {
3651 prompt::final_vote(&viable, &language)
3652 } else {
3653 full.clone()
3654 };
3655 jobs.push(SeatJob {
3656 spec,
3657 seat,
3658 prompt: text,
3659 cwd: root.join(format!("judge-{}", j + 1)),
3660 timeout,
3661 allow_write: false,
3662 sessions,
3663 artifacts: artifacts.clone(),
3664 stem: format!("vote-judge-{}", j + 1),
3665 handover: Some(full),
3666 });
3667 seats_at.push(j);
3668 }
3669
3670 self.state.event(
3671 "vote",
3672 format!(
3673 "collecting {} final votes one by one, privately",
3674 jobs.len()
3675 ),
3676 );
3677 let allowed = viable.clone();
3678 let mut quota_losses = Vec::new();
3679 let cache = self.state.config.cache_dir();
3680 let ctx = WaveCtx {
3681 carry_seats: false,
3682 pauses: Some((&self.pause, &self.interrupt)),
3683 run: &run_id,
3684 node: "vote",
3685 prompts: &prompts,
3686 cache: cache.as_deref(),
3687 round: None,
3688 };
3689 let results = ask_json_wave::<FinalVote>(
3690 jobs,
3691 Arc::clone(&self.sem),
3692 self.state.config.graph.retries,
3693 &self.roles.judge_roster,
3694 &ctx,
3695 &mut quota_losses,
3696 &mut self.state,
3697 &move |v: &FinalVote| match v.label() {
3698 Some(c) if allowed.contains(&c) => Ok(()),
3699 other => bail!("vote {other:?} is not one of {allowed:?}"),
3700 },
3701 )
3702 .await;
3703 if self.state.resource_block.is_some() {
3704 return self.stop_for_resource();
3705 }
3706 self.state.quota.extend(quota_losses);
3707
3708 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3709 let agent_id = seat.agent.clone();
3710 self.state.seats.insert(seat.key.clone(), seat);
3711 let initial = self
3712 .state
3713 .judgements
3714 .get(j)
3715 .and_then(|r| r.ranking.first().copied());
3716 let mut record = VoteRecord {
3717 judge: j + 1,
3718 agent: agent_id,
3719 vote: None,
3720 reason: String::new(),
3721 changed: false,
3722 };
3723 match res {
3724 Ok((v, _)) => {
3725 record.vote = v.label();
3726 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3727 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3728 self.state.event(
3729 "vote",
3730 format!(
3731 "judge {} voted {}{}",
3732 j + 1,
3733 record.vote.unwrap_or('?'),
3734 if record.changed { " (changed)" } else { "" }
3735 ),
3736 );
3737 }
3738 Err(e) => {
3739 self.state
3740 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3741 }
3742 }
3743 self.state.votes.push(record);
3744 self.state.save()?;
3745 }
3746 Ok(())
3747 }
3748
3749 fn tally(&mut self) -> Result<()> {
3752 if self.state.tally.is_some() {
3753 return Ok(());
3754 }
3755 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3756 let tops: Vec<char> = self
3757 .state
3758 .judgements
3759 .iter()
3760 .filter_map(|j| j.ranking.first().copied())
3761 .collect();
3762 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3763
3764 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3767 let mut cast: Vec<char> = Vec::new();
3768 for (i, j) in self.state.judgements.iter().enumerate() {
3769 let vote = self
3770 .state
3771 .votes
3772 .iter()
3773 .find(|v| v.judge == i + 1)
3774 .and_then(|v| v.vote)
3775 .or_else(|| j.ranking.first().copied());
3776 if let Some(v) = vote {
3777 *first_choice.entry(v).or_insert(0) += 1;
3778 cast.push(v);
3779 }
3780 }
3781
3782 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3783 for j in &self.state.judgements {
3784 let n = j.ranking.len();
3785 for (pos, label) in j.ranking.iter().enumerate() {
3786 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3787 }
3788 }
3789
3790 let best = first_choice.values().copied().max().unwrap_or(0);
3791 let mut leaders: Vec<char> = first_choice
3792 .iter()
3793 .filter(|(_, v)| **v == best)
3794 .map(|(k, _)| *k)
3795 .collect();
3796 let mut tie_break = None;
3797 if leaders.len() > 1 {
3798 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3799 let borda_leaders: Vec<char> = leaders
3800 .iter()
3801 .copied()
3802 .filter(|l| borda[l] == top_borda)
3803 .collect();
3804 tie_break = Some(if borda_leaders.len() == 1 {
3805 format!(
3806 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3807 leaders.len()
3808 )
3809 } else {
3810 format!(
3811 "{} way tie on both first-choice votes and Borda points, broken by label order",
3812 leaders.len()
3813 )
3814 });
3815 leaders = borda_leaders;
3816 leaders.sort_unstable();
3817 }
3818 let winner = *leaders
3819 .first()
3820 .or(viable.first())
3821 .context("no candidate to declare a winner from")?;
3822
3823 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3824 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3825 let deliberated = !self.state.deliberation.is_empty();
3826
3827 let quota_seats: std::collections::BTreeSet<&str> =
3831 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3832 let mut present = 0usize;
3833 for (i, j) in self.state.judgements.iter().enumerate() {
3834 if quota_seats.contains(j.seat.as_str()) {
3835 continue;
3836 }
3837 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3838 let voted = self
3839 .state
3840 .votes
3841 .iter()
3842 .any(|v| v.judge == i + 1 && v.vote.is_some());
3843 if ranked || voted {
3844 present += 1;
3845 }
3846 }
3847 let needs_quorum = viable.len() > 1;
3853 let judges_total = if needs_quorum {
3854 self.roles.judges.len()
3855 } else {
3856 0
3857 };
3858 let quorum = if needs_quorum {
3859 judges_total / 2 + 1
3860 } else {
3861 0
3862 };
3863 let met_quorum = !needs_quorum || present >= quorum;
3864 let uncontested = (!needs_quorum).then(|| {
3865 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3866 });
3867
3868 self.state.event(
3869 "tally",
3870 match &uncontested {
3871 Some(reason) => format!("winner {winner} — {reason}"),
3872 None => format!(
3873 "winner {winner} — votes {} | initial {} | {} changed | \
3874 {present}/{judges_total} judges{}",
3875 first_choice
3876 .iter()
3877 .map(|(k, v)| format!("{k}:{v}"))
3878 .collect::<Vec<_>>()
3879 .join(" "),
3880 if unanimous_initial {
3881 "unanimous"
3882 } else {
3883 "split"
3884 },
3885 changed_votes,
3886 if met_quorum {
3887 String::new()
3888 } else {
3889 format!(" — below quorum ({quorum} required)")
3890 },
3891 ),
3892 },
3893 );
3894 if !met_quorum {
3895 self.state.event(
3896 "stall",
3897 format!(
3898 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3899 the run stops here, resumable"
3900 ),
3901 );
3902 }
3903 self.state.tally = Some(Tally {
3904 first_choice,
3905 borda,
3906 winner,
3907 rankings: tops.len(),
3908 unanimous_initial,
3909 deliberated,
3910 changed_votes,
3911 unanimous_final,
3912 tie_break,
3913 judges: judges_total,
3914 present,
3915 quorum,
3916 met_quorum,
3917 uncontested,
3918 });
3919 self.state.status = if met_quorum {
3920 RunStatus::Reviewing
3921 } else {
3922 RunStatus::Stalled
3923 };
3924 self.state.save()?;
3925 Ok(())
3926 }
3927
3928 #[allow(clippy::too_many_lines)]
3949 async fn recover_stall(&mut self) -> Result<bool> {
3950 let run_id = self.state.id.clone();
3955 let prompts = self.state.config.prompts.clone();
3956 let quota_seats: BTreeSet<&str> =
3961 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3962 let absent: Vec<String> = self
3963 .state
3964 .judgements
3965 .iter()
3966 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3967 .map(|j| j.seat.clone())
3968 .collect();
3969 if absent.is_empty() {
3970 return Ok(false);
3971 }
3972 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3973 if viable.len() <= 1 {
3974 return Ok(false);
3975 }
3976 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3977 let language = self.state.config.graph.language.clone();
3978 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3979 let sessions = self.state.config.graph.sessions;
3980 let artifacts = agent::artifacts_dir(&self.state.dir());
3981 let root = self.state.worktree_root();
3982 let base_short = short(&self.state.base_commit);
3983 let candidates: Vec<Candidate> = viable.clone();
3984
3985 let mut positions: Vec<usize> = absent
3987 .iter()
3988 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3989 .collect();
3990 if positions.is_empty() {
3991 return Ok(false);
3992 }
3993 positions.sort_unstable();
3994 positions.dedup();
3995
3996 let mut judge_jobs = Vec::new();
3998 for &j in &positions {
3999 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4000 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4001 let seat_key = format!("judge-{}", j + 1);
4002 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003 let seat = self.seat(&seat_key, &spec.id);
4004 judge_jobs.push(SeatJob {
4005 spec,
4006 seat,
4007 prompt: prompt::judge(
4008 &self.state.instruction,
4009 &views,
4010 self.roles.judges.len(),
4011 &base_short,
4012 &language,
4013 ),
4014 cwd: root.join(seat_key),
4015 timeout,
4016 allow_write: false,
4017 sessions,
4018 artifacts: artifacts.clone(),
4019 stem: format!("judge-{}-recover", j + 1),
4020 handover: None,
4021 });
4022 }
4023
4024 let labels_for_check = labels.clone();
4025 let mut judge_losses = Vec::new();
4026 let retries = self.state.config.graph.retries;
4027 let cache = self.state.config.cache_dir();
4028 let ctx = WaveCtx {
4029 carry_seats: false,
4030 pauses: Some((&self.pause, &self.interrupt)),
4031 run: &run_id,
4032 node: "judge",
4033 prompts: &prompts,
4034 cache: cache.as_deref(),
4035 round: None,
4036 };
4037 let results = ask_json_wave::<Ranking>(
4038 judge_jobs,
4039 Arc::clone(&self.sem),
4040 retries,
4041 &self.roles.judge_roster,
4042 &ctx,
4043 &mut judge_losses,
4044 &mut self.state,
4045 &move |r: &Ranking| r.validate(&labels_for_check),
4046 )
4047 .await;
4048 if self.state.resource_block.is_some() {
4049 self.stop_for_resource()?;
4050 return Ok(false);
4051 }
4052
4053 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4055 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4056 let agent_id = seat.agent.clone();
4057 self.state.seats.insert(seat.key.clone(), seat);
4058 let record = &mut self.state.judgements[j];
4059 match res {
4060 Ok((ranking, out)) => {
4061 record.agent = agent_id;
4062 record.ranking = ranking.normalized();
4063 record.reasons = ranking.reasons;
4064 record.strengths = ranking.strengths;
4065 record.confidence = ranking.confidence;
4066 record.failed = None;
4067 record.duration_ms = out.duration_ms;
4068 recovered.insert(j);
4069 self.state.event(
4070 "recover",
4071 format!("judge {} ranked again after the limit", j + 1),
4072 );
4073 }
4074 Err(e) => {
4075 self.state
4076 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4077 }
4078 }
4079 }
4080
4081 let mut vote_jobs = Vec::new();
4083 let mut vote_pos: Vec<usize> = Vec::new();
4084 for &j in &recovered {
4085 let seat_key = format!("judge-{}", j + 1);
4086 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4087 let seat = self.seat(&seat_key, &spec.id);
4088 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4089 let text = if has_context(&spec, &seat, sessions) {
4090 prompt::final_vote(&labels, &language)
4091 } else {
4092 full.clone()
4093 };
4094 vote_jobs.push(SeatJob {
4095 spec,
4096 seat,
4097 prompt: text,
4098 cwd: root.join(seat_key),
4099 timeout,
4100 allow_write: false,
4101 sessions,
4102 artifacts: artifacts.clone(),
4103 stem: format!("vote-judge-{}-recover", j + 1),
4104 handover: Some(full),
4105 });
4106 vote_pos.push(j);
4107 }
4108 let allowed = labels.clone();
4109 let mut vote_losses = Vec::new();
4110 let vote_retries = self.state.config.graph.retries;
4111 let vote_cache = self.state.config.cache_dir();
4112 let ctx = WaveCtx {
4113 carry_seats: false,
4114 pauses: Some((&self.pause, &self.interrupt)),
4115 run: &run_id,
4116 node: "vote",
4117 prompts: &prompts,
4118 cache: vote_cache.as_deref(),
4119 round: None,
4120 };
4121 let votes = ask_json_wave::<FinalVote>(
4122 vote_jobs,
4123 Arc::clone(&self.sem),
4124 vote_retries,
4125 &self.roles.judge_roster,
4126 &ctx,
4127 &mut vote_losses,
4128 &mut self.state,
4129 &move |v: &FinalVote| match v.label() {
4130 Some(c) if allowed.contains(&c) => Ok(()),
4131 other => bail!("vote {other:?} is not one of {allowed:?}"),
4132 },
4133 )
4134 .await;
4135 if self.state.resource_block.is_some() {
4136 self.stop_for_resource()?;
4137 return Ok(false);
4138 }
4139 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4140 let agent_id = seat.agent.clone();
4141 self.state.seats.insert(seat.key.clone(), seat);
4142 match res {
4143 Ok((v, _)) => {
4144 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4145 rec.vote = v.label();
4146 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4147 } else {
4148 self.state.votes.push(VoteRecord {
4149 judge: j + 1,
4150 agent: agent_id,
4151 vote: v.label(),
4152 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4153 changed: false,
4154 });
4155 }
4156 self.state.event(
4157 "recover",
4158 format!("judge {} voted again after the limit", j + 1),
4159 );
4160 }
4161 Err(e) => {
4162 self.state
4163 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4164 }
4165 }
4166 }
4167
4168 let recovered_keys: BTreeSet<String> = recovered
4172 .iter()
4173 .map(|&j| format!("judge-{}", j + 1))
4174 .collect();
4175 self.state
4176 .quota
4177 .retain(|q| !recovered_keys.contains(&q.seat));
4178 for loss in judge_losses.into_iter().chain(vote_losses) {
4182 if recovered_keys.contains(&loss.seat) {
4183 continue;
4184 }
4185 self.state.quota.retain(|q| q.seat != loss.seat);
4186 self.state.quota.push(loss);
4187 }
4188
4189 self.state.tally = None;
4191 self.tally()?;
4192 Ok(self
4193 .state
4194 .tally
4195 .as_ref()
4196 .map(|t| t.met_quorum)
4197 .unwrap_or(false))
4198 }
4199
4200 async fn graft(&mut self) -> Result<()> {
4216 let Err(e) = self.graft_inner().await else {
4217 return Ok(());
4218 };
4219 if e.is::<crate::repo_lock::ResourceBlocked>() {
4220 return Err(e);
4221 }
4222 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4223 self.state.status = RunStatus::Blocked;
4224 self.state.event("graft", format!("blocked: {e:#}"));
4225 self.state.save()?;
4226 return Err(e);
4227 }
4228 if let Some(rec) = self.state.graft.clone()
4233 && !rec.outcome.settled()
4234 {
4235 if let Some(winner) = self.state.winner().cloned() {
4236 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4237 if re.is::<crate::repo_lock::ResourceBlocked>() {
4238 return Err(re);
4239 }
4240 self.state.status = RunStatus::Blocked;
4241 self.state.event("graft", format!("blocked: {re:#}"));
4242 self.state.save()?;
4243 return Err(re);
4244 }
4245 }
4246 if let Some(g) = self.state.graft.as_mut() {
4247 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4248 }
4249 }
4250 self.state.event("graft", format!("skipped: {e:#}"));
4251 if let Err(e) = self.state.save() {
4252 tracing::warn!("save after graft: {e:#}");
4253 }
4254 Ok(())
4255 }
4256
4257 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4260 let _coordination = self.coordinate(wt).await?;
4261 crate::recovery::restore_repair(wt, pre)
4262 .await
4263 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4264 }
4265
4266 fn graft_skipped(&mut self, why: &str) {
4268 let message = format!("skipped: {why}");
4269 if !self
4270 .state
4271 .events
4272 .iter()
4273 .any(|e| e.node == "graft" && e.message == message)
4274 {
4275 self.state.event("graft", message);
4276 }
4277 }
4278
4279 async fn graft_inner(&mut self) -> Result<()> {
4280 if let Some(rec) = self.state.graft.clone()
4283 && !rec.outcome.settled()
4284 {
4285 if self.state.reviews.is_empty()
4289 && let Some(winner) = self.state.winner().cloned()
4290 {
4291 self.graft_restore(&winner.worktree, &rec.pre).await?;
4292 }
4293 if rec.outcome == GraftOutcome::WaitingForRestore {
4294 self.state.graft = None;
4295 self.state.save()?;
4296 } else {
4297 if let Some(g) = self.state.graft.as_mut() {
4298 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4299 }
4300 self.state.event(
4301 "graft",
4302 "rolled back: the previous round was interrupted; the winner is untouched",
4303 );
4304 self.state.save()?;
4305 return Ok(());
4306 }
4307 }
4308 let plan = match graft::plan(&self.state) {
4309 Ok(plan) => plan,
4310 Err(skip) => {
4311 if skip != graft::Skip::AlreadyDone {
4312 self.graft_skipped(skip.reason());
4313 self.state.save()?;
4314 }
4315 return Ok(());
4316 }
4317 };
4318 let Some(winner) = self.state.winner().cloned() else {
4319 return Ok(());
4320 };
4321 let wt = winner.worktree.clone();
4322 if !git::is_clean(&wt).await? {
4323 self.graft_skipped("the winner's worktree has uncommitted changes");
4324 self.state.save()?;
4325 return Ok(());
4326 }
4327 let pre = git::rev_parse(&wt, "HEAD").await?;
4328 self.state.graft = Some(GraftRecord {
4331 pre: pre.clone(),
4332 post: None,
4333 lines: None,
4334 limit: None,
4335 agent: None,
4336 outcome: GraftOutcome::Running,
4337 requested: graft::requested(&plan),
4338 adoption: None,
4339 adoption_error: None,
4340 });
4341 self.state.event(
4342 "graft",
4343 format!(
4344 "porting strengths of candidate(s) {} onto {}",
4345 plan.strengths
4346 .iter()
4347 .map(|(l, _, _)| l.to_string())
4348 .collect::<Vec<_>>()
4349 .join(", "),
4350 plan.winner
4351 ),
4352 );
4353 self.state.save()?;
4354
4355 let fix_prompt = prompt::graft(
4356 &self.state.instruction,
4357 plan.winner,
4358 &plan.strengths,
4359 &self.state.config.graph.language,
4360 );
4361 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4362 let sessions = self.state.config.graph.sessions;
4363 let artifacts = agent::artifacts_dir(&self.state.dir());
4364 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4365 let (_, seat, out) = self
4366 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4367 SeatJob {
4368 prompt: fix_prompt.clone(),
4369 spec,
4370 seat,
4371 cwd: wt.clone(),
4372 timeout,
4373 allow_write: true,
4374 sessions,
4375 artifacts: artifacts.clone(),
4376 stem: "graft".to_owned(),
4377 handover: None,
4378 }
4379 })
4380 .await;
4381 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4382 self.state.graft.as_mut().expect("graft record").outcome =
4385 GraftOutcome::WaitingForRestore;
4386 return self.stop_for_resource();
4387 }
4388 let mut why: Option<String> = None;
4389 match &out {
4390 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4391 why = Some("the fixer produced nothing usable".to_owned());
4392 }
4393 AgentOutcome::Ok(_) => {}
4394 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4395 AgentOutcome::Quota(o) => {
4396 self.state.quota.push(QuotaLoss {
4397 seat: seat.key.clone(),
4398 node: "graft".to_owned(),
4399 at: Timestamp::now(),
4400 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4401 });
4402 why = Some("rate limited (quota)".to_owned());
4403 }
4404 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4405 why = Some(format!(
4406 "the fixer failed: {}",
4407 e.lines()
4408 .next()
4409 .unwrap_or("")
4410 .chars()
4411 .take(160)
4412 .collect::<String>()
4413 ));
4414 }
4415 }
4416 let agent_id = seat.agent.clone();
4417 self.state.seats.insert(seat.key.clone(), seat);
4418
4419 let mut post = None;
4420 let mut lines = None;
4421 let _coordination = self.coordinate(&wt).await?;
4422 if why.is_none() {
4423 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4424 if let Ok(r) = &rescued {
4425 self.state.note_withheld("graft", &r.withheld);
4426 }
4427 let head = git::rev_parse(&wt, "HEAD").await?;
4428 let dirty = match &rescued {
4432 Ok(r) => git::status_porcelain(&wt)
4433 .await
4434 .map(|st| {
4435 st.lines().any(|l| {
4436 !r.withheld
4437 .iter()
4438 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4439 })
4440 })
4441 .unwrap_or(true),
4442 Err(_) => true,
4443 };
4444 if dirty {
4445 why = Some("the fixer left changes that could not be committed".to_owned());
4446 } else if head == pre {
4447 why = Some("the fixer changed nothing".to_owned());
4448 } else {
4449 lines = git::changed_lines(&wt, &pre, &head).await;
4450 post = Some(head);
4451 }
4452 }
4453 let (adoption, adoption_error) = match &out {
4454 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4455 _ => (None, None),
4456 };
4457 let requested = graft::requested(&plan);
4458 let outcome = match why {
4459 None => {
4460 self.state.event(
4461 "graft",
4462 format!(
4463 "applied: {}",
4464 lines.map_or_else(
4465 || "changed lines unavailable".to_owned(),
4466 |n| format!("{n} changed lines")
4467 )
4468 ),
4469 );
4470 GraftOutcome::Applied
4471 }
4472 Some(why) => {
4473 crate::recovery::restore_repair(&wt, &pre)
4476 .await
4477 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4478 self.state.event(
4479 "graft",
4480 format!("rolled back: {why}; the winner is untouched"),
4481 );
4482 GraftOutcome::RolledBack(why)
4483 }
4484 };
4485 self.state.graft = Some(GraftRecord {
4486 pre,
4487 post,
4488 lines,
4489 limit: None,
4490 agent: Some(agent_id),
4491 outcome,
4492 requested,
4493 adoption,
4494 adoption_error,
4495 });
4496 self.state.save()?;
4497 Ok(())
4498 }
4499
4500 async fn fold_losers(&mut self) -> Result<()> {
4503 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4504 return Ok(());
4505 };
4506 if !self.state.candidate_preservation.is_empty() {
4507 bail!("candidate preservation incomplete; refusing cleanup");
4508 }
4509 let repo = self.state.repo.clone();
4510 let _coordination = self.coordinate(&repo).await?;
4511 let mut folded = Vec::new();
4512 for i in 0..self.state.candidates.len() {
4513 let c = &self.state.candidates[i];
4514 if c.label == winner || c.folded {
4515 continue;
4516 }
4517 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4518 remove_preserved_worktree(&repo, &wt).await?;
4519 if git::branch_exists(&repo, &branch).await?
4520 && !git::branch_delete(&repo, &branch).await?
4521 {
4522 bail!("candidate branch removal refused");
4523 }
4524 self.state.candidates[i].folded = true;
4525 self.state.save()?;
4526 folded.push(label.to_string());
4527 }
4528 let root = self.state.worktree_root();
4530 for j in 1..=self.roles.judges.len() {
4531 let wt = root.join(format!("judge-{j}"));
4532 if wt.exists() {
4533 remove_preserved_worktree(&repo, &wt).await?;
4534 }
4535 }
4536 if self.state.config.graph.advise {
4539 for k in 1..=self.state.config.graph.advisors {
4540 let wt = root.join(format!("advisor-{k}"));
4541 if wt.exists() {
4542 remove_preserved_worktree(&repo, &wt).await?;
4543 }
4544 }
4545 }
4546 if !folded.is_empty() {
4547 self.state
4548 .event("fold", format!("folded candidates {}", folded.join(", ")));
4549 self.state.save()?;
4550 }
4551 Ok(())
4552 }
4553
4554 async fn sync_to_base(&mut self) -> Result<()> {
4594 if self.state.status == RunStatus::AlreadyInBase {
4595 return Ok(());
4596 }
4597 let conflicted = self
4598 .state
4599 .base_sync
4600 .as_ref()
4601 .is_some_and(|s| s.conflict.is_some());
4602 let Some(winner) = self.state.winner().cloned() else {
4603 return Ok(());
4604 };
4605
4606 let repo = self.state.repo.clone();
4607 let remote = self.state.config.merge.remote.clone();
4608 let base_branch = self.state.base_branch.clone();
4609 let tracking = format!("{remote}/{base_branch}");
4610
4611 git::fetch(&repo, &remote, &base_branch).await.ok();
4612 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4617 return Ok(());
4618 };
4619
4620 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4621 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4622 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4623
4624 if (behind > 0 || conflicted || head == tip)
4633 && self
4634 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4635 .await?
4636 {
4637 return Ok(());
4638 }
4639 if conflicted {
4640 return Ok(());
4641 }
4642
4643 if behind == 0 {
4644 if let Some(from) = self
4653 .state
4654 .rebase_fixes
4655 .iter()
4656 .rev()
4657 .find_map(|r| r.from.clone())
4658 && from != head
4659 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4660 .await
4661 .is_ok_and(|o| o.ok())
4662 {
4663 git::sync_to_head(&winner.worktree).await?;
4664 }
4665 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4671 if let Some(why) = &conflict {
4672 self.state.status = RunStatus::Blocked;
4673 self.state.event("land", why.clone());
4674 }
4675 self.state.base_sync = Some(BaseSync {
4676 tip,
4677 behind: 0,
4678 attempts,
4679 conflict,
4680 already_in: None,
4681 });
4682 self.state.save()?;
4683 return Ok(());
4684 }
4685
4686 if attempts >= BASE_SYNC_ROUNDS {
4687 let why = format!(
4688 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4689 rebase(s); rebasing again would only race it",
4690 winner.branch
4691 );
4692 self.state.status = RunStatus::Blocked;
4693 self.state.base_sync = Some(BaseSync {
4694 tip,
4695 behind,
4696 attempts,
4697 conflict: Some(why.clone()),
4698 already_in: None,
4699 });
4700 self.state.event("land", why);
4701 self.state.save()?;
4702 return Ok(());
4703 }
4704
4705 self.state.event(
4706 "land",
4707 format!(
4708 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4709 winner.branch
4710 ),
4711 );
4712 self.state.save()?;
4713
4714 let branch_tracking = format!("{remote}/{}", winner.branch);
4720 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4721 let remote_tip = if self.state.task_pr_branch().is_none()
4722 && matches!(&fetched_branch, Ok(o) if o.ok())
4723 {
4724 git::rev_parse(&repo, &branch_tracking).await.ok()
4725 } else {
4726 None
4727 };
4728 if let Some(theirs) = &remote_tip
4732 && !git::is_ancestor(&repo, theirs, &head).await
4733 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4734 .await
4735 .is_ok_and(|missing| missing.is_empty())
4736 {
4737 let why = format!(
4738 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4739 them",
4740 short(theirs),
4741 winner.branch
4742 );
4743 self.state.status = RunStatus::Blocked;
4744 self.state.base_sync = Some(BaseSync {
4745 tip,
4746 behind,
4747 attempts,
4748 conflict: Some(why.clone()),
4749 already_in: None,
4750 });
4751 self.state.event("land", why);
4752 self.state.save()?;
4753 return Ok(());
4754 }
4755
4756 let scratch = self.state.dir().join("base-sync");
4757 let rebased = match crate::rebase::rebase_with_fixer(
4758 &mut self.state,
4759 &scratch,
4760 &winner.branch,
4761 &tracking,
4762 )
4763 .await
4764 {
4765 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4766 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4767 Err(e) => Err(e),
4768 };
4769 let attempts = attempts + 1;
4770 match rebased {
4771 Ok(None) => {
4772 git::sync_to_head(&winner.worktree).await?;
4776 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4777 let mut conflict = None;
4778 if let Some(pinned) = &remote_tip {
4779 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4780 match pushed {
4781 Ok(o) if o.ok() => self.state.event(
4782 "land",
4783 format!("pushed rebased {} to {remote}", winner.branch),
4784 ),
4785 Ok(o) => {
4786 conflict = Some(format!(
4787 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4788 winner.branch,
4789 short(pinned),
4790 o.stderr.chars().take(600).collect::<String>()
4791 ));
4792 }
4793 Err(e) => {
4794 conflict = Some(format!(
4795 "rebased {} locally but could not push it: {e:#}",
4796 winner.branch
4797 ));
4798 }
4799 }
4800 }
4801 if let Some(why) = &conflict {
4802 self.state.status = RunStatus::Blocked;
4803 self.state.event("land", why.clone());
4804 }
4805 self.state.base_sync = Some(BaseSync {
4806 tip: tip.clone(),
4807 behind: 0,
4808 attempts,
4809 conflict,
4810 already_in: None,
4811 });
4812 self.state
4813 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4814 }
4815 Ok(Some(conflict)) => {
4816 let why = format!(
4817 "{} conflicts with {tracking} and did not rebase: {}",
4818 winner.branch,
4819 conflict.chars().take(600).collect::<String>()
4820 );
4821 self.state.status = RunStatus::Blocked;
4822 self.state.base_sync = Some(BaseSync {
4823 tip,
4824 behind,
4825 attempts,
4826 conflict: Some(why.clone()),
4827 already_in: None,
4828 });
4829 self.state.event("land", why);
4830 }
4831 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4835 self.state.save()?;
4836 return Ok(());
4837 }
4838 Err(e) => {
4839 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4840 self.state.status = RunStatus::Blocked;
4841 self.state.base_sync = Some(BaseSync {
4842 tip,
4843 behind,
4844 attempts,
4845 conflict: Some(why.clone()),
4846 already_in: None,
4847 });
4848 self.state.event("land", why);
4849 }
4850 }
4851 self.state.save()?;
4852 Ok(())
4853 }
4854
4855 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4861 if self.state.task_pr_branch().is_some() {
4864 return None;
4865 }
4866 let repo = self.state.repo.clone();
4867 let remote = self.state.config.merge.remote.clone();
4868 let fetched = git::fetch(&repo, &remote, branch).await;
4869 if !matches!(&fetched, Ok(o) if o.ok()) {
4870 return None;
4871 }
4872 let branch_tracking = format!("{remote}/{branch}");
4873 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4874 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4875 return None;
4876 }
4877 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4878 .await
4879 .is_ok_and(|missing| missing.is_empty())
4880 {
4881 return Some(format!(
4882 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4883 them",
4884 short(&theirs)
4885 ));
4886 }
4887 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4888 Ok(o) if o.ok() => {
4889 self.state
4890 .event("land", format!("pushed rebased {branch} to {remote}"));
4891 None
4892 }
4893 Ok(o) => Some(format!(
4894 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4895 someone may have pushed): {}",
4896 short(&theirs),
4897 o.stderr.chars().take(600).collect::<String>()
4898 )),
4899 Err(e) => Some(format!(
4900 "{branch} is rebased locally but could not be pushed: {e:#}"
4901 )),
4902 }
4903 }
4904
4905 async fn settle_already_in(
4922 &mut self,
4923 branch: &str,
4924 tip: &str,
4925 head: &str,
4926 attempts: usize,
4927 behind: usize,
4928 ) -> Result<bool> {
4929 let repo = self.state.repo.clone();
4930 let remote = self.state.config.merge.remote.clone();
4931 let start = self.state.base_commit.clone();
4932 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4933 Ok(Some(e)) => e,
4934 Ok(None) => return Ok(false),
4935 Err(e) => {
4936 tracing::warn!("already-in-base check for {branch}: {e:#}");
4937 return Ok(false);
4938 }
4939 };
4940 let mut verified = vec![head.to_owned()];
4941 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4942 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4943 if matches!(&fetched, Ok(o) if o.ok())
4944 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4945 && theirs != head
4946 {
4947 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4948 Ok(Some(_)) => verified.push(theirs),
4949 _ => return Ok(false),
4950 }
4951 }
4952 let base_branch = self.state.base_branch.clone();
4953 let message = format!(
4954 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4955 land",
4956 evidence.names(),
4957 evidence.proof.as_str()
4958 );
4959 let closed =
4960 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4961 .await;
4962 match closed {
4963 Ok(Ok(url)) => self
4964 .state
4965 .event("land", format!("closed {url}: superseded on {base_branch}")),
4966 Ok(Err(why)) => self
4967 .state
4968 .event("land", format!("did not close a pull request: {why}")),
4969 Err(e) => {
4970 let why = format!(
4971 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4972 not be closed ({e:#}); resume to retry"
4973 );
4974 self.state.status = RunStatus::Blocked;
4975 self.state.base_sync = Some(BaseSync {
4976 tip: tip.to_owned(),
4977 behind,
4978 attempts,
4979 conflict: Some(why.clone()),
4980 already_in: None,
4981 });
4982 self.state.event("land", why);
4983 self.state.save()?;
4984 return Ok(true);
4985 }
4986 }
4987 self.state.status = RunStatus::AlreadyInBase;
4988 self.state.base_sync = Some(BaseSync {
4989 tip: tip.to_owned(),
4990 behind,
4991 attempts,
4992 conflict: None,
4993 already_in: Some(evidence),
4994 });
4995 self.state.event("land", message);
4996 self.state.save()?;
4997 self.settle_questions();
4998 Ok(true)
4999 }
5000
5001 fn landing_base(&self) -> String {
5011 self.state
5012 .base_sync
5013 .as_ref()
5014 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5015 }
5016
5017 pub async fn fix_selected(
5050 &mut self,
5051 ids: &[String],
5052 reason: &str,
5053 allow_stale: bool,
5054 ) -> Result<()> {
5055 let reason = reason.trim();
5056 if reason.is_empty() {
5057 bail!("a fix request needs a reason — that is the operator's own record of why");
5058 }
5059 if ids.is_empty() {
5060 bail!("no finding id given");
5061 }
5062 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5063 bail!(
5064 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5065 has already concluded — can be given a targeted fix. A run still \
5066 in progress should simply be resumed; a `merged` run's branch has \
5067 already landed, so its answer is a fresh `magi review <branch>`, \
5068 not reopening this run's own record",
5069 self.state.id,
5070 self.state.status.as_str()
5071 );
5072 }
5073 let Some(winner) = self.state.winner().cloned() else {
5074 bail!("run {} has no winning candidate to fix", self.state.id);
5075 };
5076 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5077 bail!(
5078 "branch `{}` no longer exists; this run cannot be extended",
5079 winner.branch
5080 );
5081 }
5082 let home = crate::run::home();
5083 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5084 bail!(
5085 "run {} is currently being worked on by another magi process",
5086 self.state.id
5087 );
5088 }
5089 let _claim = FixClaim::acquire(&self.state.dir())?;
5095
5096 let mut seen = BTreeSet::new();
5100 let mut findings = Vec::new();
5101 let mut missing = Vec::new();
5102 for id in ids {
5103 if !seen.insert(id.clone()) {
5104 continue;
5105 }
5106 match self.state.finding(id) {
5107 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5108 id: f.id.clone(),
5109 severity: f.severity,
5110 reviewer_vote: rec.vote,
5111 round: round.round,
5112 round_head: round.head.clone(),
5113 reviewer: rec.reviewer,
5114 agent: rec.agent.clone(),
5115 file: f.file.clone(),
5116 line: f.line,
5117 title: f.title.clone(),
5118 detail: f.detail.clone(),
5119 outcome: OperatorFixOutcome::Pending,
5120 }),
5121 None => missing.push(id.clone()),
5122 }
5123 }
5124 if !missing.is_empty() {
5125 bail!(
5126 "unknown finding id(s): {}; nothing was changed",
5127 missing.join(", ")
5128 );
5129 }
5130
5131 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5132 let stale_details: Vec<(String, String)> = findings
5133 .iter()
5134 .filter(|f| f.round_head != head_at_request)
5135 .map(|f| (f.id.clone(), f.round_head.clone()))
5136 .collect();
5137 let stale = !stale_details.is_empty();
5138 if stale && !allow_stale {
5139 bail!(
5140 "the branch has moved since some finding(s) were raised — {} — now \
5141 at {}; pass --allow-stale to fix anyway, or re-run review first",
5142 stale_details
5143 .iter()
5144 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5145 .collect::<Vec<_>>()
5146 .join(", "),
5147 short(&head_at_request)
5148 );
5149 }
5150
5151 let request = OperatorFixRequest {
5152 requested_at: Timestamp::now(),
5153 reason: reason.to_owned(),
5154 findings,
5155 head_at_request: head_at_request.clone(),
5156 allow_stale,
5157 stale,
5158 fix: None,
5159 result_head: None,
5160 follow_up_review_run: None,
5161 };
5162 self.state.event(
5163 "fix",
5164 format!(
5165 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5166 request.findings.len(),
5167 request
5168 .findings
5169 .iter()
5170 .map(|f| f.id.as_str())
5171 .collect::<Vec<_>>()
5172 .join(", "),
5173 ),
5174 );
5175 self.state.operator_fixes.push(request);
5182 self.state.save()?;
5183 let request_index = self.state.operator_fixes.len() - 1;
5184
5185 let repo = self.state.repo.clone();
5194 let coordination = self.coordinate(&repo).await?;
5195 if winner.worktree.exists() {
5196 let dirty = git::git(
5199 &winner.worktree,
5200 &["status", "--porcelain", "--untracked-files=all"],
5201 )
5202 .await?;
5203 let only_withheld = dirty.lines().all(|l| {
5204 l.strip_prefix("?? ")
5205 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5206 });
5207 if !only_withheld {
5208 bail!(
5209 "`{}` has uncommitted changes; refusing to touch it — commit or \
5210 discard them first",
5211 winner.worktree.display()
5212 );
5213 }
5214 git::worktree_remove(&self.state.repo, &winner.worktree)
5215 .await
5216 .ok();
5217 }
5218 let fix_worktree = self.state.worktree_root().join("operator-fix");
5219 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5220 git::git(
5221 &self.state.repo,
5222 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5223 )
5224 .await
5225 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5226 if !git::is_clean(&fix_worktree).await? {
5227 git::worktree_remove(&self.state.repo, &fix_worktree)
5228 .await
5229 .ok();
5230 bail!(
5231 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5232 winner.branch
5233 );
5234 }
5235 let repo_for_setup = self.state.repo.clone();
5236 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5237 Ok(()) => {
5238 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5239 }
5240 Err(e) => Err(e),
5241 };
5242 if let Err(e) = setup {
5243 git::worktree_remove(&self.state.repo, &fix_worktree)
5244 .await
5245 .ok();
5246 return Err(e);
5247 }
5248
5249 let run_id = self.state.id.clone();
5250 let prompts = self.state.config.prompts.clone();
5251 let language = self.state.config.graph.language.clone();
5252 let sessions = self.state.config.graph.sessions;
5253 let artifacts = agent::artifacts_dir(&self.state.dir());
5254 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5255 .findings
5256 .iter()
5257 .map(|f| Finding {
5258 id: f.id.clone(),
5259 severity: f.severity,
5260 file: f.file.clone(),
5261 line: f.line,
5262 title: f.title.clone(),
5263 detail: f.detail.clone(),
5264 })
5265 .collect();
5266 let fix_prompt = prompt::operator_fix_in(
5267 &self.state.instruction,
5268 &finding_list,
5269 reason,
5270 &stale_details,
5271 &head_at_request,
5272 &language,
5273 &self.state.config.graph.github_language,
5274 );
5275 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5276 drop(coordination);
5277 let (job, seat, out) = self
5278 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5279 prompt: fix_prompt.clone(),
5280 spec,
5281 seat,
5282 cwd: fix_worktree.clone(),
5283 timeout,
5284 allow_write: true,
5285 sessions,
5286 artifacts: artifacts.clone(),
5287 stem: "operator-fix".to_owned(),
5288 handover: None,
5289 })
5290 .await;
5291 let agent_id = seat.agent.clone();
5292
5293 let mut fix = FixRecord {
5294 agent: agent_id,
5295 addressed: Vec::new(),
5296 rejected: Vec::new(),
5297 notes: String::new(),
5298 committed: false,
5299 failed: None,
5300 duration_ms: 0,
5301 continuation: None,
5302 };
5303 let mut final_seat = seat.clone();
5304 match out {
5305 AgentOutcome::Ok(o) => {
5306 fix.duration_ms = o.duration_ms;
5307 let parsed = verdict::extract_json::<FixReport>(&o.text);
5308 let incomplete_reason = match &parsed {
5309 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5310 "the reply parsed, but it reported a command whose own CLI \
5311 never confirmed an exit status"
5312 .to_owned(),
5313 ),
5314 Ok(_) => None,
5315 Err(e) => Some(e.to_string()),
5316 };
5317 match incomplete_reason {
5318 None => {
5319 let report = parsed.expect("checked Ok above");
5320 fix.addressed = report.addressed;
5321 fix.rejected = report.rejected;
5322 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5323 }
5324 Some(reason) => {
5325 let (resumed_seat, resolved, failure, cont) = self
5326 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5327 .await;
5328 fix.duration_ms += cont.cumulative_wait_ms;
5329 fix.continuation = Some(cont);
5330 final_seat = resumed_seat;
5331 match resolved {
5332 Some(report) => {
5333 fix.addressed = report.addressed;
5334 fix.rejected = report.rejected;
5335 fix.notes =
5336 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5337 }
5338 None => fix.failed = failure,
5339 }
5340 }
5341 }
5342 }
5343 AgentOutcome::Dropped(o) => {
5344 fix.duration_ms = o.duration_ms;
5345 let why = o
5346 .dropped
5347 .as_ref()
5348 .map(|d| d.why.as_str())
5349 .unwrap_or("the CLI ended the stream without delivering its answer");
5350 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5351 }
5352 AgentOutcome::Quota(o) => {
5353 self.state.quota.push(QuotaLoss {
5354 seat: final_seat.key.clone(),
5355 node: "fix".to_owned(),
5356 at: Timestamp::now(),
5357 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5358 });
5359 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5360 }
5361 AgentOutcome::Failed(e) => fix.failed = Some(e),
5362 AgentOutcome::ResourceBlocked(e) => {
5363 fix.failed = Some(format!("the fixer was not started: {e}"));
5364 }
5365 }
5366 if fix.continuation.is_none() {
5367 fix.continuation = Some(ContinuationRecord::not_needed());
5368 }
5369 self.state.seats.insert(final_seat.key.clone(), final_seat);
5370
5371 let rescue_message = format!(
5372 "magi: operator-selected fix ({}) (uncommitted work)",
5373 self.state.operator_fixes[request_index]
5374 .findings
5375 .iter()
5376 .map(|f| f.id.as_str())
5377 .collect::<Vec<_>>()
5378 .join(", ")
5379 );
5380 let repo = self.state.repo.clone();
5381 let coordination = self.coordinate(&repo).await?;
5382 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5383 self.state.note_withheld("fix", &r.withheld);
5384 }
5385 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5386 fix.committed = after != head_at_request;
5387 git::worktree_remove(&self.state.repo, &fix_worktree)
5388 .await
5389 .ok();
5390 drop(coordination);
5391
5392 self.state.event(
5393 "fix",
5394 match &fix.failed {
5395 Some(reason) => format!(
5396 "operator fix: adoption report was lost ({reason}); {}",
5397 if fix.committed {
5398 "committed"
5399 } else {
5400 "NO new commit"
5401 }
5402 ),
5403 None => format!(
5404 "operator fix: {} addressed, {} rejected, {}",
5405 fix.addressed.len(),
5406 fix.rejected.len(),
5407 if fix.committed {
5408 "committed"
5409 } else {
5410 "NO new commit"
5411 }
5412 ),
5413 },
5414 );
5415
5416 for f in &mut self.state.operator_fixes[request_index].findings {
5423 f.outcome = if fix.failed.is_some() {
5424 OperatorFixOutcome::Unreported
5425 } else if fix.addressed.contains(&f.id) {
5426 OperatorFixOutcome::Addressed
5427 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5428 OperatorFixOutcome::Rejected { why: r.why.clone() }
5429 } else {
5430 OperatorFixOutcome::Unreported
5431 };
5432 }
5433
5434 let committed = fix.committed;
5435 if committed {
5436 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5437 }
5438 self.state.operator_fixes[request_index].fix = Some(fix);
5439 self.state.save()?;
5442
5443 if committed {
5444 self.state.event(
5445 "fix",
5446 format!(
5447 "operator fix committed {}; opening a follow-up review-only run",
5448 short(&after)
5449 ),
5450 );
5451 let origin =
5454 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5455 match Self::review(
5456 &self.state.repo,
5457 &winner.branch,
5458 self.state.config.clone(),
5459 origin,
5460 )
5461 .await
5462 {
5463 Ok(mut follow_up) => {
5464 follow_up.state.event(
5465 "start",
5466 format!(
5467 "requested by an operator fix on run {} for finding(s) {}",
5468 self.state.id,
5469 self.state.operator_fixes[request_index]
5470 .findings
5471 .iter()
5472 .map(|f| f.id.as_str())
5473 .collect::<Vec<_>>()
5474 .join(", "),
5475 ),
5476 );
5477 follow_up.state.save()?;
5478 let follow_up_id = follow_up.state.id.clone();
5479 let adopted = match crate::direct::adopt(
5482 &follow_up.state,
5483 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5484 ) {
5485 Ok(a) => a,
5486 Err(e) => {
5487 self.state.event(
5490 "fix",
5491 format!(
5492 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5493 ),
5494 );
5495 self.state.save()?;
5496 return Ok(());
5497 }
5498 };
5499 let executed = follow_up.execute().await;
5500 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5501 if let Some(a) = adopted {
5502 a.finish(&follow_up.state, executed);
5503 }
5504 if let Some(e) = failure {
5505 self.state.event(
5506 "fix",
5507 format!(
5508 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5509 ),
5510 );
5511 }
5512 self.state.operator_fixes[request_index].follow_up_review_run =
5513 Some(follow_up_id);
5514 }
5515 Err(e) => {
5516 self.state.event(
5517 "fix",
5518 format!("committed the fix but could not open a follow-up review: {e:#}"),
5519 );
5520 }
5521 }
5522 self.state.save()?;
5523 }
5524
5525 Ok(())
5526 }
5527
5528 async fn ask_fixer(
5548 &mut self,
5549 winner: &Candidate,
5550 node: &'static str,
5551 round: Option<usize>,
5552 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5553 ) -> (SeatJob, SeatState, AgentOutcome) {
5554 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5555 self.ask_fixer_among(attempts, node, round, None, build)
5556 .await
5557 }
5558
5559 async fn ask_fixer_among(
5564 &mut self,
5565 attempts: Vec<(AgentSpec, String)>,
5566 node: &'static str,
5567 round: Option<usize>,
5568 restore: Option<(&Path, &str)>,
5569 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5570 ) -> (SeatJob, SeatState, AgentOutcome) {
5571 let run_id = self.state.id.clone();
5572 let prompts = self.state.config.prompts.clone();
5573 let cache = self.state.config.cache_dir();
5574 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5575 let mut last = None;
5576 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5577 let seat = self.seat(&key, &spec.id);
5578 let mut job = build(spec.clone(), seat);
5579 if i > 0 {
5580 job.stem = format!("{}-{}", job.stem, spec.id);
5581 }
5582 let ctx = WaveCtx {
5583 carry_seats: false,
5584 pauses: Some((&self.pause, &self.interrupt)),
5585 run: &run_id,
5586 node,
5587 prompts: &prompts,
5588 cache: cache.as_deref(),
5589 round,
5590 };
5591 let (seat, out) =
5592 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5593 let advances = match &out {
5596 AgentOutcome::Ok(o) => agent::output_advances(o),
5597 AgentOutcome::ResourceBlocked(_) => false,
5598 _ => true,
5599 };
5600 if let Some(next) = ids.get(i + 1)
5601 && advances
5602 {
5603 self.state.seats.insert(seat.key.clone(), seat.clone());
5604 let class =
5605 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5606 let reason = match &out {
5607 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5608 other => fail_reason(other),
5609 };
5610 if let Some((wt, rev)) = restore
5614 && let Err(e) = self.graft_restore(wt, rev).await
5615 {
5616 let why =
5617 format!("could not restore the worktree before the next fixer: {e:#}");
5618 tracing::warn!("{why}");
5619 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5620 AgentOutcome::ResourceBlocked(why)
5621 } else {
5622 AgentOutcome::Failed(why)
5623 };
5624 last = Some((job, seat, out));
5625 break;
5626 }
5627 record_handover(
5628 &mut self.state,
5629 node,
5630 &seat.key,
5631 &spec.id,
5632 next,
5633 &class,
5634 &reason,
5635 );
5636 continue;
5637 }
5638 last = Some((job, seat, out));
5639 break;
5640 }
5641 last.expect("the fixer chain always has an entry")
5642 }
5643
5644 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5655 use crate::recovery::{Mode, Repair};
5656 if !self.state.reviews.is_empty() {
5657 return Ok(());
5658 }
5659 let Some(mut rec) = self.state.recovery.clone() else {
5660 return Ok(());
5661 };
5662 if rec.mode != Mode::VerifyRepair {
5663 return Ok(());
5664 }
5665 let wt = winner.worktree.clone();
5666 let mut head = git::rev_parse(&wt, "HEAD").await?;
5667 match rec.repair.clone() {
5668 Repair::Done { .. } => return Ok(()),
5669 Repair::WaitingForRestore { before } => {
5670 self.graft_restore(&wt, &before).await?;
5671 head = before;
5672 rec.repair = Repair::NotStarted;
5673 self.state.recovery = Some(rec.clone());
5674 self.state.save()?;
5675 }
5676 Repair::Started { before } => {
5677 if let Err(e) = self.graft_restore(&wt, &before).await {
5681 if e.is::<crate::repo_lock::ResourceBlocked>() {
5682 return Err(e);
5683 }
5684 return self
5685 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5686 .await;
5687 }
5688 rec.repair = Repair::Done {
5689 before: before.clone(),
5690 after: before,
5691 committed: false,
5692 failed: Some("interrupted; rolled back".to_owned()),
5693 };
5694 self.state
5695 .event("recovery", "the repair was interrupted and rolled back");
5696 self.state.recovery = Some(rec);
5697 self.state.save()?;
5698 return Ok(());
5699 }
5700 Repair::NotStarted => {}
5701 }
5702 if crate::recovery::recheck(&mut rec, &head) {
5703 self.state.event("recovery", rec.summary());
5704 self.state.recovery = Some(rec);
5705 self.state.save()?;
5706 return Ok(());
5707 }
5708 rec.repair = Repair::Started {
5709 before: head.clone(),
5710 };
5711 self.state.recovery = Some(rec.clone());
5712 self.state.save()?;
5713
5714 let language = self.state.config.graph.language.clone();
5715 let sessions = self.state.config.graph.sessions;
5716 let artifacts = agent::artifacts_dir(&self.state.dir());
5717 let fix_prompt = prompt::recovery_fix(
5718 &self.state.instruction,
5719 &rec,
5720 self.state.config.graph.review_rounds,
5721 &language,
5722 );
5723 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5724 self.state.event(
5725 "recovery",
5726 format!(
5727 "repair: the fixer starts from the failed verification on {}",
5728 short(&head)
5729 ),
5730 );
5731 let (_job, seat, out) = self
5732 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5733 prompt: fix_prompt.clone(),
5734 spec,
5735 seat,
5736 cwd: wt.clone(),
5737 timeout,
5738 allow_write: true,
5739 sessions,
5740 artifacts: artifacts.clone(),
5741 stem: "fix-recovery".to_owned(),
5742 handover: None,
5743 })
5744 .await;
5745 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5746 rec.repair = Repair::WaitingForRestore { before: head };
5747 self.state.recovery = Some(rec);
5748 return self.stop_for_resource();
5749 }
5750 let failed = match &out {
5751 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5752 AgentOutcome::Quota(o) => {
5753 self.state.quota.push(QuotaLoss {
5754 seat: seat.key.clone(),
5755 node: "fix".to_owned(),
5756 at: Timestamp::now(),
5757 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5758 });
5759 Some("rate limited (quota); the fixer could not run".to_owned())
5760 }
5761 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5762 AgentOutcome::Failed(e) => Some(e.clone()),
5763 };
5764 self.state.seats.insert(seat.key.clone(), seat);
5765 let coordination = self.coordinate(&wt).await?;
5766 let failed = match failed {
5767 Some(why) => {
5769 if let Err(e) = git::restore_to(&wt, &head).await {
5770 return self
5771 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5772 .await;
5773 }
5774 Some(why)
5775 }
5776 None => {
5777 if let Ok(r) = git::rescue_commit(
5778 &wt,
5779 "magi: recovery repair of the failed verification (uncommitted work)",
5780 )
5781 .await
5782 {
5783 self.state.note_withheld("fix", &r.withheld);
5784 }
5785 None
5786 }
5787 };
5788 let after = git::rev_parse(&wt, "HEAD").await?;
5789 drop(coordination);
5790 self.state.event(
5791 "recovery",
5792 match &failed {
5793 Some(why) => format!("repair: nothing kept ({why})"),
5794 None if after != head => {
5795 format!("repair: committed {}", short(&after))
5796 }
5797 None => "repair: the fixer made no commit".to_owned(),
5798 },
5799 );
5800 rec.repair = Repair::Done {
5801 before: head.clone(),
5802 committed: after != head,
5803 after,
5804 failed,
5805 };
5806 self.state.recovery = Some(rec);
5807 self.state.save()?;
5808 Ok(())
5809 }
5810
5811 async fn stop_recovery(
5814 &mut self,
5815 mut rec: crate::recovery::RecoveryRecord,
5816 head: &str,
5817 why: String,
5818 ) -> Result<()> {
5819 rec.repair = crate::recovery::Repair::Started {
5821 before: head.to_owned(),
5822 };
5823 self.state.recovery = Some(rec);
5824 self.state
5825 .event("recovery", format!("repair could not be undone: {why}"));
5826 self.state.status = RunStatus::Blocked;
5827 self.state.save()?;
5828 bail!("recovery repair could not be rolled back: {why}")
5829 }
5830
5831 async fn review_loop(&mut self) -> Result<()> {
5832 if self
5837 .state
5838 .base_sync
5839 .as_ref()
5840 .is_some_and(|s| s.conflict.is_some())
5841 {
5842 return Ok(());
5843 }
5844 let run_id = self.state.id.clone();
5849 let prompts = self.state.config.prompts.clone();
5850 let Some(winner) = self.state.winner().cloned() else {
5851 return Ok(());
5852 };
5853 let max_rounds = self.state.config.graph.review_rounds;
5854 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5864 if status == RunStatus::Gating {
5867 self.record_contested_handoff();
5868 }
5869 self.state.status = status;
5870 self.state.save()?;
5871 return Ok(());
5872 }
5873 self.state.status = RunStatus::Reviewing;
5874 if self
5884 .state
5885 .reviews
5886 .last()
5887 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5888 {
5889 let shell = self.state.config.shell();
5890 return self
5891 .stop_reviewing(
5892 "the last round's own verification never resolved",
5893 &shell,
5894 &winner.worktree,
5895 )
5896 .await;
5897 }
5898
5899 self.recovery_repair(&winner).await?;
5900 if self.resource_halt()? {
5901 return Ok(());
5902 }
5903
5904 let repo = self.state.repo.clone();
5905 let root = self.state.worktree_root();
5906 let language = self.state.config.graph.language.clone();
5907 let sessions = self.state.config.graph.sessions;
5908 let artifacts = self.merge_repair_artifacts();
5909 let base = self.landing_base();
5910 let base_short = short(&base);
5911 let reviewers = self.roles.reviewers.clone();
5912 let shell = self.state.config.shell();
5913
5914 for round in (self.state.reviews.len() + 1)..=max_rounds {
5915 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5916 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5917 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5918 let prev_verification = self
5927 .state
5928 .reviews
5929 .last()
5930 .and_then(|r| r.verification_summary(&head));
5931 let prior_section = match (&self.state.recovery, round) {
5934 (Some(rec), 1) if self.state.reviews.is_empty() => {
5935 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5936 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5937 Some(crate::recovery::head_of(
5938 &git::diff(&winner.worktree, reviewed, "HEAD")
5939 .await
5940 .unwrap_or_default(),
5941 crate::recovery::SINCE_DIFF_MAX,
5942 ))
5943 }
5944 _ => None,
5945 };
5946 Some(prompt::recovery_review(rec, since.as_deref()))
5947 }
5948 _ => None,
5949 };
5950
5951 let mut jobs = Vec::new();
5955 let coordination = self.coordinate(&repo).await?;
5956 for (r, spec) in reviewers.iter().cloned().enumerate() {
5957 let wt = root.join(format!("review-{}", r + 1));
5958 if wt.exists() {
5961 git::reset_detached(&wt, &head).await?;
5962 } else {
5963 git::worktree_add_detached(&repo, &wt, &head).await?;
5964 }
5965 self.setup_seat_worktree(&repo, &wt, false).await?;
5966 let seat_key = format!("review-{}", r + 1);
5967 let spec = pick_start_spec(
5970 &self.roles.reviewer_roster,
5971 spec,
5972 self.state.seat_history.get(&seat_key),
5973 );
5974 let seat = self.seat(&seat_key, &spec.id);
5975 jobs.push(SeatJob {
5976 prompt: prompt::review(&prompt::ReviewCtx {
5977 instruction: &self.state.instruction,
5978 branch: &winner.branch,
5979 base_short: &base_short,
5980 stat: &stat,
5981 patch: &patch,
5982 verification: prev_verification.as_ref(),
5983 prior: prior_section.as_deref(),
5984 reviewers: reviewers.len(),
5985 round,
5986 rounds: max_rounds,
5987 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5990 lens: Lens::for_seat(r),
5991 language: &language,
5992 github_language: &self.state.config.graph.github_language,
5993 }),
5994 spec,
5995 seat,
5996 cwd: wt,
5997 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5998 allow_write: false,
5999 sessions,
6000 artifacts: artifacts.clone(),
6001 stem: format!("review-{round}-{}", r + 1),
6002 handover: None,
6003 });
6004 }
6005
6006 drop(coordination);
6007 self.state.event(
6008 "review",
6009 format!(
6010 "round {round}: {} reviewers on {}",
6011 jobs.len(),
6012 short(&head)
6013 ),
6014 );
6015 let mut quota_losses = Vec::new();
6016 let review_retries = self.state.config.graph.retries;
6017 let review_cache = self.state.config.cache_dir();
6018 let ctx = WaveCtx {
6019 carry_seats: true,
6020 pauses: Some((&self.pause, &self.interrupt)),
6021 run: &run_id,
6022 node: "review",
6023 prompts: &prompts,
6024 cache: review_cache.as_deref(),
6025 round: Some(round),
6026 };
6027 let results = ask_json_wave::<Review>(
6028 jobs,
6029 Arc::clone(&self.sem),
6030 review_retries,
6031 &self.roles.reviewer_roster,
6032 &ctx,
6033 &mut quota_losses,
6034 &mut self.state,
6035 &|_: &Review| Ok(()),
6036 )
6037 .await;
6038 if self.state.resource_block.is_some() {
6039 return self.stop_for_resource();
6040 }
6041 let round_quota_missing = quota_losses.len();
6045 self.state.quota.extend(quota_losses);
6046
6047 let mut records = Vec::new();
6048 let mut all_findings = Vec::new();
6049 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6050 let agent_id = seat.agent.clone();
6051 self.state.seats.insert(seat.key.clone(), seat);
6052 let mut record = ReviewRecord {
6053 reviewer: r + 1,
6054 agent: agent_id,
6055 summary: String::new(),
6056 findings: Vec::new(),
6057 vote: None,
6058 failed: None,
6059 duration_ms: 0,
6060 attempts,
6066 };
6067 match res {
6068 Ok((review, out)) => {
6069 record.summary =
6077 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6078 record.vote = Some(review.vote);
6079 record.duration_ms = out.duration_ms;
6080 for (n, mut f) in review.findings.into_iter().enumerate() {
6081 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6084 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6085 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6086 f.file = f
6092 .file
6093 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6094 all_findings.push(f.clone());
6095 record.findings.push(f);
6096 }
6097 self.state.event(
6098 "review",
6099 format!(
6100 "round {round}: reviewer {} voted {} with {} finding(s)",
6101 r + 1,
6102 review.vote.label(),
6103 record.findings.len()
6104 ),
6105 );
6106 }
6107 Err(e) => {
6108 record.failed = Some(e.to_string());
6109 self.state.event(
6110 "review",
6111 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6112 );
6113 }
6114 }
6115 records.push(record);
6116 }
6117
6118 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6125 let vote_split =
6126 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6127 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6128 if vote_split {
6129 self.state.event(
6130 "review",
6131 format!(
6132 "round {round}: votes split ({}) — one round of reconsideration",
6133 initial_votes
6134 .iter()
6135 .map(|v| v.label())
6136 .collect::<Vec<_>>()
6137 .join(", ")
6138 ),
6139 );
6140 let panel: Vec<ReviewSeatReport<'_>> = records
6143 .iter()
6144 .filter_map(|r| {
6145 r.vote.map(|vote| ReviewSeatReport {
6146 reviewer: r.reviewer,
6147 vote,
6148 summary: &r.summary,
6149 findings: &r.findings,
6150 })
6151 })
6152 .collect();
6153
6154 let mut jobs = Vec::new();
6155 let mut seats_at = Vec::new();
6156 for (r, spec) in reviewers.iter().cloned().enumerate() {
6157 if records[r].vote.is_none() {
6161 continue;
6162 }
6163 let wt = root.join(format!("review-{}", r + 1));
6164 let seat_key = format!("review-{}", r + 1);
6165 let spec = self.occupant(&seat_key, spec);
6166 let seat = self.seat(&seat_key, &spec.id);
6167 let build = |with_patch: bool| {
6175 prompt::review_reconsider(&ReviewReconsiderCtx {
6176 instruction: &self.state.instruction,
6177 reviewer: r + 1,
6178 lens: Lens::for_seat(r),
6179 panel: &panel,
6180 patch: with_patch.then_some(ReviewPatch {
6181 branch: &winner.branch,
6182 base_short: &base_short,
6183 stat: &stat,
6184 patch: &patch,
6185 }),
6186 round,
6187 rounds: max_rounds,
6188 language: &language,
6189 })
6190 };
6191 let full = build(true);
6192 let prompt = if has_context(&spec, &seat, sessions) {
6193 build(false)
6194 } else {
6195 full.clone()
6196 };
6197 jobs.push(SeatJob {
6198 prompt,
6199 spec,
6200 seat,
6201 cwd: wt,
6202 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6203 allow_write: false,
6204 sessions,
6205 artifacts: artifacts.clone(),
6206 stem: format!("review-{round}-reconsider-{}", r + 1),
6207 handover: Some(full),
6208 });
6209 seats_at.push(r);
6210 }
6211
6212 let mut recon_quota_losses = Vec::new();
6213 let recon_cache = self.state.config.cache_dir();
6214 let recon_ctx = WaveCtx {
6215 carry_seats: true,
6216 pauses: Some((&self.pause, &self.interrupt)),
6217 run: &run_id,
6218 node: "review",
6219 prompts: &prompts,
6220 cache: recon_cache.as_deref(),
6221 round: Some(round),
6222 };
6223 let recon_results = ask_json_wave::<ReviewRevote>(
6224 jobs,
6225 Arc::clone(&self.sem),
6226 review_retries,
6227 &self.roles.reviewer_roster,
6228 &recon_ctx,
6229 &mut recon_quota_losses,
6230 &mut self.state,
6231 &|_: &ReviewRevote| Ok(()),
6232 )
6233 .await;
6234 if self.state.resource_block.is_some() {
6235 return self.stop_for_resource();
6236 }
6237 self.state.quota.extend(recon_quota_losses);
6238
6239 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6240 let agent_id = seat.agent.clone();
6241 self.state.seats.insert(seat.key.clone(), seat);
6242 let mut rec = ReviewRevoteRecord {
6243 reviewer: r + 1,
6244 agent: agent_id,
6245 vote: None,
6246 reason: String::new(),
6247 failed: None,
6248 };
6249 match res {
6250 Ok((rv, _)) => {
6251 rec.vote = Some(rv.vote);
6252 rec.reason =
6253 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6254 self.state.event(
6255 "review",
6256 format!(
6257 "round {round}: reviewer {} revoted {}",
6258 r + 1,
6259 rv.vote.label()
6260 ),
6261 );
6262 }
6263 Err(e) => {
6264 rec.failed = Some(e.to_string());
6265 self.state.event(
6266 "review",
6267 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6268 );
6269 }
6270 }
6271 reconsideration.push(rec);
6272 }
6273 } else if initial_votes.len() > 1 {
6274 self.state.event(
6275 "review",
6276 format!(
6277 "round {round}: votes agreed ({}) — no reconsideration",
6278 initial_votes[0].label()
6279 ),
6280 );
6281 }
6282
6283 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6284 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6285 let defer_e2e =
6296 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6297 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6298 let reason =
6299 format!("{blocking} blocking finding(s) already required a fix this round");
6300 self.state.event(
6301 "verify",
6302 format!(
6303 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6304 {}); it will run once a round has none left",
6305 short(&head)
6306 ),
6307 );
6308 (Vec::new(), false, true, Some(reason))
6309 } else {
6310 let e2e_commands = self.state.config.verify.e2e.clone();
6311 let cache_dir = self.state.config.cache_dir();
6312 let context = format!("round {round}");
6313 let wait = self.repository_wait();
6314 let (e2e, verify_retried) = with_cache_lease(
6315 &mut self.state,
6316 cache_dir.as_deref(),
6317 "e2e",
6318 "e2e",
6319 &winner.worktree,
6320 &head,
6321 verify_timeout,
6322 &context,
6323 &wait,
6324 |state, budget| {
6325 let shell = shell.clone();
6326 let e2e_commands = e2e_commands.clone();
6327 let worktree = winner.worktree.clone();
6328 let context = context.clone();
6329 async move {
6330 run_e2e_with_retry(
6331 state,
6332 &shell,
6333 &e2e_commands,
6334 &worktree,
6335 budget,
6336 &context,
6337 )
6338 .await
6339 }
6340 },
6341 )
6342 .await?;
6343 (e2e, verify_retried, false, None)
6344 };
6345
6346 let expected = records.len();
6347 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6348 let incomplete = answered < expected;
6349 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6350 let policy = self.state.config.graph.incomplete_review;
6351 let clean = round_is_clean(
6352 blocking,
6353 e2e_ok,
6354 answered,
6355 expected,
6356 round_quota_missing,
6357 policy,
6358 );
6359
6360 let mut round_record = ReviewRound {
6361 round,
6362 head: head.clone(),
6363 verified_head: None,
6364 verified_at: None,
6365 reviews: records,
6366 e2e,
6367 verify_retried,
6368 e2e_deferred,
6369 e2e_defer_reason,
6370 fix: None,
6371 blocking,
6372 answered,
6373 expected,
6374 clean,
6375 progressed: false,
6376 vote_split,
6377 reconsideration,
6378 verdict: None,
6379 };
6380 round_record.verdict = ReviewVote::worst(
6384 round_record
6385 .final_votes()
6386 .into_iter()
6387 .map(|(_, _, vote)| vote),
6388 );
6389 if !matches!(
6400 round_record.e2e_status(),
6401 E2eStatus::Deferred | E2eStatus::NotConfigured
6402 ) {
6403 round_record.verified_head = Some(head.clone());
6404 round_record.verified_at = Some(Timestamp::now());
6405 }
6406 let this_round_verification = round_record.verification_summary(&head);
6407
6408 if incomplete {
6409 let missing: Vec<String> = round_record
6410 .reviews
6411 .iter()
6412 .filter(|r| r.failed.is_some())
6413 .map(|r| format!("review-{}", r.reviewer))
6414 .collect();
6415 self.state.event(
6416 "review",
6417 format!(
6418 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6419 missing.join(", ")
6420 ),
6421 );
6422 }
6423
6424 if clean {
6425 self.state.event(
6426 "review",
6427 if incomplete && policy == IncompleteReviewPolicy::Warn {
6428 format!(
6429 "round {round}: clean (warn policy, incomplete panel) — no \
6430 blocking findings from the seats that answered, verification green"
6431 )
6432 } else if incomplete {
6433 format!(
6434 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6435 quorum) — no blocking findings from the seats that answered, \
6436 verification green",
6437 expected - answered
6438 )
6439 } else {
6440 format!("round {round}: clean — no blocking findings, verification green")
6441 },
6442 );
6443 self.state.reviews.push(round_record);
6444 self.state.status = RunStatus::Gating;
6445 self.state.save()?;
6446 return Ok(());
6447 }
6448
6449 if incomplete && blocking == 0 && e2e_ok {
6457 self.state.reviews.push(round_record);
6458 self.state.save()?;
6459 if round == max_rounds {
6460 self.state.status = RunStatus::Blocked;
6461 self.state.event(
6462 "review",
6463 format!(
6464 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6465 refusing to call it clean",
6466 expected - answered
6467 ),
6468 );
6469 return Ok(());
6470 }
6471 continue;
6472 }
6473
6474 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6486 self.state.reviews.push(round_record);
6487 return self
6488 .stop_reviewing(
6489 "the round's own verification could not run",
6490 &shell,
6491 &winner.worktree,
6492 )
6493 .await;
6494 }
6495
6496 if round == max_rounds {
6497 self.state.reviews.push(round_record);
6498 return self
6499 .stop_reviewing(
6500 &format!(
6501 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6502 ),
6503 &shell,
6504 &winner.worktree,
6505 )
6506 .await;
6507 }
6508
6509 let blocking_findings: Vec<_> = all_findings
6512 .iter()
6513 .filter(|f| f.severity.blocks())
6514 .cloned()
6515 .collect();
6516 let fix_prompt = prompt::fix_in(
6517 &self.state.instruction,
6518 &blocking_findings,
6519 this_round_verification.as_ref(),
6520 round,
6521 max_rounds,
6522 &language,
6523 &self.state.config.graph.github_language,
6524 );
6525 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6526 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6527 let (job, seat, out) = self
6528 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6529 prompt: fix_prompt.clone(),
6530 spec,
6531 seat,
6532 cwd: winner.worktree.clone(),
6533 timeout,
6534 allow_write: true,
6535 sessions,
6536 artifacts: artifacts.clone(),
6537 stem: format!("fix-{round}"),
6538 handover: None,
6539 })
6540 .await;
6541 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6544 return self.stop_for_resource();
6545 }
6546 let agent_id = seat.agent.clone();
6547
6548 let mut fix = FixRecord {
6549 agent: agent_id,
6550 addressed: Vec::new(),
6551 rejected: Vec::new(),
6552 notes: String::new(),
6553 committed: false,
6554 failed: None,
6555 duration_ms: 0,
6556 continuation: None,
6557 };
6558 let mut continuation = ContinuationRecord::not_needed();
6559 let mut final_seat = seat.clone();
6560 match out {
6561 AgentOutcome::Ok(o) => {
6562 fix.duration_ms = o.duration_ms;
6563 let parsed = verdict::extract_json::<FixReport>(&o.text);
6564 let incomplete_reason = match &parsed {
6571 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6572 "the reply parsed, but it reported a command whose own CLI never \
6573 confirmed an exit status"
6574 .to_owned(),
6575 ),
6576 Ok(_) => None,
6577 Err(e) => Some(e.to_string()),
6578 };
6579 match incomplete_reason {
6580 None => {
6581 let report = parsed.expect("checked Ok above");
6582 fix.addressed = report.addressed;
6583 fix.rejected = report.rejected;
6584 fix.notes =
6585 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6586 }
6587 Some(reason) => {
6588 let (resumed_seat, resolved, failure, cont) = self
6589 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6590 .await;
6591 fix.duration_ms += cont.cumulative_wait_ms;
6592 continuation = cont;
6593 final_seat = resumed_seat;
6594 match resolved {
6595 Some(report) => {
6596 fix.addressed = report.addressed;
6597 fix.rejected = report.rejected;
6598 fix.notes = blind::sanitize_prose(
6599 &report.notes,
6600 &self.state.config.blind,
6601 );
6602 }
6603 None => fix.failed = failure,
6604 }
6605 }
6606 }
6607 }
6608 AgentOutcome::Dropped(o) => {
6610 fix.duration_ms = o.duration_ms;
6611 let why = o
6612 .dropped
6613 .as_ref()
6614 .map(|d| d.why.as_str())
6615 .unwrap_or("the CLI ended the stream without delivering its answer");
6616 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6617 }
6618 AgentOutcome::Quota(o) => {
6619 self.state.quota.push(QuotaLoss {
6620 seat: final_seat.key.clone(),
6621 node: "fix".to_owned(),
6622 at: Timestamp::now(),
6623 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6624 });
6625 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6626 }
6627 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6628 }
6629 fix.continuation = Some(continuation);
6630 self.state.seats.insert(final_seat.key.clone(), final_seat);
6631 let coordination = self.coordinate(&winner.worktree).await?;
6632 if let Ok(r) = git::rescue_commit(
6633 &winner.worktree,
6634 &format!("magi: review round {round} fixes (uncommitted work)"),
6635 )
6636 .await
6637 {
6638 self.state.note_withheld("fix", &r.withheld);
6639 }
6640 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6641 fix.committed = after != before;
6642 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6650 drop(coordination);
6651 let progressed = diff_after != patch;
6652 let commit_note = if fix.committed {
6653 "committed"
6654 } else {
6655 "NO new commit"
6656 };
6657 let tree_note = if progressed {
6658 "changed vs base"
6659 } else {
6660 "unchanged vs base"
6661 };
6662 self.state.event(
6663 "fix",
6664 match &fix.failed {
6665 Some(reason) => {
6671 format!(
6672 "round {round}: fixer's adoption report was lost ({reason}); \
6673 {commit_note}, tree {tree_note}"
6674 )
6675 }
6676 None => format!(
6677 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6678 {tree_note}{}",
6679 fix.addressed.len(),
6680 fix.rejected.len(),
6681 if continuation.outcome == ContinuationOutcome::Resumed {
6682 format!(
6683 " (adoption report recovered after {} continuation(s))",
6684 continuation.attempts
6685 )
6686 } else {
6687 String::new()
6688 },
6689 ),
6690 },
6691 );
6692 round_record.fix = Some(fix);
6693 round_record.progressed = progressed;
6694 self.state.reviews.push(round_record);
6695 self.state.save()?;
6696
6697 if matches!(
6710 continuation.outcome,
6711 ContinuationOutcome::Exhausted
6712 | ContinuationOutcome::QuotaLost
6713 | ContinuationOutcome::NoSession
6714 ) {
6715 return self
6716 .stop_reviewing(
6717 "the fixer's adoption report never came back, even after resuming its \
6718 own seat; refusing to start another round against the same worktree \
6719 while that is unresolved",
6720 &shell,
6721 &winner.worktree,
6722 )
6723 .await;
6724 }
6725
6726 let streak = self
6727 .state
6728 .reviews
6729 .iter()
6730 .rev()
6731 .take_while(|r| !r.progressed)
6732 .count();
6733 if streak >= STAGNANT_LIMIT {
6734 return self
6735 .stop_reviewing(
6736 &format!(
6737 "the tree has not moved against base for {streak} round(s) in a row"
6738 ),
6739 &shell,
6740 &winner.worktree,
6741 )
6742 .await;
6743 }
6744 }
6745 Ok(())
6746 }
6747
6748 fn record_contested_handoff(&mut self) {
6782 if self.state.contested_handoff.is_some() {
6783 return;
6784 }
6785 let Some(contested) = self
6786 .state
6787 .reviews
6788 .last()
6789 .and_then(ReviewRound::contested_handoff)
6790 else {
6791 return;
6792 };
6793 self.state.event(
6794 "review",
6795 format!(
6796 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6797 wait for the owner's approval",
6798 contested.findings.len(),
6799 contested.rejecters.len()
6800 ),
6801 );
6802 self.state.contested_handoff = Some(contested);
6803 }
6804
6805 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6806 let round_idx = self.state.reviews.len() - 1;
6807 let needs_catchup_run = matches!(
6815 self.state.reviews[round_idx].e2e_status(),
6816 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6817 );
6818 if needs_catchup_run {
6819 let round = self.state.reviews[round_idx].round;
6820 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6821 let commands = self.state.config.verify.e2e.clone();
6822 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6823 let cache_dir = self.state.config.cache_dir();
6824 let context = format!(
6825 "round {round}: verification unresolved, catching up before the final decision"
6826 );
6827 let wait = self.repository_wait();
6828 let (outcomes, verify_retried) = with_cache_lease(
6829 &mut self.state,
6830 cache_dir.as_deref(),
6831 "e2e",
6832 "e2e",
6833 worktree,
6834 &attempted_head,
6835 timeout,
6836 &context,
6837 &wait,
6838 |state, budget| {
6839 let shell = shell.to_vec();
6840 let commands = commands.clone();
6841 let context = context.clone();
6842 async move {
6843 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6844 .await
6845 }
6846 },
6847 )
6848 .await?;
6849 let last = &mut self.state.reviews[round_idx];
6850 last.e2e = outcomes;
6851 last.verify_retried = verify_retried;
6852 last.verified_head = Some(attempted_head);
6859 last.verified_at = Some(Timestamp::now());
6860 if verify_inconclusive(&last.e2e) {
6861 self.state.save()?;
6868 return Ok(());
6869 }
6870 last.e2e_deferred = false;
6871 }
6872 let last = &self.state.reviews[round_idx];
6873 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6874
6875 match last.e2e_status() {
6876 E2eStatus::Failed => {
6877 let red: Vec<String> = last
6878 .e2e
6879 .iter()
6880 .filter(|o| !o.ok())
6881 .map(|o| {
6882 format!(
6883 "`{}` -> {:?}\n{}",
6884 o.command,
6885 o.code,
6886 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6887 )
6888 })
6889 .collect();
6890 self.state
6891 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6892 self.state.status = RunStatus::Blocked;
6893 }
6894 E2eStatus::ResourceBlocked => {
6899 self.state.event(
6900 "review",
6901 format!(
6902 "{why}; e2e could not run (shared build cache unavailable); not \
6903 deciding yet"
6904 ),
6905 );
6906 }
6907 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6908 self.state.event(
6909 "review",
6910 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6911 );
6912 self.record_contested_handoff();
6913 self.state.status = RunStatus::Gating;
6914 }
6915 }
6916 self.state.save()?;
6917 Ok(())
6918 }
6919
6920 async fn gate(&mut self) -> Result<()> {
6923 if self.state.status == RunStatus::Failed
6935 || self
6936 .state
6937 .base_sync
6938 .as_ref()
6939 .is_some_and(|s| s.conflict.is_some())
6940 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6941 != Some(RunStatus::Gating)
6942 {
6943 return Ok(());
6944 }
6945 if self.state.gate_ran {
6946 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6957 self.state.status = RunStatus::Blocked;
6958 self.state.save()?;
6959 }
6960 return Ok(());
6961 }
6962 let Some(winner) = self.state.winner().cloned() else {
6963 return Ok(());
6964 };
6965 self.state.status = RunStatus::Gating;
6966 let mut outcomes = self.run_gate(&winner).await?;
6967 loop {
6968 if verify_inconclusive(&outcomes) {
6979 self.state.save()?;
6980 return Ok(());
6981 }
6982 if outcomes.iter().all(CommandOutcome::ok) {
6983 break;
6984 }
6985 match self.gate_fix_round(&winner, &outcomes).await? {
6986 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6987 GateFix::Stop => break,
6988 GateFix::Defer => {
6989 self.state.save()?;
6990 return Ok(());
6991 }
6992 }
6993 }
6994 let passed = outcomes.iter().all(CommandOutcome::ok);
6995 self.state.gate = outcomes;
6996 self.state.gate_ran = true;
6997 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6998 if !passed {
6999 self.state.status = RunStatus::Blocked;
7000 let spent = self.state.gate_fixes.len();
7001 self.state.event(
7002 "gate",
7003 if spent == 0 {
7004 "gate failed; not merging".to_owned()
7005 } else {
7006 format!("gate failed after {spent} gate-fix round(s); not merging")
7007 },
7008 );
7009 }
7010 self.state.save()?;
7011 Ok(())
7012 }
7013
7014 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7024 let commands = state.config.verify.pre_gate.clone();
7025 if commands.is_empty() {
7026 return;
7027 }
7028 let shell = state.config.shell();
7029 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7030 let (outcomes, _) = run_commands(
7031 state,
7032 "pre_gate",
7033 "pre_gate",
7034 0,
7035 &shell,
7036 &commands,
7037 &winner.worktree,
7038 timeout,
7039 )
7040 .await;
7041 for o in &outcomes {
7042 if !o.ok() {
7043 tracing::warn!(
7044 "pre_gate `{}` failed ({:?}); the gate decides",
7045 o.command,
7046 o.code
7047 );
7048 }
7049 state.event(
7050 "pre_gate",
7051 format!(
7052 "`{}` -> {}",
7053 o.command,
7054 if o.ok() {
7055 "pass".to_owned()
7056 } else {
7057 format!(
7058 "FAIL ({:?})\n{}",
7059 o.code,
7060 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7061 )
7062 }
7063 ),
7064 );
7065 }
7066 state.pre_gate = outcomes;
7067 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7068 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7069 Ok(head) => {
7070 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7071 state.pre_gate_commit = Some(head);
7072 }
7073 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7074 },
7075 Ok(false) => {}
7076 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7077 }
7078 if let Err(e) = state.save() {
7079 tracing::warn!("could not persist the pre_gate record: {e:#}");
7080 }
7081 }
7082
7083 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7086 let wait = self.repository_wait();
7087 run_landing_gate(&mut self.state, winner, &wait).await
7088 }
7089
7090 async fn gate_fix_round(
7103 &mut self,
7104 winner: &Candidate,
7105 outcomes: &[CommandOutcome],
7106 ) -> Result<GateFix> {
7107 let cap = self.state.config.graph.gate_fix_rounds;
7108 let spent = self.state.gate_fixes.len();
7109 if spent >= cap {
7110 if cap > 0 {
7111 self.state.event(
7112 "gate",
7113 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7114 );
7115 }
7116 return Ok(GateFix::Stop);
7117 }
7118 if !gate_fixable(outcomes) {
7119 self.state.event(
7120 "gate",
7121 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7122 command or similar); not spending a fix round on it",
7123 );
7124 return Ok(GateFix::Stop);
7125 }
7126 let min_free = self.state.config.disk.min_free_bytes;
7127 if min_free > 0 {
7128 match crate::disk::free_bytes(&winner.worktree) {
7129 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7130 Ok(free) => {
7131 self.state.event(
7132 "gate",
7133 format!(
7134 "only {free} bytes free ({min_free} required by `[disk] \
7135 min_free_bytes`); not spending a fix round on a failure the disk \
7136 may explain"
7137 ),
7138 );
7139 return Ok(GateFix::Stop);
7140 }
7141 Err(e) => {
7142 self.state.event(
7143 "gate",
7144 format!("free disk space could not be measured ({e:#}); no fix round"),
7145 );
7146 return Ok(GateFix::Stop);
7147 }
7148 }
7149 }
7150
7151 let attempt = spent + 1;
7152 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7153 let base = self.landing_base();
7154 let fix_prompt = prompt::gate_fix_in(
7155 &self.state.instruction,
7156 &failed,
7157 attempt,
7158 cap,
7159 &self.state.config.graph.language,
7160 &self.state.config.graph.github_language,
7161 );
7162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7163 let sessions = self.state.config.graph.sessions;
7164 let artifacts = self.merge_repair_artifacts();
7165 self.state.event(
7166 "gate",
7167 format!("gate failed; gate-fix round {attempt} of {cap}"),
7168 );
7169 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7170 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7171 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7174 head: before.clone(),
7175 passed: false,
7176 outcomes: Vec::new(),
7177 });
7178 self.state.save()?;
7179 let (_, seat, out) = self
7180 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7181 prompt: fix_prompt.clone(),
7182 spec,
7183 seat,
7184 cwd: winner.worktree.clone(),
7185 timeout,
7186 allow_write: true,
7187 sessions,
7188 artifacts: artifacts.clone(),
7189 stem: format!("gate-fix-{attempt}"),
7190 handover: None,
7191 })
7192 .await;
7193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7196 return Ok(GateFix::Defer);
7197 }
7198 let mut record = GateFixRecord {
7199 agent: seat.agent.clone(),
7200 failed,
7201 notes: String::new(),
7202 committed: false,
7203 error: None,
7204 };
7205 match out {
7206 AgentOutcome::Ok(o) => {
7207 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7210 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7211 }
7212 }
7213 AgentOutcome::Dropped(_) => {
7214 record.error = Some("the CLI dropped the stream".to_owned());
7215 }
7216 AgentOutcome::Quota(o) => {
7217 self.state.quota.push(QuotaLoss {
7218 seat: seat.key.clone(),
7219 node: "gate-fix".to_owned(),
7220 at: Timestamp::now(),
7221 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7222 });
7223 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7224 }
7225 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7226 }
7227 self.state.seats.insert(seat.key.clone(), seat);
7228 let coordination = self.coordinate(&winner.worktree).await?;
7229 if let Ok(r) = git::rescue_commit(
7230 &winner.worktree,
7231 &format!("magi: gate fix {attempt} (uncommitted work)"),
7232 )
7233 .await
7234 {
7235 self.state.note_withheld("gate-fix", &r.withheld);
7236 }
7237 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7238 record.committed = after != before;
7239 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7240 let note = record.error.clone();
7241 self.state.gate_fixes.push(record);
7242 self.state.save()?;
7243 if !changed {
7244 self.state.event(
7245 "gate",
7246 match note {
7247 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7248 None => format!("gate-fix round {attempt}: the tree did not change"),
7249 },
7250 );
7251 return Ok(GateFix::Stop);
7252 }
7253 self.state.event(
7254 "gate",
7255 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7256 );
7257
7258 drop(coordination);
7259 let wait = self.repository_wait();
7260 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7261 }
7262
7263 async fn guarded_pr_message(
7270 &mut self,
7271 winner: &Candidate,
7272 facts: Option<&BranchFacts>,
7273 posting: bool,
7274 ) -> Result<Option<PrMessage>> {
7275 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7276 let head = match final_head.as_deref() {
7277 Some(h) => crate::pr_outcome::FinalHead::At(h),
7278 None => crate::pr_outcome::FinalHead::Unknown,
7279 };
7280 let summarized = if posting {
7282 Some(self.summarize_change(winner, final_head.as_deref()).await)
7283 } else {
7284 None
7285 };
7286 if self.state.resource_block.is_some() {
7291 self.state.event(
7292 "pr-summary",
7293 "a shared resource was busy; nothing was published, resume to summarize again",
7294 );
7295 return Ok(None);
7296 }
7297 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7298 anyhow::bail!(
7299 "the final commit moved during public description generation; retry before publishing"
7300 );
7301 }
7302 let note = match (&summarized, &self.state.change_summary) {
7303 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7304 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7305 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7306 (None, _) => ChangeNote::NotAsked,
7307 };
7308 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7309 if !posting {
7310 let identity = crate::scrub::Identity::current();
7311 return Ok(Some(PrMessage {
7312 title: crate::scrub::scrub(&pr.title, &identity),
7313 body: crate::scrub::scrub(&pr.body, &identity),
7314 }));
7315 }
7316 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7319 if let Some(known) = self
7320 .state
7321 .github_text
7322 .clone()
7323 .filter(|g| g.fingerprint == raw_fp)
7324 {
7325 return self.settle_withheld(winner, &pr, known).await;
7326 }
7327 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7328 let mut violations =
7329 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7330 let identity = crate::scrub::Identity::current();
7331 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7332 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7333 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7334 {
7335 violations.push(crate::github_text::Violation::SensitiveData);
7336 }
7337 let has_language = violations
7341 .iter()
7342 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7343 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7344 self.state.event(
7345 "github-text",
7346 if has_language {
7347 format!("description rejected: {violations:?}; requesting one rewrite")
7348 } else {
7349 format!("description rejected: {violations:?}; asking the owner")
7350 },
7351 );
7352 let mut rewritten = false;
7353 let mut rejected_rewrite = None;
7354 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7355 {
7356 let output = text;
7357 if let Some(title) = summary_title(&output) {
7358 let raw_body = summary_without_title(&output);
7361 let clean_title = crate::scrub::scrub(&title, &identity);
7362 let mut body = crate::scrub::scrub(&raw_body, &identity);
7363 let redacted = clean_title != title || body != raw_body;
7364 let title = clean_title;
7365 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7366 if !complete {
7367 self.state.event(
7368 "github-text",
7369 "description rewrite omitted required findings",
7370 );
7371 }
7372 if !body.trim().is_empty() {
7373 body.push_str("\n\n");
7374 body.push_str(&public_pr_evidence(&self.state, head));
7375 }
7376 let marker = format!("magi:run/{}", self.state.id);
7377 if !body.trim().is_empty() && !body.contains(&marker) {
7378 body.push_str(&format!("\n\n{marker}"));
7379 }
7380 let rewrite = PrMessage {
7381 title: title.clone(),
7382 body: body.clone(),
7383 };
7384 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7385 let mut failed = crate::github_text::check_for(
7386 &self.state.config.graph,
7387 &title,
7388 &body,
7389 redecision,
7390 );
7391 if (!crate::github_text::shareable(&title)
7392 || !crate::github_text::shareable(&body))
7393 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7394 {
7395 failed.push(crate::github_text::Violation::SensitiveData);
7396 }
7397 if complete
7398 && !title.trim().is_empty()
7399 && !body.trim().is_empty()
7400 && !failed.is_empty()
7401 {
7402 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7403 title: title.clone(),
7404 body: body.clone(),
7405 failed: failed
7406 .iter()
7407 .map(|v| crate::github_text::category(*v).to_owned())
7408 .collect(),
7409 });
7410 }
7411 if complete
7412 && !title.trim().is_empty()
7413 && !raw_body.trim().is_empty()
7414 && failed.is_empty()
7415 {
7416 decision = redecision;
7417 pr = PrMessage { title, body };
7418 rewritten = true;
7419 self.state
7420 .event("github-text", "description rewrite passed");
7421 if redacted {
7422 self.state.event(
7423 "github-text",
7424 "sensitive data removed from description rewrite",
7425 );
7426 }
7427 }
7428 }
7429 }
7430 if !rewritten {
7431 self.state.event(
7432 "github-text",
7433 "description rewrite unavailable or rejected; asking the owner",
7434 );
7435 if let Some(w) =
7436 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7437 {
7438 self.retire_github_text_question("the withheld text changed");
7441 let record = crate::run::GithubTextAsk {
7442 rewrite: rejected_rewrite,
7443 fingerprint: raw_fp,
7444 title: w.title,
7445 body: w.body,
7446 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7447 question: None,
7448 asks: 0,
7449 resolved: false,
7450 chosen_title: None,
7451 choice: None,
7452 };
7453 return self.settle_withheld(winner, &pr, record).await;
7454 }
7455 }
7456 }
7457 let (title, body) =
7458 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7459 Ok(Some(PrMessage { title, body }))
7460 }
7461
7462 async fn summarize_change(
7473 &mut self,
7474 winner: &Candidate,
7475 head: Option<&str>,
7476 ) -> std::result::Result<(), String> {
7477 let Some(head) = head else {
7478 return Err("the final commit could not be read".to_owned());
7479 };
7480 let remote = self.state.config.merge.remote.clone();
7481 let tracking = format!("{remote}/{}", self.state.base_branch);
7482 if !git::rev_exists(&winner.worktree, &tracking).await {
7483 return Err(format!("the remote base `{tracking}` could not be read"));
7484 }
7485 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7486 Ok(b) if !b.is_empty() => b,
7487 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7488 };
7489 let evidence = crate::pr_outcome::summary_evidence(
7490 &self.state,
7491 crate::pr_outcome::FinalHead::At(head),
7492 );
7493 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7494 if self
7495 .state
7496 .change_summary
7497 .as_ref()
7498 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7499 {
7500 return Ok(());
7501 }
7502 if self.state.change_summary.take().is_some() {
7503 self.state.event(
7504 "pr-summary",
7505 "the final diff or evidence changed; the earlier change summary was discarded",
7506 );
7507 }
7508 if let Err(e) = write_artifact(
7510 &self.state,
7511 &format!("public-description-{fingerprint}.md"),
7512 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7513 ) {
7514 tracing::warn!("could not save public description evidence: {e:#}");
7515 }
7516 let chain = match agent::pick_chain(
7517 &self.state.config.agents,
7518 self.state.config.roles.synthesizer.as_ref(),
7519 &agent::installed,
7520 "synthesizer",
7521 ) {
7522 Ok(chain) => chain,
7523 Err(e) => {
7524 tracing::warn!("no agent for the change summary: {e:#}");
7525 self.state.event("pr-summary", "no summary agent available");
7526 return Err("no agent was available to summarize it".to_owned());
7527 }
7528 };
7529 let subjects = git::subjects(&winner.worktree, &base, head)
7530 .await
7531 .unwrap_or_default();
7532 let stat = git::diff_stat(&winner.worktree, &base, head)
7533 .await
7534 .unwrap_or_default();
7535 let stat: String = stat.chars().take(6_000).collect();
7536 let mut prompt = crate::prompt::change_summary_in(
7537 &base,
7538 head,
7539 &subjects,
7540 stat.trim(),
7541 &self.state.config.graph.github_language,
7542 );
7543 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7544 let sessions = self.state.config.graph.sessions;
7545 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7546 let prompts = self.state.config.prompts.clone();
7547 let cache = self.state.config.cache_dir();
7548 let run = self.state.id.clone();
7549 for (i, spec) in chain.into_iter().enumerate() {
7550 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7551 let job = SeatJob {
7552 spec: spec.clone(),
7553 seat,
7554 cwd: winner.worktree.clone(),
7555 prompt: prompt.clone(),
7556 timeout,
7557 allow_write: false,
7558 sessions,
7559 artifacts: agent::artifacts_dir(&self.state.dir()),
7560 stem: if i == 0 {
7561 "change-summary".to_owned()
7562 } else {
7563 format!("change-summary-{}", spec.id)
7564 },
7565 handover: None,
7566 };
7567 let ctx = WaveCtx {
7568 run: &run,
7569 node: "pr-summary",
7570 prompts: &prompts,
7571 cache: cache.as_deref(),
7572 round: None,
7573 carry_seats: false,
7574 pauses: Some((&self.pause, &self.interrupt)),
7575 };
7576 let (seat, outcome) =
7577 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7578 self.state.seats.insert(seat.key.clone(), seat);
7579 let text = match outcome {
7580 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7581 AgentOutcome::ResourceBlocked(why) => {
7582 if self.state.resource_block.is_none() {
7585 self.state.resource_block = Some(crate::run::ResourceBlock {
7586 node: "pr-summary".to_owned(),
7587 kind: "shared resource".to_owned(),
7588 detail: why,
7589 at: Timestamp::now(),
7590 });
7591 }
7592 return Err("a shared resource was busy".to_owned());
7593 }
7594 _ => continue,
7595 };
7596 let blind = &self.state.config.blind;
7597 let Some(title) = summary_title(&text) else {
7598 continue;
7599 };
7600 let body: String = summary_without_title(&text)
7601 .lines()
7602 .filter(|l| !l.contains("magi:run/"))
7603 .collect::<Vec<_>>()
7604 .join("\n");
7605 let body = blind::sanitize_prose(body.trim(), blind);
7606 let title = blind::sanitize_prose(&title, blind);
7607 if body.trim().is_empty() || title.trim().is_empty() {
7608 continue;
7609 }
7610 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7611 continue;
7612 }
7613 if git::rev_parse(&winner.worktree, "HEAD")
7615 .await
7616 .ok()
7617 .as_deref()
7618 != Some(head)
7619 {
7620 return Err("the branch moved while the summary was written".to_owned());
7621 }
7622 if git::merge_base(&winner.worktree, &tracking, head)
7623 .await
7624 .ok()
7625 .as_deref()
7626 != Some(base.as_str())
7627 || crate::pr_outcome::summary_evidence(
7628 &self.state,
7629 crate::pr_outcome::FinalHead::At(head),
7630 ) != evidence
7631 {
7632 return Err("the summary evidence changed while it was written".to_owned());
7633 }
7634 self.state.change_summary = Some(crate::run::ChangeSummary {
7635 head: head.to_owned(),
7636 base,
7637 evidence: fingerprint,
7638 title,
7639 body,
7640 });
7641 self.state.event(
7642 "pr-summary",
7643 format!("change summary written by {} for the final diff", spec.id),
7644 );
7645 if let Err(e) = self.state.save() {
7646 tracing::warn!("could not save the change summary: {e:#}");
7647 }
7648 return Ok(());
7649 }
7650 self.state
7651 .event("pr-summary", "no agent produced a change summary");
7652 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7653 }
7654
7655 async fn rewrite_pr_text(
7663 &mut self,
7664 winner: &Candidate,
7665 pr: &PrMessage,
7666 violations: &[crate::github_text::Violation],
7667 ) -> Option<String> {
7668 let prompt = format!(
7669 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7670 pr.title, pr.body
7671 );
7672 let sessions = self.state.config.graph.sessions;
7673 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7674 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7676 let key = format!("impl-{}", winner.label);
7677 if let Some(spec) = self
7678 .state
7679 .config
7680 .agents
7681 .iter()
7682 .find(|a| a.id == winner.agent)
7683 .cloned()
7684 && let Some(existing) = self.state.seats.get(&key)
7685 && existing.agent == spec.id
7686 && agent::has_session(spec.kind, existing, sessions)
7687 {
7688 attempts.push((spec, existing.clone(), true));
7689 } else {
7690 let chain = agent::pick_chain(
7691 &self.state.config.agents,
7692 self.state.config.roles.language_judge.as_ref(),
7693 &agent::installed,
7694 "language_judge",
7695 );
7696 match chain {
7697 Ok(chain) => {
7698 for spec in chain {
7699 let seat = crate::fixer::seat_for(
7700 &mut self.state,
7701 "github-text-rewrite",
7702 &spec.id,
7703 );
7704 attempts.push((spec, seat, false));
7705 }
7706 }
7707 Err(e) => {
7708 tracing::warn!("no agent for the description rewrite: {e:#}");
7709 self.state
7710 .event("github-text", "no rewrite agent available");
7711 return None;
7712 }
7713 }
7714 }
7715 let prompts = self.state.config.prompts.clone();
7716 let cache = self.state.config.cache_dir();
7717 let run = self.state.id.clone();
7718 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7719 let job = SeatJob {
7720 spec: spec.clone(),
7721 seat,
7722 cwd: winner.worktree.clone(),
7723 prompt: prompt.clone(),
7724 timeout: retry_budget(timeout_full, resumed),
7725 allow_write: false,
7726 sessions,
7727 artifacts: agent::artifacts_dir(&self.state.dir()),
7728 stem: if i == 0 {
7729 "github-text-rewrite".to_owned()
7730 } else {
7731 format!("github-text-rewrite-{}", spec.id)
7732 },
7733 handover: None,
7734 };
7735 let ctx = WaveCtx {
7736 run: &run,
7737 node: "github-text",
7738 prompts: &prompts,
7739 cache: cache.as_deref(),
7740 round: None,
7741 carry_seats: false,
7742 pauses: Some((&self.pause, &self.interrupt)),
7743 };
7744 let (seat, outcome) =
7745 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7746 self.state.seats.insert(seat.key.clone(), seat);
7747 match outcome {
7748 AgentOutcome::Ok(o)
7749 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7750 {
7751 return Some(o.text);
7752 }
7753 AgentOutcome::ResourceBlocked(_) => return None,
7754 _ => {}
7755 }
7756 }
7757 None
7758 }
7759
7760 fn retire_github_text_question(&mut self, why: &str) {
7762 let Some(id) = self
7763 .state
7764 .github_text
7765 .as_ref()
7766 .and_then(|g| g.question.clone())
7767 else {
7768 return;
7769 };
7770 let _ = ask::Questions::open().update(&id, |q| {
7771 q.abandon(why);
7772 Ok(())
7773 });
7774 }
7775
7776 async fn settle_withheld(
7780 &mut self,
7781 winner: &Candidate,
7782 pr: &PrMessage,
7783 mut g: crate::run::GithubTextAsk,
7784 ) -> Result<Option<PrMessage>> {
7785 use crate::github_text as gt;
7786 let store = ask::Questions::open();
7787 let timeout = self.state.config.graph.answer_timeout;
7788 if !g.resolved {
7789 let mut standing = g
7790 .question
7791 .as_ref()
7792 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7793 if let Some(q) = standing.as_ref()
7794 && q.status.open()
7795 && gt::expired(q, timeout)
7796 {
7797 let why = format!("no answer within {}s of asking", timeout.max(1));
7798 standing = store
7799 .update(&q.id, |q| {
7800 q.abandon(&why);
7801 Ok(())
7802 })
7803 .ok()
7804 .map(|(q, ())| q);
7805 }
7806 let mut ask_again: Option<Option<Vec<String>>> = None;
7807 match standing {
7808 None if g.asks <= 1 => {
7811 g.asks = 0;
7812 ask_again = Some(None);
7813 }
7814 None => {
7815 g.resolved = true;
7816 g.chosen_title = None;
7817 g.choice = Some(GithubTextChoice::Fallback);
7818 }
7819 Some(q) if q.status.open() => {
7820 self.park_on_github_text(&g)?;
7821 return Ok(None);
7822 }
7823 Some(q) => match gt::read_reply(&q) {
7824 gt::Reply::Fallback => {
7825 g.resolved = true;
7826 g.chosen_title = None;
7827 g.choice = Some(GithubTextChoice::Fallback);
7828 }
7829 gt::Reply::Original => {
7830 g.resolved = true;
7831 g.chosen_title = None;
7832 g.choice = Some(GithubTextChoice::Original);
7833 }
7834 gt::Reply::Rewrite => {
7835 g.resolved = true;
7836 g.chosen_title = None;
7837 g.choice = Some(if g.rewrite.is_some() {
7838 GithubTextChoice::Rewrite
7839 } else {
7840 GithubTextChoice::Fallback
7841 });
7842 }
7843 gt::Reply::Title(text) => {
7844 let first = text
7845 .lines()
7846 .map(str::trim)
7847 .find(|l| !l.is_empty())
7848 .unwrap_or("");
7849 let decision = if first.is_empty() {
7850 None
7851 } else {
7852 let candidate = PrMessage {
7853 title: first.to_owned(),
7854 body: String::new(),
7855 };
7856 self.judge_pr_language(&winner.worktree, &candidate).await
7857 };
7858 match gt::vet_title(
7859 &text,
7860 decision,
7861 self.state.config.graph.github_english(),
7862 ) {
7863 Ok(title) => {
7864 g.resolved = true;
7865 g.chosen_title = Some(title);
7866 g.choice = Some(GithubTextChoice::MyTitle);
7867 }
7868 Err(cats) if g.asks < 2 => {
7869 ask_again = Some(Some(
7870 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7871 ))
7872 }
7873 Err(_) => {
7874 g.resolved = true;
7875 g.chosen_title = None;
7876 g.choice = Some(GithubTextChoice::Fallback);
7877 }
7878 }
7879 }
7880 },
7881 }
7882 if let Some(rejected) = ask_again {
7883 let w = gt::Withheld {
7886 title: g.title,
7887 body: g.body,
7888 categories: g.categories.clone(),
7889 };
7890 let artifact = match crate::run::write_artifact(
7893 &self.state,
7894 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7895 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7896 ) {
7897 Ok(p) => Some(p),
7898 Err(e) => {
7899 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7900 None
7901 }
7902 };
7903 let (detail, comparison) = gt::question_content(
7904 &w,
7905 &pr.title,
7906 &pr.body,
7907 &self.state.id,
7908 rejected.as_deref(),
7909 artifact.as_deref(),
7910 g.rewrite.as_ref(),
7911 );
7912 let mut q = ask::Question::new(
7913 self.state.id.clone(),
7914 gt::ASK_NODE.to_owned(),
7915 gt::ASK_SEAT.to_owned(),
7916 gt::question_summary(&self.state.config.graph.language, &w),
7917 detail,
7918 gt::question_choices(&w, g.rewrite.as_ref()),
7919 );
7920 q.posting_comparison = Some(comparison);
7921 q.answer_timeout = timeout;
7922 g.question = Some(q.id.clone());
7925 g.asks += 1;
7926 self.state.github_text = Some(g.clone());
7927 self.state.event(
7928 "github-text",
7929 format!("asking the owner about the withheld text ({})", q.short()),
7930 );
7931 self.state.save()?;
7932 store
7933 .put_executable(&mut q)
7934 .context("file the posting-gate question")?;
7935 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7936 tracing::warn!(
7937 "could not notify about posting-gate question {}: {e:#}",
7938 q.short()
7939 );
7940 }
7941 self.park_on_github_text(&g)?;
7942 return Ok(None);
7943 }
7944 }
7945 let resolved = g.chosen_title.clone();
7946 let original = g.choice == Some(GithubTextChoice::Original);
7947 self.state.event(
7949 "github-text",
7950 if original {
7951 format!(
7952 "posting the original text for: {}",
7953 match (g.title, g.body) {
7954 (true, true) => "title, description",
7955 (true, false) => "title",
7956 _ => "description",
7957 }
7958 )
7959 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7960 "posting the saved rewrite title and description".to_owned()
7961 } else if resolved.is_some() {
7962 "posting the owner's replacement title".to_owned()
7963 } else {
7964 "posting the neutral text for the withheld fields".to_owned()
7965 },
7966 );
7967 self.state.github_text = Some(g.clone());
7968 self.state.save()?;
7969 if g.choice == Some(GithubTextChoice::Rewrite)
7970 && let Some(r) = g.rewrite.as_ref()
7971 {
7972 return Ok(Some(PrMessage {
7973 title: r.title.clone(),
7974 body: r.body.clone(),
7975 }));
7976 }
7977 let w = gt::Withheld {
7978 title: g.title,
7979 body: g.body,
7980 categories: g.categories.clone(),
7981 };
7982 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7983 if let Some(t) = resolved {
7984 title = t;
7985 }
7986 let english = gt::LanguageDecision {
7988 title_english: true,
7989 body_english: true,
7990 body_complete: true,
7991 };
7992 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7993 if original {
7994 if g.title {
7997 title = pr.title.clone();
7998 }
7999 if g.body {
8000 body = pr.body.clone();
8001 }
8002 }
8003 Ok(Some(PrMessage { title, body }))
8004 }
8005
8006 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8007 self.state.github_text = Some(g.clone());
8008 self.state.end_pass_parked(ParkCause::Question);
8009 self.state.parked = true;
8010 self.state.event(
8011 "github-text",
8012 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8013 );
8014 self.state.save()
8015 }
8016
8017 async fn judge_pr_language(
8020 &mut self,
8021 cwd: &Path,
8022 pr: &PrMessage,
8023 ) -> Option<crate::github_text::LanguageDecision> {
8024 self.state.config.roles.language_judge.as_ref()?;
8025 if !self.state.config.graph.language_gate() {
8026 return None;
8027 }
8028 let decision =
8029 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8030 self.state.event(
8031 "github-text",
8032 if decision.is_some() {
8033 "language decided by the language judge"
8034 } else {
8035 "language judge unavailable; using built-in heuristics"
8036 },
8037 );
8038 decision
8039 }
8040
8041 async fn merge(&mut self) -> Result<()> {
8042 if self
8057 .state
8058 .base_sync
8059 .as_ref()
8060 .is_some_and(|s| s.conflict.is_some())
8061 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8062 != Some(RunStatus::Gating)
8063 || !self.state.gate_allows_progress()
8071 {
8072 return Ok(());
8073 }
8074 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8083 return Ok(());
8084 }
8085 let Some(winner) = self.state.winner().cloned() else {
8086 return Ok(());
8087 };
8088 let repo = self.state.repo.clone();
8089 let base = self.state.base_branch.clone();
8090 let mode = self.state.config.merge.mode;
8091 let style = self.state.config.merge.style;
8092 let facts = if is_review_run(&self.state) {
8093 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8094 let start = review_base(
8095 &repo,
8096 &self.state.config.merge.remote,
8097 &base,
8098 &self.state.base_commit,
8099 &winner.branch,
8100 )
8101 .await;
8102 branch_facts(&repo, &start, &winner.branch).await
8103 } else {
8104 None
8105 };
8106 let leaked = if is_review_run(&self.state) {
8109 leaked_subjects(&self.state, &winner.branch).await
8110 } else {
8111 Some(Vec::new())
8112 };
8113 let Some(pr) = self
8117 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8118 .await?
8119 else {
8120 return Ok(());
8121 };
8122 let message = pr.commit_message();
8123
8124 let outcome = match mode {
8125 MergeMode::None => MergeOutcome {
8126 mode,
8127 ok: true,
8128 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8129 empty: false,
8130 },
8131 MergeMode::Pr | MergeMode::Local
8132 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8133 {
8134 MergeOutcome {
8135 mode,
8136 ok: false,
8137 detail: empty_candidate_detail(&self.state, &base),
8138 empty: true,
8139 }
8140 }
8141 MergeMode::Local => {
8142 let on = git::current_branch(&repo).await?;
8143 if on.as_deref() != Some(base.as_str()) {
8144 MergeOutcome {
8145 mode,
8146 ok: false,
8147 detail: format!(
8148 "{} has {} checked out, not the base branch {base}",
8149 repo.display(),
8150 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8151 ),
8152 empty: false,
8153 }
8154 } else if !git::is_clean(&repo).await? {
8155 MergeOutcome {
8156 mode,
8157 ok: false,
8158 detail: format!("{} is dirty; refusing to merge", repo.display()),
8159 empty: false,
8160 }
8161 } else {
8162 let out = match style {
8163 MergeStyle::Merge => {
8164 git::merge_no_ff(&repo, &winner.branch, &message).await?
8165 }
8166 MergeStyle::Squash => {
8167 git::merge_squash(&repo, &winner.branch, &message).await?
8168 }
8169 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8170 };
8171 MergeOutcome {
8172 mode,
8173 ok: out.ok(),
8174 detail: if out.ok() { out.stdout } else { out.stderr },
8175 empty: false,
8176 }
8177 }
8178 }
8179 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8180 self.state.pr_publication_pending = true;
8181 self.state.save()?;
8182 let out =
8183 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8184 self.state.pr_publication_pending = out.is_err();
8185 MergeOutcome {
8186 mode,
8187 ok: out.is_ok(),
8188 detail: match out {
8189 Ok(url) => url,
8190 Err(e) => format!("{e:#}"),
8191 },
8192 empty: false,
8193 }
8194 }
8195 MergeMode::Pr => {
8196 let remote = self.state.config.merge.remote.clone();
8197 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8198 if !pushed.ok() {
8199 MergeOutcome {
8200 mode,
8201 ok: false,
8202 detail: pushed.stderr,
8203 empty: false,
8204 }
8205 } else {
8206 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8212 let out = match pr_merge_plan(found) {
8213 PrPlan::Create => {
8214 gh_pr_create(
8215 &winner.worktree,
8216 &base,
8217 &winner.branch,
8218 &pr.title,
8219 &pr.body,
8220 )
8221 .await
8222 }
8223 PrPlan::Adopt { url, title } => {
8224 self.state
8225 .event("merge", format!("Pr: adopted open pull request {url}"));
8226 if self.state.github_text.as_ref().is_some_and(|g| {
8227 g.resolved
8228 && g.choice == Some(GithubTextChoice::Rewrite)
8229 && g.rewrite.is_some()
8230 }) {
8231 land::set_pr_text_verbatim(
8234 &winner.worktree,
8235 &url,
8236 &pr.title,
8237 &pr.body,
8238 )
8239 .await
8240 .map(|()| url)
8241 } else {
8242 if title != pr.title
8243 && (!is_review_run(&self.state)
8244 || leaked
8245 .as_deref()
8246 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8247 && let Err(e) =
8248 if self.state.github_text.as_ref().is_some_and(|g| {
8249 g.resolved
8250 && g.title
8251 && g.choice == Some(GithubTextChoice::Original)
8252 }) {
8253 land::set_pr_title_verbatim(
8255 &winner.worktree,
8256 &url,
8257 &pr.title,
8258 )
8259 .await
8260 } else {
8261 land::set_pr_title(
8262 &mut self.state,
8263 &winner.worktree,
8264 &url,
8265 &pr.title,
8266 )
8267 .await
8268 }
8269 {
8270 tracing::warn!("could not refresh title of {url}: {e:#}");
8271 self.state
8272 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8273 }
8274 Ok(url)
8275 }
8276 }
8277 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8278 };
8279 match out {
8280 Ok(url) => MergeOutcome {
8281 mode,
8282 ok: true,
8283 detail: url,
8284 empty: false,
8285 },
8286 Err(e) => MergeOutcome {
8287 mode,
8288 ok: false,
8289 detail: e.to_string(),
8290 empty: false,
8291 },
8292 }
8293 }
8294 }
8295 };
8296
8297 self.state.status = match (mode, outcome.ok) {
8298 (MergeMode::None, _) => RunStatus::Ready,
8299 (_, true) if outcome.empty => RunStatus::Ready,
8300 (MergeMode::Pr, true) => RunStatus::Ready,
8301 (_, true) => RunStatus::Merged,
8302 (_, false) => RunStatus::Blocked,
8303 };
8304 self.state.event(
8305 "merge",
8306 format!(
8307 "{:?}: {}",
8308 mode,
8309 outcome.detail.lines().next().unwrap_or("")
8310 ),
8311 );
8312 self.state.merge = Some(outcome);
8313 self.state.save()?;
8314
8315 if self.state.config.graph.land
8321 && mode == MergeMode::Pr
8322 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8323 {
8324 self.run_land().await?;
8325 }
8326 self.settle_questions();
8331 Ok(())
8332 }
8333
8334 async fn run_land(&mut self) -> Result<()> {
8345 crate::pr_head::ensure_owner(&self.state)?;
8346 let url = self
8347 .state
8348 .pr
8349 .as_ref()
8350 .map(|pr| pr.url.clone())
8351 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8352 .unwrap_or_default();
8353 let url = url.lines().next().unwrap_or("").trim().to_owned();
8354 if !url.starts_with("http") {
8355 return Ok(());
8356 }
8357 loop {
8360 match land::land(&mut self.state, &url).await {
8361 Ok(_)
8362 if self
8363 .state
8364 .merge_repairs
8365 .last()
8366 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8367 {
8368 if self.repair_merge_review().await? {
8369 continue;
8370 }
8371 }
8372 Ok(pr) if self.state.parked => {
8373 let _ = pr;
8377 }
8378 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8379 self.state.status = RunStatus::Superseded;
8380 self.state.save()?;
8381 }
8382 Ok(pr) => {
8383 self.state.status = match pr.state {
8384 land::PrLifecycle::Merged => RunStatus::Merged,
8385 _ => RunStatus::Blocked,
8386 };
8387 if bump::should_release_bump(self.state.status)
8394 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8395 {
8396 self.state
8402 .event("bump", format!("release bump skipped: {e:#}"));
8403 }
8404 if self.state.status == RunStatus::Merged {
8407 crate::followup::after_merge(&mut self.state, &pr.url).await;
8408 }
8409 self.state.save()?;
8410 }
8411 Err(e) => {
8412 if e.is::<crate::repo_lock::ResourceBlocked>() {
8413 return Err(e);
8414 }
8415 self.state.status = RunStatus::Blocked;
8416 self.state.event("land", format!("gave up: {e}"));
8417 self.state.save()?;
8418 }
8419 }
8420 break;
8421 }
8422 Ok(())
8423 }
8424
8425 async fn repair_merge_review(&mut self) -> Result<bool> {
8428 use crate::run::MergeRepairPhase as Phase;
8429 crate::pr_head::ensure_owner(&self.state)?;
8430 let i = self.state.merge_repairs.len() - 1;
8431 let request = self.state.merge_repairs[i].clone();
8432 if request.phase == Phase::Fixing {
8433 let restored = match self.state.winner().cloned() {
8434 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8435 None => Err(anyhow::anyhow!("repair has no winning branch")),
8436 };
8437 let why = match restored {
8438 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8439 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8440 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8441 };
8442 self.hold_merge_repair(i, why)?;
8443 return Ok(false);
8444 }
8445 let failure = if i >= self.state.config.graph.land_rounds
8446 || self.state.config.graph.review_rounds == 0
8447 {
8448 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8449 } else {
8450 None
8451 };
8452 if let Some(why) = failure {
8453 self.hold_merge_repair(i, why)?;
8454 return Ok(false);
8455 }
8456 if request.phase == Phase::Requested {
8457 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8458 Ok(pr) => pr,
8459 Err(e) => {
8460 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8461 return Ok(false);
8462 }
8463 };
8464 self.state.merge_repairs[i].phase = Phase::Fixing;
8465 self.state.land_approval = None;
8466 self.state.land_armed_head = None;
8467 self.state.save()?;
8468 let evidence = serde_json::to_string_pretty(&request)?;
8471 let reason = format!(
8472 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8473 );
8474 let budget = self.state.config.graph.land_rounds;
8475 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8476 match result {
8477 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8478 Ok(land::Fixed::Failed(why)) => {
8479 self.hold_merge_repair(
8480 i,
8481 format!(
8482 "repair failed: {why}; inspect the branch and run an explicit review"
8483 ),
8484 )?;
8485 return Ok(false);
8486 }
8487 Err(e) => {
8488 self.hold_merge_repair(
8489 i,
8490 format!(
8491 "repair failed: {e:#}; inspect the branch and run an explicit review"
8492 ),
8493 )?;
8494 return Ok(false);
8495 }
8496 }
8497 self.begin_merge_repair_review(i)?;
8501 }
8502 self.review_loop().await?;
8503 self.gate().await?;
8504 if self.resource_halt()? || self.state.parked {
8505 return Ok(false);
8506 }
8507 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8508 != Some(RunStatus::Gating)
8509 || !self.state.gate_allows_progress()
8510 {
8511 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8512 return Ok(false);
8513 }
8514 let winner = self
8517 .state
8518 .winner()
8519 .cloned()
8520 .context("repair has no winning branch")?;
8521 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8522 if self.state.gate_head.as_deref() != Some(after.as_str())
8523 || !git::is_clean(&winner.worktree).await?
8524 {
8525 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8526 return Ok(false);
8527 }
8528 let expected = self
8529 .state
8530 .pr_pushed_head
8531 .as_deref()
8532 .unwrap_or(&request.head)
8533 .to_owned();
8534 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8535 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8538 let refspec = format!("{after}:refs/heads/{public}");
8539 let push = git::git_raw(
8540 &winner.worktree,
8541 &["push", &lease, &self.state.config.merge.remote, &refspec],
8542 )
8543 .await;
8544 match push {
8545 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8546 Ok(out) => {
8547 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8548 return Ok(false);
8549 }
8550 Err(e) => {
8551 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8552 return Ok(false);
8553 }
8554 }
8555 self.state.merge_repairs[i].phase = Phase::Complete;
8556 self.state.status = RunStatus::Landing;
8557 self.state.event(
8558 "repair",
8559 "new review and gate completed; fresh owner merge decision required",
8560 );
8561 self.state.save()?;
8562 Ok(true)
8563 }
8564
8565 fn merge_repair_artifacts(&self) -> PathBuf {
8566 let artifacts = agent::artifacts_dir(&self.state.dir());
8567 match self
8568 .state
8569 .merge_repairs
8570 .last()
8571 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8572 {
8573 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8574 None => artifacts,
8575 }
8576 }
8577
8578 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8579 self.state.reviews.clear();
8580 self.state.gate.clear();
8581 self.state.gate_ran = false;
8582 self.state.gate_head = None;
8583 self.state.gate_fixes.clear();
8584 self.state.gate_repair_e2e = None;
8585 self.state.pre_gate.clear();
8586 self.state.pre_gate_commit = None;
8587 self.state.contested_handoff = None;
8588 self.state.land_reruns = None;
8589 self.state.land_approval = None;
8590 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8591 self.state.status = RunStatus::Reviewing;
8592 self.state.save()
8593 }
8594
8595 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8596 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8597 self.state.merge_repairs[index].failure = Some(why.clone());
8598 self.state.status = RunStatus::Blocked;
8599 self.state.event("repair", why);
8600 self.state.save()
8601 }
8602
8603 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8607 if let Some(existing) = self.state.seats.get(key)
8608 && existing.agent == agent
8609 {
8610 return existing.clone();
8611 }
8612 let fresh = if self.state.seats.contains_key(key) {
8615 handover_seat(key, agent, self.state.next_seat_seed())
8616 } else {
8617 SeatState::new(key, agent, self.state.seed)
8618 };
8619 self.state.seats.insert(key.to_owned(), fresh.clone());
8620 fresh
8621 }
8622
8623 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8629 match self.state.seats.get(key) {
8630 Some(s) if s.agent != spec.id => {
8631 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8632 }
8633 _ => spec,
8634 }
8635 }
8636
8637 fn view(&self, c: &Candidate) -> CandidateView {
8639 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8640 .unwrap_or_default();
8641 let (patch, _) = blind::sanitize_patch(
8642 &format!("candidate {} patch", c.label),
8643 &raw,
8644 &self.state.config.blind,
8645 );
8646 CandidateView {
8647 label: c.label,
8648 branch: c.branch.clone(),
8649 summary: c.summary.clone(),
8650 stat: c.stat.clone(),
8651 patch,
8652 }
8653 }
8654
8655 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8657 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8658 prompt::judge(
8659 "(see above)",
8660 &views,
8661 self.roles.judges.len(),
8662 base_short,
8663 "en",
8664 )
8665 }
8666
8667 fn vote_prompt_full(
8673 &self,
8674 j: usize,
8675 labels: &[char],
8676 language: &str,
8677 candidates: &[Candidate],
8678 base_short: &str,
8679 ) -> String {
8680 let mut text = format!(
8681 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8682 prompt::final_vote(labels, language),
8683 self.state.instruction,
8684 self.candidate_block(candidates, base_short)
8685 );
8686 if let Some(own) = self
8687 .state
8688 .judgements
8689 .get(j)
8690 .filter(|r| !r.ranking.is_empty())
8691 {
8692 let reasons = own
8693 .reasons
8694 .iter()
8695 .map(|(k, v)| format!("- {k}: {v}"))
8696 .collect::<Vec<_>>()
8697 .join("\n");
8698 text.push_str(&format!(
8699 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8700 own.ranking.iter().collect::<String>(),
8701 if reasons.is_empty() {
8702 ""
8703 } else {
8704 ", because:\n"
8705 }
8706 ));
8707 }
8708 if !self.state.deliberation.is_empty() {
8709 text.push_str("\n# What was argued before this vote\n");
8710 for t in self.transcript(&[], j) {
8711 text.push_str(&format!(
8712 "\n## {}{}\n\n{}\n",
8713 t.who,
8714 if t.is_self { " (you)" } else { "" },
8715 t.body.trim()
8716 ));
8717 }
8718 }
8719 text
8720 }
8721
8722 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8729 let mut turns = Vec::new();
8730 for j in &self.state.judgements {
8731 if j.ranking.is_empty() {
8732 continue;
8733 }
8734 let reasons = j
8735 .reasons
8736 .iter()
8737 .map(|(k, v)| format!("- {k}: {v}"))
8738 .collect::<Vec<_>>()
8739 .join("\n");
8740 turns.push(Turn {
8741 who: format!("Judge {} (opening ranking)", j.judge),
8742 is_self: j.judge == self_idx + 1,
8743 body: format!(
8744 "Ranked {}{}{reasons}",
8745 j.ranking.iter().collect::<String>(),
8746 if reasons.is_empty() {
8747 ""
8748 } else {
8749 ", because:\n"
8750 }
8751 ),
8752 });
8753 }
8754 for t in self
8755 .state
8756 .deliberation
8757 .iter()
8758 .flat_map(|r| r.turns.iter())
8759 .chain(current)
8760 {
8761 turns.push(Turn {
8762 who: format!("Judge {}", t.judge),
8763 is_self: t.judge == self_idx + 1,
8764 body: t.body.clone(),
8765 });
8766 }
8767 turns
8768 }
8769}
8770
8771fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8773 agent::has_session(spec.kind, seat, sessions)
8774}
8775
8776fn next_untried_in_roster<'a>(
8797 roster: &'a [AgentSpec],
8798 start: usize,
8799 tried: &BTreeSet<String>,
8800) -> Option<&'a AgentSpec> {
8801 roster
8802 .get(start + 1..)?
8803 .iter()
8804 .find(|s| !tried.contains(&s.id))
8805}
8806
8807fn pick_successor<'a>(
8828 roster: &'a [AgentSpec],
8829 start: usize,
8830 tried: &BTreeSet<String>,
8831 carried: Option<&BTreeSet<String>>,
8832 others: &BTreeSet<String>,
8833) -> Option<&'a AgentSpec> {
8834 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8835 roster
8836 .get(start + 1..)
8837 .and_then(|tail| {
8838 tail.iter()
8839 .filter(free)
8840 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8841 })
8842 .or_else(|| {
8843 carried
8844 .is_some()
8845 .then(|| roster.iter().find(free))
8846 .flatten()
8847 })
8848 .or_else(|| match carried {
8849 Some(c) => next_for_seat(roster, start, tried, c),
8850 None => next_untried_in_roster(roster, start, tried),
8851 })
8852}
8853
8854fn next_for_seat<'a>(
8864 roster: &'a [AgentSpec],
8865 start: usize,
8866 round_tried: &BTreeSet<String>,
8867 carried_failed: &BTreeSet<String>,
8868) -> Option<&'a AgentSpec> {
8869 roster
8870 .get(start + 1..)?
8871 .iter()
8872 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8873 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8874}
8875
8876fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8882 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8883 return spec;
8884 };
8885 let ok = |id: &str| !h.failed.contains(id);
8886 if let Some(last) = h.last_ok.as_deref()
8887 && ok(last)
8888 && let Some(s) = roster.iter().find(|s| s.id == last)
8889 {
8890 return s.clone();
8891 }
8892 if ok(&spec.id) {
8893 return spec;
8894 }
8895 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8896 roster
8897 .get(from + 1..)
8898 .into_iter()
8899 .flatten()
8900 .chain(roster.iter())
8901 .find(|s| ok(&s.id))
8902 .cloned()
8903 .unwrap_or(spec)
8904}
8905
8906pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8910 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8911}
8912
8913const TIMED_OUT: &str = "timed out";
8916
8917impl FailClass {
8918 fn of(out: &AgentOutcome) -> Option<Self> {
8920 match out {
8921 AgentOutcome::Ok(_) => None,
8922 AgentOutcome::Quota(_) => Some(Self::Quota),
8923 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8924 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8925 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8926 AgentOutcome::ResourceBlocked(_) => None,
8929 }
8930 }
8931
8932 fn stem_word(&self) -> &'static str {
8934 match self {
8935 Self::Quota => "quota",
8936 _ => "handover",
8937 }
8938 }
8939}
8940
8941fn failure_signature(msg: &str) -> String {
8945 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8946 if let Some(code) = line
8949 .strip_prefix("process exited with no usable output (exit code ")
8950 .and_then(|s| s.strip_suffix(')'))
8951 .and_then(|s| s.parse::<i32>().ok())
8952 {
8953 line = format!("exited with some({code}) and no usable output");
8954 } else if matches!(
8955 line.as_str(),
8956 "process exited with no usable output (no exit code reported)"
8957 | "process exited with no usable output (exit code unavailable)"
8958 ) {
8959 line = "exited with none and no usable output".to_owned();
8960 }
8961 let mut out = Vec::new();
8962 for word in line.split_whitespace() {
8963 if word.contains('/') || word.contains('\\') {
8964 out.push("<path>".to_owned());
8965 continue;
8966 }
8967 let mut w = String::new();
8968 let mut in_digits = false;
8969 for c in word.chars() {
8970 if c.is_ascii_digit() {
8971 if !in_digits {
8972 w.push('#');
8973 }
8974 in_digits = true;
8975 } else {
8976 in_digits = false;
8977 w.push(c);
8978 }
8979 }
8980 out.push(w);
8981 }
8982 out.join(" ").chars().take(120).collect()
8983}
8984
8985fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8992 match cur {
8993 FailClass::Quota | FailClass::Timeout => true,
8994 FailClass::Other(_) => prev != Some(cur),
8995 }
8996}
8997
8998fn fail_reason(out: &AgentOutcome) -> String {
9000 match out {
9001 AgentOutcome::Ok(_) => String::new(),
9002 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9003 AgentOutcome::Dropped(o) => format!(
9004 "the CLI dropped the stream ({})",
9005 o.dropped
9006 .as_ref()
9007 .map(|d| d.why.as_str())
9008 .unwrap_or("it ended without delivering its answer")
9009 ),
9010 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9011 e.lines().next().unwrap_or("").chars().take(160).collect()
9012 }
9013 }
9014}
9015
9016pub(crate) fn record_handover(
9019 state: &mut RunState,
9020 node: &str,
9021 seat: &str,
9022 from: &str,
9023 to: &str,
9024 class: &FailClass,
9025 reason: &str,
9026) {
9027 let message = if *class == FailClass::Quota {
9028 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9029 } else {
9030 format!("{seat}: handed over {from} -> {to} ({reason})")
9031 };
9032 state.event(node, message);
9033 state.handovers.push(Handover {
9034 at: Timestamp::now(),
9035 node: node.to_owned(),
9036 seat: seat.to_owned(),
9037 from: from.to_owned(),
9038 to: to.to_owned(),
9039 reason: reason.to_owned(),
9040 });
9041}
9042
9043fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9057 commands.iter().any(|c| c.exit_code.is_none())
9058}
9059
9060fn verified_noop_claim(
9073 usable: bool,
9074 commands: &[agent::CommandEvidence],
9075 text: &str,
9076) -> Option<String> {
9077 (usable && !has_unconfirmed_command(commands))
9078 .then(|| verdict::verified_noop(text))
9079 .flatten()
9080}
9081
9082fn short(commit: &str) -> String {
9083 commit.chars().take(7).collect()
9084}
9085
9086fn make_executable(path: &Path) -> Result<()> {
9087 #[cfg(unix)]
9088 {
9089 use std::os::unix::fs::PermissionsExt as _;
9090 let mut perms = std::fs::metadata(path)?.permissions();
9091 perms.set_mode(0o755);
9092 std::fs::set_permissions(path, perms)?;
9093 }
9094 #[cfg(not(unix))]
9095 {
9096 let _ = path;
9097 }
9098 Ok(())
9099}
9100
9101struct WaveCtx<'a> {
9108 run: &'a str,
9111 node: &'a str,
9113 prompts: &'a Prompts,
9114 cache: Option<&'a Path>,
9116 round: Option<usize>,
9119 carry_seats: bool,
9123 pauses: Option<(&'a Pause, &'a Pause)>,
9126}
9127
9128fn repository_wait(
9130 state: &RunState,
9131 pauses: Option<(&Pause, &Pause)>,
9132) -> crate::repo_lock::WaitPolicy {
9133 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9134 let inherited = crate::repo_lock::WaitPolicy::current();
9135 crate::repo_lock::WaitPolicy::new(
9136 Duration::from_secs(state.config.graph.timeout_resource_wait),
9137 move || {
9138 inherited.cancelled()
9139 || pauses
9140 .as_ref()
9141 .is_some_and(|(a, b)| a.parked() || b.parked())
9142 },
9143 )
9144}
9145
9146fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9147 if error.is::<crate::repo_lock::ResourceBlocked>() {
9148 state.resource_block = Some(crate::run::ResourceBlock {
9149 node: "repository".into(),
9150 kind: if error.to_string().contains("disk pressure") {
9151 "disk pressure"
9152 } else {
9153 "repository coordination"
9154 }
9155 .into(),
9156 detail: error.to_string(),
9157 at: Timestamp::now(),
9158 });
9159 state.event("resource", error.to_string());
9160 }
9161}
9162
9163async fn coordinate_operation(
9164 state: &mut RunState,
9165 cwd: &Path,
9166 wait: &crate::repo_lock::WaitPolicy,
9167) -> Result<Option<crate::repo_lock::SeatLease>> {
9168 let result =
9169 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9170 .await;
9171 if let Err(e) = &result {
9172 record_repository_block(state, e);
9173 }
9174 result
9175}
9176
9177async fn run_one(
9179 job: SeatJob,
9180 sem: Arc<Semaphore>,
9181 ctx: &WaveCtx<'_>,
9182 state: &mut RunState,
9183 attempt: usize,
9184) -> (SeatState, AgentOutcome) {
9185 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9186 .await
9187 .pop()
9188 .expect("one job in, one result out");
9189 (seat, out)
9190}
9191
9192async fn wave(
9198 jobs: Vec<SeatJob>,
9199 sem: Arc<Semaphore>,
9200 ctx: &WaveCtx<'_>,
9201 state: &mut RunState,
9202 attempt: usize,
9203) -> Vec<(usize, SeatState, AgentOutcome)> {
9204 let WaveCtx {
9205 run,
9206 node,
9207 prompts,
9208 cache,
9209 round,
9210 carry_seats: _,
9211 pauses,
9212 } = *ctx;
9213 for job in &jobs {
9214 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9215 }
9216 if let Err(e) = state.save() {
9217 tracing::warn!("could not persist in-progress seats: {e:#}");
9222 }
9223 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9239 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9243 let cache_guard = if let Some(cache_dir) = cache {
9244 if jobs_had_a_writer {
9245 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9246 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9247 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9248 {
9249 Ok(Some(g)) => Some(g),
9250 Ok(None) => {
9253 state.resource_block = Some(crate::run::ResourceBlock {
9254 node: node.to_owned(),
9255 kind: "shared build cache".to_owned(),
9256 detail: "the run is parking".to_owned(),
9257 at: Timestamp::now(),
9258 });
9259 return block_wave(
9260 jobs,
9261 state,
9262 node,
9263 attempt,
9264 "parked while waiting for the shared build cache",
9265 );
9266 }
9267 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9270 Err(e) => {
9271 let why = format!("{e:#}");
9275 state.resource_block = Some(crate::run::ResourceBlock {
9276 node: node.to_owned(),
9277 kind: "shared build cache".to_owned(),
9278 detail: why.clone(),
9279 at: Timestamp::now(),
9280 });
9281 state.event(
9282 "resource",
9283 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9284 );
9285 return block_wave(jobs, state, node, attempt, &why);
9286 }
9287 }
9288 } else {
9289 None
9290 }
9291 } else {
9292 None
9293 };
9294 state.seats_spawning(node, attempt);
9297 if let Err(e) = state.save() {
9298 tracing::warn!("could not persist the start of a wave: {e:#}");
9299 }
9300 let slot_limit = resource_limit
9304 + jobs
9305 .iter()
9306 .map(|j| j.timeout)
9307 .max()
9308 .unwrap_or_default()
9309 .saturating_mul(jobs.len() as u32);
9310 let mut set = tokio::task::JoinSet::new();
9311 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9314 let overlay = prompts.overlay(node);
9315 let repository_wait = repository_wait(state, pauses);
9316 for (i, mut job) in jobs.into_iter().enumerate() {
9317 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9318 if cache.is_some() {
9319 job.prompt.push('\n');
9320 job.prompt
9321 .push_str(&prompt::build_cache_note(node, job.allow_write));
9322 }
9323 let sem = Arc::clone(&sem);
9324 let run = run.to_owned();
9325 let node = node.to_owned();
9326 let attachments = if node == "implement" {
9329 state.attachments.clone()
9330 } else {
9331 Vec::new()
9332 };
9333 let cache = cache
9344 .filter(|_| job.allow_write && cache_guard.is_some())
9345 .map(Path::to_path_buf);
9346 let spawned_tx = spawned_tx.clone();
9347 let repository_wait = repository_wait.clone();
9348 let coordination = state.config.coordination.clone();
9349 let fetched_base = state.fetched_base.clone();
9350 set.spawn(async move {
9351 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9355 Ok(p) => p,
9356 Err(_) => {
9357 return (
9358 i,
9359 job.seat,
9360 AgentOutcome::ResourceBlocked(format!(
9361 "no parallel slot became free within {}s",
9362 slot_limit.as_secs()
9363 )),
9364 false,
9365 );
9366 }
9367 };
9368 spawned_tx.send(job.seat.key.clone()).ok();
9369 let mut seat = job.seat;
9370 let out = repository_wait
9371 .scope(agent::invoke(
9372 &job.spec,
9373 &mut seat,
9374 &Invocation {
9375 scope: crate::agent::InvocationScope::Repository,
9376 coordination: coordination.as_ref(),
9377 fetched_base: fetched_base.as_ref(),
9378 cwd: &job.cwd,
9379 prompt: &job.prompt,
9380 timeout: job.timeout,
9381 allow_write: job.allow_write,
9382 unsandboxed: false,
9383 sessions: job.sessions,
9384 artifacts: &job.artifacts,
9385 stem: &job.stem,
9386 run: &run,
9387 node: &node,
9388 cache_dir: cache.as_deref(),
9389 attachments: &attachments,
9390 writable: &[],
9391 },
9392 ))
9393 .await;
9394 let interrupted = out
9395 .as_ref()
9396 .err()
9397 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9398 let out = match out {
9399 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9400 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9401 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9409 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9410 Ok(o) => AgentOutcome::Failed(format!(
9411 "process exited with no usable output ({})",
9412 crate::notices::exit_status(o.exit_code)
9413 )),
9414 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9415 AgentOutcome::ResourceBlocked(e.to_string())
9416 }
9417 Err(e) => AgentOutcome::Failed(e.to_string()),
9418 };
9419 (i, seat, out, interrupted)
9420 });
9421 }
9422 drop(spawned_tx);
9423 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9424 loop {
9425 let joined = tokio::select! {
9426 Some(key) = spawned_rx.recv() => {
9427 state.seat_spawned(&key);
9429 if let Err(e) = state.save() {
9430 tracing::warn!("could not persist a seat's start: {e:#}");
9431 }
9432 continue;
9433 }
9434 joined = set.join_next() => joined,
9435 };
9436 let Some(joined) = joined else {
9437 break;
9438 };
9439 let (i, seat, out, interrupted) = match joined {
9440 Ok(v) => v,
9441 Err(e) => {
9445 tracing::error!("agent task panicked: {e}");
9446 continue;
9447 }
9448 };
9449 state.seat_finished(&seat.key);
9450 if interrupted
9451 && node == "implement"
9452 && let Some(label) = state
9453 .candidates
9454 .iter()
9455 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9456 .map(|candidate| candidate.label)
9457 {
9458 state.candidate_preservation.insert(
9459 label,
9460 "interrupted implementation awaiting persistence".into(),
9461 );
9462 state.interrupted_implementations.insert(label);
9463 }
9464 if let AgentOutcome::ResourceBlocked(why) = &out
9465 && state.resource_block.is_none()
9466 {
9467 state.resource_block = Some(crate::run::ResourceBlock {
9468 node: node.to_owned(),
9469 kind: if why.contains("disk pressure") {
9470 "disk pressure"
9471 } else if why.contains("repository coordination") {
9472 "repository coordination"
9473 } else {
9474 "parallel slot"
9475 }
9476 .to_owned(),
9477 detail: why.clone(),
9478 at: Timestamp::now(),
9479 });
9480 let activity = if interrupted {
9481 "agent execution interrupted"
9482 } else {
9483 "no agent was started"
9484 };
9485 state.event("resource", format!("{node}: {activity} - {why}"));
9486 }
9487 record_jobs(state, node, round, &seat.key, &out);
9488 if let Err(e) = state.save() {
9489 tracing::warn!("could not persist a seat's completion: {e:#}");
9490 }
9491 if collected.len() <= i {
9492 collected.resize_with(i + 1, || None);
9493 }
9494 collected[i] = Some((i, seat, out));
9495 }
9496 if state
9502 .active
9503 .values()
9504 .any(|a| a.node == node && a.attempt == attempt)
9505 {
9506 state
9507 .active
9508 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9509 if let Err(e) = state.save() {
9510 tracing::warn!("could not persist the end of a wave: {e:#}");
9511 }
9512 }
9513 if let Some(cache_dir) = cache
9520 && jobs_had_a_writer
9521 {
9522 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9523 }
9524 if let Some(guard) = cache_guard {
9525 guard.release();
9526 }
9527 collected.into_iter().flatten().collect()
9528}
9529
9530fn record_jobs(
9541 state: &mut RunState,
9542 node: &str,
9543 round: Option<usize>,
9544 seat: &str,
9545 out: &AgentOutcome,
9546) {
9547 let commands: &[agent::CommandEvidence] = match out {
9548 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9549 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9550 };
9551 let checked_at = Timestamp::now();
9552 for c in commands {
9553 state.jobs.push(JobRecord {
9554 node: node.to_owned(),
9555 round,
9556 seat: seat.to_owned(),
9557 id: c.id.clone(),
9558 description: c.description.clone(),
9559 checked_at,
9560 status: match c.exit_code {
9561 Some(0) => JobStatus::Completed,
9562 Some(_) => JobStatus::Failed,
9563 None => JobStatus::Unknown,
9564 },
9565 exit_code: c.exit_code,
9566 result_summary: c.result_summary.clone(),
9567 source: c.source.clone(),
9568 });
9569 }
9570}
9571
9572fn round_is_clean(
9593 blocking: usize,
9594 e2e_ok: bool,
9595 answered: usize,
9596 expected: usize,
9597 quota_missing: usize,
9598 policy: IncompleteReviewPolicy,
9599) -> bool {
9600 if blocking != 0 || !e2e_ok {
9601 return false;
9602 }
9603 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9604 return true;
9605 }
9606 answered > 0 && expected - answered <= quota_missing
9607}
9608
9609pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9633 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9634 return Some(RunStatus::Gating);
9635 }
9636 let last = reviews.last()?;
9637 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9638 if reviews.len() < max_rounds && !stagnant {
9639 return None;
9640 }
9641 if last.incomplete() && last.blocking == 0 {
9642 return Some(RunStatus::Blocked);
9643 }
9644 if last.e2e_status() == E2eStatus::ResourceBlocked {
9645 return None;
9646 }
9647 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9648 RunStatus::Gating
9649 } else {
9650 RunStatus::Blocked
9651 })
9652}
9653
9654pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9669 if nudged {
9670 (full / 4).max(Duration::from_secs(120)).min(full)
9671 } else {
9672 full
9673 }
9674}
9675
9676#[allow(clippy::too_many_arguments)]
9705async fn ask_json_wave<T>(
9706 jobs: Vec<SeatJob>,
9707 sem: Arc<Semaphore>,
9708 retries: usize,
9709 roster: &[AgentSpec],
9710 ctx: &WaveCtx<'_>,
9711 losses: &mut Vec<QuotaLoss>,
9712 state: &mut RunState,
9713 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9714) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9715where
9716 T: serde::de::DeserializeOwned + Send + 'static,
9717{
9718 ask_wave_with(
9719 jobs,
9720 sem,
9721 retries,
9722 roster,
9723 ctx,
9724 losses,
9725 state,
9726 &|text: &str| {
9727 let v = verdict::extract_json::<T>(text)?;
9728 validate(&v)?;
9729 Ok(v)
9730 },
9731 )
9732 .await
9733}
9734
9735#[allow(clippy::too_many_arguments)]
9744async fn ask_wave_with<T>(
9745 jobs: Vec<SeatJob>,
9746 sem: Arc<Semaphore>,
9747 retries: usize,
9748 roster: &[AgentSpec],
9749 ctx: &WaveCtx<'_>,
9750 losses: &mut Vec<QuotaLoss>,
9751 state: &mut RunState,
9752 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9753) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9754where
9755 T: Send + 'static,
9756{
9757 let n = jobs.len();
9758 let originals: Vec<SeatJob> = jobs;
9759 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9760 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9761 let mut nudges: Vec<usize> = vec![0; n];
9768 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9772 let mut tried: Vec<BTreeSet<String>> = specs
9773 .iter()
9774 .map(|s| BTreeSet::from([s.id.clone()]))
9775 .collect();
9776 let starts: Vec<usize> = specs
9777 .iter()
9778 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9779 .collect();
9780 let carry = ctx.carry_seats && !roster.is_empty();
9781 let carried: Vec<BTreeSet<String>> = originals
9784 .iter()
9785 .map(|j| {
9786 state
9787 .seat_history
9788 .get(&j.seat.key)
9789 .filter(|_| carry)
9790 .map(|h| h.failed.clone())
9791 .unwrap_or_default()
9792 })
9793 .collect();
9794 let mut prev: Vec<Option<FailClass>> = originals
9795 .iter()
9796 .map(|j| {
9797 state
9798 .seat_history
9799 .get(&j.seat.key)
9800 .filter(|_| carry)
9801 .and_then(|h| h.last_fail.clone())
9802 })
9803 .collect();
9804 let next_agent =
9805 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9806 let others: BTreeSet<String> = specs
9807 .iter()
9808 .enumerate()
9809 .filter(|(j, _)| *j != i)
9810 .map(|(_, s)| s.id.clone())
9811 .collect();
9812 pick_successor(
9813 roster,
9814 starts[i],
9815 tried,
9816 carry.then(|| &carried[i]),
9817 &others,
9818 )
9819 .cloned()
9820 };
9821 let mut fresh: Vec<Option<String>> = vec![None; n];
9822 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9823 let mut pending: Vec<usize> = (0..n).collect();
9824
9825 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9830 for round in 0..max_rounds {
9831 if pending.is_empty() {
9832 break;
9833 }
9834 let mut batch = Vec::with_capacity(pending.len());
9835 let mut renudged: Vec<&str> = Vec::new();
9836 for &i in &pending {
9837 let src = &originals[i];
9838 let full: &str = match &src.handover {
9843 Some(h) if specs[i].id != src.spec.id => h,
9844 _ => &src.prompt,
9845 };
9846 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9849 (
9850 full.to_owned(),
9851 src.timeout,
9852 format!("{}-{word}-{}", src.stem, specs[i].id),
9853 )
9854 } else if nudges[i] == 0 {
9855 (full.to_owned(), src.timeout, src.stem.clone())
9856 } else {
9857 renudged.push(src.seat.key.as_str());
9858 let why = done[i]
9859 .as_ref()
9860 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9861 .unwrap_or_else(|| "no parsable answer".to_owned());
9862 let nudge = prompt::nudge(&why);
9863 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9864 let prompt = if nudged {
9865 nudge
9866 } else {
9867 format!("{full}\n\n---\n\n{nudge}")
9868 };
9869 (
9870 prompt,
9871 retry_budget(src.timeout, nudged),
9872 format!("{}-retry{}", src.stem, nudges[i]),
9873 )
9874 };
9875 batch.push(SeatJob {
9876 spec: specs[i].clone(),
9877 seat: seats[i].clone(),
9878 cwd: src.cwd.clone(),
9879 prompt,
9880 timeout,
9881 allow_write: src.allow_write,
9882 sessions: src.sessions,
9883 artifacts: src.artifacts.clone(),
9884 stem,
9885 handover: None,
9886 });
9887 }
9888
9889 if !renudged.is_empty() {
9890 state.event(
9891 ctx.node,
9892 format!("retry {round}: re-asking {}", renudged.join(", ")),
9893 );
9894 }
9895 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9896 let mut still = Vec::new();
9897 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9898 seats[i] = seat;
9899 if let AgentOutcome::ResourceBlocked(why) = &out {
9902 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9903 continue;
9904 }
9905 let class = FailClass::of(&out);
9906 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9909 && nudges[i] < retries
9910 && !roster.is_empty();
9911 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9912 let next = should_hand_over(prev[i].as_ref(), &cur)
9913 .then(|| next_agent(i, &tried[i], &specs))
9914 .flatten();
9915 if carry {
9916 let h = state
9917 .seat_history
9918 .entry(originals[i].seat.key.clone())
9919 .or_default();
9920 h.failed.insert(specs[i].id.clone());
9921 h.last_fail = Some(cur.clone());
9922 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9923 h.last_ok = None;
9924 }
9925 if let Err(e) = state.save() {
9928 tracing::warn!("could not persist a seat's failure history: {e:#}");
9929 }
9930 }
9931 if let Some(next) = next {
9932 record_handover(
9933 state,
9934 ctx.node,
9935 &originals[i].seat.key,
9936 &specs[i].id,
9937 &next.id,
9938 &cur,
9939 &fail_reason(&out),
9940 );
9941 tried[i].insert(next.id.clone());
9942 prev[i] = Some(cur.clone());
9943 seats[i] =
9944 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9945 specs[i] = next;
9946 fresh[i] = Some(cur.stem_word().to_owned());
9947 nudges[i] = 0;
9948 done[i] = Some(Err(anyhow::anyhow!(
9949 "handed over after: {}",
9950 fail_reason(&out)
9951 )));
9952 still.push(i);
9953 continue;
9954 }
9955 }
9956 if carry
9957 && !nudge_first
9958 && let Some(cur) = class.clone()
9959 {
9960 let h = state
9962 .seat_history
9963 .entry(originals[i].seat.key.clone())
9964 .or_default();
9965 h.failed.insert(specs[i].id.clone());
9966 h.last_fail = Some(cur);
9967 }
9968 let parsed = match out {
9969 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9970 AgentOutcome::Quota(o) => {
9971 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9972 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9973 }
9974 AgentOutcome::Dropped(o) => {
9978 let why = o
9979 .dropped
9980 .as_ref()
9981 .map(|d| d.why.as_str())
9982 .unwrap_or("the CLI ended the stream without delivering its answer");
9983 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9984 }
9985 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9986 Err(anyhow::anyhow!(e))
9987 }
9988 };
9989 let quota = class == Some(FailClass::Quota);
9990 let failed = parsed.is_err();
9991 done[i] = Some(parsed);
9992 if carry && !failed {
9993 let h = state
9994 .seat_history
9995 .entry(originals[i].seat.key.clone())
9996 .or_default();
9997 h.failed.remove(&specs[i].id);
9998 h.last_ok = Some(specs[i].id.clone());
9999 h.last_fail = None;
10000 }
10001 let agent_failure = class.is_some()
10009 && !nudge_first
10010 && !roster.is_empty()
10011 && next_agent(i, &tried[i], &specs).is_some();
10012 if failed && !quota && !agent_failure && nudges[i] < retries {
10013 nudges[i] += 1;
10014 still.push(i);
10015 }
10016 }
10017 pending = still;
10018 }
10019
10020 for (i, q) in last_quota.into_iter().enumerate() {
10023 if let Some(reset) = q {
10024 losses.push(QuotaLoss {
10025 seat: originals[i].seat.key.clone(),
10026 node: ctx.node.to_owned(),
10027 at: Timestamp::now(),
10028 reset,
10029 });
10030 }
10031 }
10032
10033 seats
10034 .into_iter()
10035 .zip(done)
10036 .zip(nudges)
10037 .map(|((seat, res), attempts)| {
10038 (
10039 seat,
10040 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10041 attempts,
10042 )
10043 })
10044 .collect()
10045}
10046
10047fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10049 results
10050 .iter()
10051 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10052}
10053
10054fn block_wave(
10058 jobs: Vec<SeatJob>,
10059 state: &mut RunState,
10060 node: &str,
10061 attempt: usize,
10062 why: &str,
10063) -> Vec<(usize, SeatState, AgentOutcome)> {
10064 state.resource_wait = None;
10065 state
10066 .active
10067 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10068 if let Err(e) = state.save() {
10069 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10070 }
10071 jobs.into_iter()
10072 .enumerate()
10073 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10074 .collect()
10075}
10076
10077async fn acquire_cache_lease(
10090 state: &mut RunState,
10091 cache_dir: &Path,
10092 owner: &crate::cache::Owner,
10093 budget: Duration,
10094 context: &str,
10095 cancel: &(dyn Fn() -> bool + Sync),
10096) -> Result<Option<crate::cache::Guard>> {
10097 let home = crate::run::home();
10098 let started = Instant::now();
10099 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10100 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10101 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10102 Err(e) => {
10103 state.event(
10104 "verify",
10105 format!("{context}: could not check the shared build cache: {e:#}"),
10106 );
10107 if let Err(e2) = state.save() {
10108 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10109 }
10110 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10111 }
10112 };
10113 state.event(
10114 "verify",
10115 format!(
10116 "{context}: waiting for the shared build cache at {} ({})",
10117 cache_dir.display(),
10118 busy.describe()
10119 ),
10120 );
10121 state.resource_wait = Some(crate::run::ResourceWait {
10124 node: context.to_owned(),
10125 kind: "shared build cache".to_owned(),
10126 holder: busy.describe(),
10127 since: Timestamp::now(),
10128 limit_secs: budget.as_secs(),
10129 });
10130 if let Err(e) = state.save() {
10131 tracing::warn!("could not persist a cache wait: {e:#}");
10132 }
10133 let remaining = budget.saturating_sub(started.elapsed());
10134 let waited = crate::cache::wait_for_unless(
10135 &home,
10136 cache_dir,
10137 owner,
10138 remaining,
10139 Duration::from_secs(5),
10140 cancel,
10141 )
10142 .await;
10143 state.resource_wait = None;
10144 match waited {
10145 Ok(g) => Ok(g),
10146 Err(e) => {
10147 state.event("verify", format!("{context}: {e:#}"));
10148 if let Err(e2) = state.save() {
10149 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10150 }
10151 Err(e)
10152 }
10153 }
10154}
10155
10156#[derive(Debug)]
10161struct CacheCheckError(String);
10162
10163impl std::fmt::Display for CacheCheckError {
10164 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10165 f.write_str(&self.0)
10166 }
10167}
10168
10169impl std::error::Error for CacheCheckError {}
10170
10171#[allow(clippy::too_many_arguments)]
10192async fn with_cache_lease<'s, F, Fut>(
10193 state: &'s mut RunState,
10194 cache_dir: Option<&Path>,
10195 node: &str,
10196 seat: &str,
10197 worktree: &Path,
10198 head: &str,
10199 budget: Duration,
10200 context: &str,
10201 wait: &crate::repo_lock::WaitPolicy,
10202 body: F,
10203) -> Result<(Vec<CommandOutcome>, bool)>
10204where
10205 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10206 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10207{
10208 let started = Instant::now();
10211 let home = crate::run::home();
10212 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10213 let guard = if let Some(cache_dir) = cache_dir {
10214 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10215 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10216 wait.cancelled()
10217 })
10218 .await
10219 .and_then(|g| {
10220 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10221 }) {
10222 Ok(g) => Some(g),
10223 Err(e) => {
10224 return Ok((
10225 vec![CommandOutcome {
10226 command: "(waiting for the shared build cache)".into(),
10227 code: None,
10228 output_tail: e.to_string(),
10229 duration_ms: started.elapsed().as_millis() as u64,
10230 resource_blocked: true,
10231 }],
10232 false,
10233 ));
10234 }
10235 }
10236 } else {
10237 None
10238 };
10239 let _coordination = coordinate_operation(state, worktree, wait).await?;
10240 if let Some(cache_dir) = cache_dir {
10241 let identity = crate::cache::Identity::new(worktree, head);
10242 if let Err(e) = crate::cache::ensure_fresh(
10243 &home,
10244 cache_dir,
10245 &identity,
10246 &state.config.verify.cargo_manifests(),
10247 ) {
10248 state.event(
10249 "verify",
10250 format!(
10251 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10252 worktree.display(),
10253 short(head)
10254 ),
10255 );
10256 if let Some(guard) = guard {
10257 guard.release();
10258 }
10259 return Ok((
10260 vec![CommandOutcome {
10261 command: "(confirming the shared build cache is fresh)".into(),
10262 code: None,
10263 output_tail: e.to_string(),
10264 duration_ms: started.elapsed().as_millis() as u64,
10265 resource_blocked: true,
10266 }],
10267 false,
10268 ));
10269 }
10270 }
10271 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10272 if !timed_out_pids.is_empty() {
10273 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10274 }
10275 drop(_coordination);
10278 if let Some(guard) = guard {
10279 guard.release();
10280 }
10281 Ok((outcomes, retried))
10282}
10283
10284async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10296 wait_for_pids_with(
10297 pids,
10298 crate::proc::pid_alive,
10299 LEASE_RELEASE_POLL,
10300 LEASE_RELEASE_MAX_WAIT,
10301 )
10302 .await;
10303}
10304
10305async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10311 pids: &[u32],
10312 alive: F,
10313 poll: Duration,
10314 max_wait: Duration,
10315) {
10316 let deadline = Instant::now() + max_wait;
10317 loop {
10318 if pids.iter().all(|&pid| !alive(pid)) {
10319 return;
10320 }
10321 if Instant::now() >= deadline {
10322 return;
10323 }
10324 tokio::time::sleep(poll).await;
10325 }
10326}
10327
10328fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10335 outcomes.iter().any(|o| o.resource_blocked)
10336}
10337
10338pub(crate) enum GateFix {
10340 Retry,
10342 Stop,
10345 Defer,
10348}
10349
10350fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10358 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10359 red.peek().is_some()
10360 && red.all(|o| {
10361 !o.resource_blocked
10362 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10363 && !o.output_tail.trim().is_empty()
10364 })
10365}
10366
10367fn e2e_outcome_label(o: &CommandOutcome) -> String {
10371 if o.ok() {
10372 return "pass".to_owned();
10373 }
10374 let reason = if o.build_failed() {
10375 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10376 } else {
10377 format!("FAIL ({:?})", o.code)
10378 };
10379 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10380}
10381
10382async fn run_e2e_with_retry(
10390 state: &mut RunState,
10391 shell: &[String],
10392 commands: &[String],
10393 worktree: &Path,
10394 timeout: Duration,
10395 context: &str,
10396) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10397 let (mut e2e, mut timed_out_pids) = run_commands(
10398 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10399 )
10400 .await;
10401 for o in &e2e {
10402 state.event(
10403 "verify",
10404 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10405 );
10406 }
10407 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10411 if verify_retried {
10412 state.event(
10413 "verify",
10414 format!(
10415 "{context}: verify could not build/link, not a test result — retrying once \
10416 before concluding"
10417 ),
10418 );
10419 let retried = run_commands(
10420 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10421 )
10422 .await;
10423 e2e = retried.0;
10424 timed_out_pids.extend(retried.1);
10427 for o in &e2e {
10428 state.event(
10429 "verify",
10430 format!(
10431 "{context}: retry `{}` -> {}",
10432 o.command,
10433 e2e_outcome_label(o)
10434 ),
10435 );
10436 }
10437 }
10438 (e2e, verify_retried, timed_out_pids)
10439}
10440
10441#[allow(clippy::too_many_arguments)]
10457async fn run_commands(
10458 state: &mut RunState,
10459 node: &str,
10460 task: &str,
10461 attempt: usize,
10462 shell: &[String],
10463 commands: &[String],
10464 cwd: &Path,
10465 timeout: Duration,
10466) -> (Vec<CommandOutcome>, Vec<u32>) {
10467 if commands.is_empty() {
10468 return (Vec::new(), Vec::new());
10473 }
10474 let mut out = Vec::new();
10475 let mut timed_out_pids = Vec::new();
10476 let total = commands.len();
10477 for (idx, command) in commands.iter().enumerate() {
10478 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10479 if let Err(e) = state.save() {
10480 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10481 }
10482 let started = Instant::now();
10483 let mut cmd = tokio::process::Command::new(&shell[0]);
10484 cmd.quiet();
10485 cmd.args(&shell[1..])
10486 .arg(command)
10487 .current_dir(cwd)
10488 .stdin(std::process::Stdio::null())
10489 .stdout(std::process::Stdio::piped())
10490 .stderr(std::process::Stdio::piped())
10491 .kill_on_drop(true);
10492 let mut outputs = Vec::new();
10493 let protected = (|| -> Result<()> {
10494 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10495 outputs.push(guard);
10496 }
10497 if let Some(cache) = state.config.cache_dir() {
10498 outputs.push(crate::retention::OutputLease::cache(
10499 &crate::run::home(),
10500 &cache,
10501 false,
10502 )?);
10503 }
10504 Ok(())
10505 })();
10506 if let Err(e) = protected {
10507 out.push(CommandOutcome {
10508 command: command.clone(),
10509 code: None,
10510 output_tail: format!("output ownership unavailable: {e:#}"),
10511 duration_ms: 0,
10512 resource_blocked: true,
10513 });
10514 continue;
10515 }
10516 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10517 let (code, body) = match spawned {
10518 Ok(mut running) => {
10519 let pid = running.child().id();
10520 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10521 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10522 let mut paths = vec![cwd.to_owned()];
10523 if let Some(cache) = state.config.cache_dir() {
10524 paths.push(cache);
10525 }
10526 let mut pressure = None;
10527 let status = tokio::select! {
10528 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10529 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10530 pressure = Some(reason); None
10531 }
10532 };
10533 let (code, failure) = match status {
10534 Some(Ok(Ok(status))) => (
10535 status.code(),
10536 status
10537 .code()
10538 .is_none()
10539 .then(|| format!("command terminated: {status}")),
10540 ),
10541 Some(Ok(Err(error))) => {
10542 (None, Some(format!("failed waiting for command: {error}")))
10543 }
10544 Some(Err(_)) => {
10545 if !running.coordinated()
10549 && let Some(pid) = pid
10550 {
10551 timed_out_pids.push(pid);
10552 }
10553 (
10554 None,
10555 Some(format!("timed out after {}s", timeout.as_secs())),
10556 )
10557 }
10558 None => (None, None),
10559 };
10560 running.finish().await;
10561 if let Some(reason) = pressure {
10562 state.resource_block = Some(crate::run::ResourceBlock {
10563 node: node.to_owned(),
10564 kind: "disk pressure".to_owned(),
10565 detail: reason.clone(),
10566 at: Timestamp::now(),
10567 });
10568 out.push(CommandOutcome {
10569 command: command.clone(),
10570 code: None,
10571 output_tail: reason,
10572 duration_ms: started.elapsed().as_millis() as u64,
10573 resource_blocked: true,
10574 });
10575 break;
10576 }
10577 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10578 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10579 if let Some(failure) = failure {
10580 body.push('\n');
10581 body.push_str(&failure);
10582 }
10583 (code, body)
10584 }
10585 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10586 };
10587 out.push(CommandOutcome {
10588 command: command.clone(),
10589 code,
10590 output_tail: tail(&body, OUTPUT_TAIL),
10591 duration_ms: started.elapsed().as_millis() as u64,
10592 resource_blocked: false,
10593 });
10594 }
10595 state.task_finished(task);
10596 if let Err(e) = state.save() {
10597 tracing::warn!("could not persist the end of {task}: {e:#}");
10598 }
10599 (out, timed_out_pids)
10600}
10601
10602fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10614 let repo = repo.display();
10615 match style {
10616 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10617 MergeStyle::Squash => {
10618 let subject = message
10621 .lines()
10622 .next()
10623 .unwrap_or(branch)
10624 .replace(['\\', '"', '$', '`'], "");
10625 format!(
10626 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10627 )
10628 }
10629 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10630 }
10631}
10632
10633const PR_TITLE_MAX: usize = 240;
10644
10645fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10656 let line = queue::first_line(text)?;
10657 let chars: Vec<char> = line.chars().collect();
10658 let (kept, mark) = if chars.len() <= max {
10659 (line.to_owned(), "")
10660 } else if let Some(end) = sentence_end(&chars, max) {
10661 (chars[..end].iter().collect::<String>(), "")
10662 } else {
10663 let room = max.saturating_sub(3);
10664 let cut = if chars[room].is_whitespace() {
10667 room
10668 } else {
10669 chars[..room]
10670 .iter()
10671 .rposition(|c| c.is_whitespace())
10672 .unwrap_or(room)
10673 };
10674 let head: String = chars[..cut].iter().collect();
10675 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10676 (head.to_owned(), "...")
10677 };
10678 if kept.is_empty() {
10679 return None;
10680 }
10681 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10682 return None;
10683 }
10684 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10685 return None;
10686 }
10687 Some(format!("{kept}{mark}"))
10688}
10689
10690fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10694 const MIN: usize = 20;
10695 for i in MIN..max.min(chars.len()) {
10696 if !matches!(chars[i], '.' | '!' | '?') {
10697 continue;
10698 }
10699 let Some(&next) = chars.get(i + 1) else {
10700 continue;
10701 };
10702 if !next.is_whitespace() {
10703 continue;
10704 }
10705 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10706 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10707 continue;
10708 }
10709 let word: String = chars[..i]
10710 .iter()
10711 .rev()
10712 .take_while(|c| !c.is_whitespace())
10713 .collect::<Vec<_>>()
10714 .into_iter()
10715 .rev()
10716 .collect();
10717 let word = word.to_ascii_lowercase();
10718 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10719 continue;
10720 }
10721 let end = chars[..i]
10722 .iter()
10723 .rposition(|c| !c.is_whitespace())
10724 .map_or(i, |p| p + 1);
10725 return Some(end);
10726 }
10727 None
10728}
10729
10730struct PrMessage {
10735 title: String,
10736 body: String,
10737}
10738
10739impl PrMessage {
10740 fn commit_message(&self) -> String {
10744 format!("{}\n\n{}", self.title, self.body)
10745 }
10746}
10747
10748fn title_marker(line: &str) -> Option<&str> {
10750 let line = line.trim();
10751 let head = line.get(..6)?;
10752 head.eq_ignore_ascii_case("title:")
10753 .then(|| line[6..].trim())
10754}
10755
10756fn summary_title(summary: &str) -> Option<String> {
10761 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10762 let raw = title_marker(first)?;
10763 if raw.is_empty() {
10764 return None;
10765 }
10766 let title = queue::title_from(raw, PR_TITLE_MAX);
10767 let lower = title.to_ascii_lowercase();
10768 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10769 return None;
10770 }
10771 Some(title)
10772}
10773
10774const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10776
10777pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10779
10780pub(crate) fn is_review_run(state: &RunState) -> bool {
10784 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10785}
10786
10787fn review_title(state: &RunState) -> Option<String> {
10792 english_subject(
10793 state.reviewed_commits.as_ref()?.first()?,
10794 state.config.graph.github_english(),
10795 )
10796}
10797
10798fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10801 let raw = raw.trim();
10802 if raw.is_empty() {
10803 return None;
10804 }
10805 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10806 return None;
10807 }
10808 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10809 return None;
10810 }
10811 let title = queue::title_from(raw, PR_TITLE_MAX);
10812 let lower = title.to_ascii_lowercase();
10813 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10814 return None;
10815 }
10816 Some(title)
10817}
10818
10819#[derive(Debug, Clone, PartialEq, Eq)]
10822struct BranchFacts {
10823 commits: Vec<(String, String)>,
10825 stat: String,
10827}
10828
10829const STAT_FILE_LINES: usize = 25;
10831const COMMITS_MAX_CHARS: usize = 20_000;
10833
10834async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10837 let commits = git::commit_log(repo, base, branch).await.ok()?;
10838 if commits.is_empty() {
10839 return None;
10840 }
10841 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10842 let lines: Vec<&str> = stat.lines().collect();
10843 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10844 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10845 let more = format!(" ... {omitted} more file(s)");
10846 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10847 kept.push(&more);
10848 kept.push(lines[lines.len() - 1]);
10849 kept.join("\n")
10850 } else {
10851 lines.join("\n")
10852 };
10853 Some(BranchFacts { commits, stat })
10854}
10855
10856fn markdown_safe(text: &str) -> String {
10859 text.replace("</details>", "</details>")
10860 .replace("\x60\x60\x60", "~~~")
10861}
10862
10863fn neutral_title(state: &RunState, winner: char) -> String {
10864 format!(
10865 "chore: land candidate {} of run {}",
10866 winner.to_ascii_uppercase(),
10867 state.id
10868 )
10869}
10870
10871pub(crate) async fn repair_landing_gate(
10874 state: &mut RunState,
10875 winner: &Candidate,
10876 outcomes: &[CommandOutcome],
10877) -> Result<GateFix> {
10878 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10879 return Ok(GateFix::Stop);
10880 }
10881 let mut runner = Runner {
10882 roles: state.config.resolve_roles()?,
10883 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10884 state: state.clone(),
10885 pause: Pause::new(),
10886 interrupt: Pause::new(),
10887 };
10888 let result = runner.gate_fix_round(winner, outcomes).await;
10889 *state = runner.state;
10890 result
10891}
10892
10893pub(crate) async fn check_gate_repair_e2e(
10896 state: &mut RunState,
10897 winner: &Candidate,
10898 wait: &crate::repo_lock::WaitPolicy,
10899) -> Result<GateFix> {
10900 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10901 return Ok(GateFix::Retry);
10902 }
10903 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10904 if state
10905 .gate_repair_e2e
10906 .as_ref()
10907 .is_some_and(|r| r.head == head && r.passed)
10908 {
10909 return Ok(GateFix::Retry);
10910 }
10911 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10912 head: head.clone(),
10913 passed: false,
10914 outcomes: Vec::new(),
10915 });
10916 state.save()?;
10917 let commands = state.config.verify.e2e.clone();
10918 let outcomes = if commands.is_empty() {
10919 Vec::new()
10920 } else {
10921 let shell = state.config.shell();
10922 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10923 let cache_dir = state.config.cache_dir();
10924 let (outcomes, _) = with_cache_lease(
10925 state,
10926 cache_dir.as_deref(),
10927 "e2e",
10928 "e2e",
10929 &winner.worktree,
10930 &head,
10931 timeout,
10932 "gate repair e2e",
10933 wait,
10934 |state, budget| async move {
10935 run_e2e_with_retry(
10936 state,
10937 &shell,
10938 &commands,
10939 &winner.worktree,
10940 budget,
10941 "gate repair e2e",
10942 )
10943 .await
10944 },
10945 )
10946 .await?;
10947 outcomes
10948 };
10949 let deferred = verify_inconclusive(&outcomes);
10950 let passed = outcomes.iter().all(CommandOutcome::ok)
10951 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10952 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10953 head,
10954 passed,
10955 outcomes,
10956 });
10957 state.save()?;
10958 if deferred {
10959 Ok(GateFix::Defer)
10960 } else if !passed {
10961 state.event(
10962 "gate",
10963 "verify.e2e failed after gate repair; nothing published",
10964 );
10965 Ok(GateFix::Stop)
10966 } else {
10967 Ok(GateFix::Retry)
10968 }
10969}
10970
10971pub(crate) async fn run_landing_gate(
10973 state: &mut RunState,
10974 winner: &Candidate,
10975 wait: &crate::repo_lock::WaitPolicy,
10976) -> Result<Vec<CommandOutcome>> {
10977 let shell = state.config.shell();
10978 let gate_commands = state.config.verify.gate.clone();
10979 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10980 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10981 Vec::new()
10982 } else {
10983 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10984 let cache_dir = state.config.cache_dir();
10985 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10986 let (outcomes, _) = with_cache_lease(
10987 state,
10988 cache_dir.as_deref(),
10989 "gate",
10990 "gate",
10991 &winner.worktree,
10992 &head,
10993 timeout,
10994 "final gate",
10995 wait,
10996 |state, budget| {
10997 let shell = shell.clone();
10998 let gate_commands = gate_commands.clone();
10999 let winner = winner.clone();
11000 async move {
11001 Runner::run_pre_gate(state, &winner).await;
11002 let (outcomes, timed_out_pids) = run_commands(
11003 state,
11004 "gate",
11005 "gate",
11006 0,
11007 &shell,
11008 &gate_commands,
11009 &winner.worktree,
11010 budget,
11011 )
11012 .await;
11013 (outcomes, false, timed_out_pids)
11014 }
11015 },
11016 )
11017 .await?;
11018 outcomes
11019 };
11020 if outcomes.is_empty() {
11021 state.event("gate", "no gate commands configured; not tested");
11026 }
11027 for o in &outcomes {
11028 state.event(
11029 "gate",
11030 format!(
11031 "`{}` -> {}",
11032 o.command,
11033 if o.ok() {
11034 "pass".to_owned()
11035 } else {
11036 format!(
11037 "FAIL ({:?})\n{}",
11038 o.code,
11039 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11040 )
11041 }
11042 ),
11043 );
11044 }
11045 Ok(outcomes)
11046}
11047
11048pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11054 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11055 ""
11056 } else {
11057 pr_title
11058 }
11059}
11060
11061pub fn landing_subject_source(state: &RunState) -> String {
11065 if is_review_run(state) {
11066 let winner = state.candidates.first().map_or('A', |c| c.label);
11067 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11068 }
11069 state.instruction.clone()
11070}
11071
11072fn summary_without_title(summary: &str) -> String {
11075 let mut lines = summary.trim().lines().peekable();
11076 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11077 lines.next();
11078 }
11079 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11080}
11081
11082#[cfg(test)]
11094fn pr_message(state: &RunState, winner: char) -> PrMessage {
11095 pr_message_with(state, winner, None)
11096}
11097
11098#[cfg(test)]
11101fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11102 let raw = pr_message_raw(
11103 state,
11104 winner,
11105 facts,
11106 crate::pr_outcome::FinalHead::Unreported,
11107 ChangeNote::NotAsked,
11108 );
11109 let id = crate::scrub::Identity::current();
11110 PrMessage {
11111 title: crate::scrub::scrub(&raw.title, &id),
11112 body: crate::scrub::scrub(&raw.body, &id),
11113 }
11114}
11115
11116#[derive(Debug, Clone, Copy)]
11118enum ChangeNote<'a> {
11119 NotAsked,
11121 Summary(&'a crate::run::ChangeSummary),
11123 Unavailable(&'a str),
11125}
11126
11127fn pr_message_raw(
11128 state: &RunState,
11129 winner: char,
11130 facts: Option<&BranchFacts>,
11131 head: crate::pr_outcome::FinalHead<'_>,
11132 change: ChangeNote<'_>,
11133) -> PrMessage {
11134 if !matches!(change, ChangeNote::NotAsked) {
11135 let (title, mut body) = match change {
11136 ChangeNote::Summary(c) => (
11137 c.title.clone(),
11138 format!("## Summary\n\n{}\n\n", c.body.trim()),
11139 ),
11140 ChangeNote::Unavailable(why) => (
11141 neutral_title(state, winner),
11142 format!(
11143 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11144 ),
11145 ),
11146 ChangeNote::NotAsked => unreachable!(),
11147 };
11148 body.push_str(&public_pr_evidence(state, head));
11149 body.push_str(&format!(
11150 "\n---\nmagi:run/{} magi:candidate-{}\n",
11151 state.id,
11152 winner.to_ascii_lowercase()
11153 ));
11154 return PrMessage { title, body };
11155 }
11156 pr_preview(state, winner, facts, head)
11157}
11158
11159fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11162 let mut body = crate::pr_outcome::review_rounds(state);
11163 body.push_str(&crate::pr_outcome::final_verification(state, head));
11164 let open = state.last_round_findings();
11165 if !open.is_empty() {
11166 body.push_str("## Open review findings\n\n");
11167 for f in open {
11168 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11169 }
11170 body.push('\n');
11171 }
11172 if let Some(c) = state.change_summary.as_ref()
11173 && head == crate::pr_outcome::FinalHead::At(&c.head)
11174 {
11175 body.push_str(&format!(
11176 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11177 c.base, c.head
11178 ));
11179 }
11180 body
11181}
11182
11183fn pr_preview(
11185 state: &RunState,
11186 winner: char,
11187 facts: Option<&BranchFacts>,
11188 head: crate::pr_outcome::FinalHead<'_>,
11189) -> PrMessage {
11190 let summary = state
11191 .candidates
11192 .iter()
11193 .find(|c| c.label == winner)
11194 .map(|c| c.summary.as_str())
11195 .unwrap_or_default();
11196 let review = is_review_run(state);
11199 let title = if review {
11200 facts
11201 .and_then(|f| {
11202 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11203 })
11204 .or_else(|| review_title(state))
11205 .or_else(|| {
11206 state
11207 .candidates
11208 .iter()
11209 .find(|c| c.label == winner)
11210 .filter(|c| !c.branch.starts_with("magi/"))
11211 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11212 })
11213 .unwrap_or_else(|| neutral_title(state, winner))
11214 } else {
11215 summary_title(summary).unwrap_or_else(|| {
11216 english_title(
11217 &state.instruction,
11218 PR_TITLE_MAX,
11219 state.config.graph.github_english(),
11220 )
11221 .unwrap_or_else(|| neutral_title(state, winner))
11222 })
11223 };
11224
11225 let mut body = String::new();
11226 let what = summary_without_title(summary);
11227 if !what.is_empty() {
11228 body.push_str("## Summary\n\n");
11229 body.push_str(&what);
11230 body.push_str("\n\n");
11231 }
11232
11233 body.push_str(&crate::pr_outcome::review_rounds(state));
11235 body.push_str(&crate::pr_outcome::final_verification(state, head));
11236 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11237
11238 let open = state.open_findings();
11239 if !open.is_empty() {
11240 body.push_str("## Open review findings\n\n");
11241 for f in &open {
11242 body.push_str(&format!(
11243 "- `{}` [{:?}] {}\n",
11244 f.id, f.severity, "Context retained in local review records."
11245 ));
11246 }
11247 body.push('\n');
11248 }
11249
11250 if let Some(fix) = fix
11251 && !fix.rejected.is_empty()
11252 {
11253 body.push_str("## Declined by the fixer\n\n");
11254 for r in &fix.rejected {
11255 body.push_str(&format!(
11256 "- `{}`: {}\n",
11257 r.id, "Reason retained in local review records."
11258 ));
11259 }
11260 body.push('\n');
11261 }
11262
11263 if review {
11264 body.push_str("## Commits under review\n\n");
11266 if let Some(facts) = facts {
11267 let mut left = COMMITS_MAX_CHARS;
11268 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11269 let mut entry = format!("- {}\n", markdown_safe(subject));
11270 for l in markdown_safe(text).lines() {
11271 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11272 }
11273 if left == 0 {
11274 body.push_str(&format!(
11275 "- ... {} more commit(s)\n",
11276 facts.commits.len() - i
11277 ));
11278 break;
11279 }
11280 if entry.len() > left {
11281 let mut end = left;
11284 while !entry.is_char_boundary(end) {
11285 end -= 1;
11286 }
11287 entry.truncate(end);
11288 entry.push_str("\n ... (truncated)\n");
11289 left = 0;
11290 } else {
11291 left -= entry.len();
11292 }
11293 body.push_str(&entry);
11294 }
11295 if !facts.stat.trim().is_empty() {
11296 body.push_str(&format!(
11297 "\n## Diff stat\n\n```\n{}\n```\n",
11298 markdown_safe(facts.stat.trim())
11299 ));
11300 }
11301 } else {
11302 match &state.reviewed_commits {
11303 Some(subjects) => {
11304 for s in subjects {
11305 body.push_str(&format!("- {}\n", s.trim()));
11306 }
11307 }
11308 None => {
11309 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11312 body.push_str(rest.trim());
11313 body.push('\n');
11314 }
11315 }
11316 }
11317 } else {
11318 let task = state.instruction.trim();
11319 let task = if task.is_empty() {
11320 "(empty task)"
11321 } else {
11322 task
11323 };
11324 body.push_str(&format!(
11325 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11326 task.replace("</details>", "</details>")
11327 ));
11328 }
11329
11330 body.push_str(&format!(
11331 "\n---\nmagi:run/{} magi:candidate-{}\n",
11332 state.id,
11333 winner.to_ascii_lowercase()
11334 ));
11335
11336 PrMessage { title, body }
11337}
11338
11339fn seeded_instruction(state: &RunState) -> String {
11342 let mut inputs = Vec::new();
11343 if let Some(start) = &state.sources.start_from {
11344 inputs.push(format!(
11345 "Start from `{}` at `{}` (including ancestry).",
11346 start.token, start.sha
11347 ));
11348 }
11349 for pick in &state.sources.pick_commits {
11350 inputs.push(format!(
11351 "Pick `{}` at `{}` (single commit).",
11352 pick.token, pick.sha
11353 ));
11354 }
11355 if inputs.is_empty() {
11356 state.instruction.clone()
11357 } else {
11358 format!(
11359 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11360 state.instruction,
11361 inputs.join("\n")
11362 )
11363 }
11364}
11365
11366async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11370 let base = &state.base_branch;
11371 let mut against = base.clone();
11372 if mode == MergeMode::Pr {
11373 let remote = &state.config.merge.remote;
11377 let tracking = format!("{remote}/{base}");
11378 let fetched = git::fetch(repo, remote, base).await;
11379 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11380 against = tracking;
11381 } else {
11382 return false;
11383 }
11384 }
11385 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11386}
11387
11388fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11391 let mut detail = format!(
11392 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11393 nothing to open a pull request for"
11394 );
11395 detail.push_str(
11396 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11397 );
11398 detail
11399}
11400
11401#[derive(Debug, PartialEq, Eq)]
11404enum PrPlan {
11405 Create,
11406 Adopt { url: String, title: String },
11407 Stop(String),
11408}
11409
11410fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11413 match found {
11414 Ok(land::OpenPr::None) => PrPlan::Create,
11415 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11416 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11417 "several open pull requests exist for this branch, not picking one: {}",
11418 urls.join(" ")
11419 )),
11420 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11421 }
11422}
11423
11424pub(crate) async fn gh_pr_create(
11426 cwd: &Path,
11427 base: &str,
11428 head: &str,
11429 title: &str,
11430 body: &str,
11431) -> Result<String> {
11432 let out = tokio::process::Command::new("gh")
11433 .args([
11434 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11435 ])
11436 .current_dir(cwd)
11437 .quiet()
11438 .stdin(std::process::Stdio::null())
11439 .output()
11440 .await
11441 .context("spawn gh")?;
11442 if out.status.success() {
11443 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11444 } else {
11445 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11446 }
11447}
11448
11449async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11451 match std::fs::symlink_metadata(path) {
11452 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11453 git::worktree_remove_clean(repo, path).await?;
11456 return Ok(());
11457 }
11458 Err(e) => return Err(e.into()),
11459 Ok(_) => {}
11460 }
11461 if !git::is_clean(path).await? {
11462 bail!(
11463 "worktree has uncommitted work; refusing removal: {}",
11464 path.display()
11465 );
11466 }
11467 if !git::worktree_remove_clean(repo, path).await? {
11468 bail!("worktree removal refused: {}", path.display());
11469 }
11470 Ok(())
11471}
11472
11473pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11482 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11483 bail!("refusing fold: {reason}");
11484 }
11485 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11486 bail!("candidate preservation incomplete; refusing cleanup");
11487 }
11488 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11492 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11493 let _driver = crate::driver::maintenance(home, &state.id)?;
11494 let mut output_guards = Vec::new();
11495 for entry in std::fs::read_dir(state.worktree_root())
11496 .into_iter()
11497 .flatten()
11498 .flatten()
11499 {
11500 if entry.path().join(".git").is_file() {
11501 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11502 }
11503 }
11504 let repo = state.repo.clone();
11505 let wait = repository_wait(state, None);
11508 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11509 Ok(lease) => lease,
11510 Err(e) => {
11511 state.save_under(home)?;
11512 return Err(e);
11513 }
11514 };
11515 let _source_outputs = if state.status == RunStatus::Merged
11518 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11519 {
11520 verify_merged_fold_sources(state, drop_winner, home).await?
11521 } else {
11522 Vec::new()
11523 };
11524 let root = state.worktree_root();
11525 let winner = state.tally.as_ref().map(|t| t.winner);
11526 let mut removed = Vec::new();
11527
11528 for i in 0..state.candidates.len() {
11529 let c = state.candidates[i].clone();
11530 let is_winner = Some(c.label) == winner;
11531 if is_winner && !drop_winner {
11532 continue;
11533 }
11534 let existed = c.worktree.exists();
11535 remove_preserved_worktree(&repo, &c.worktree).await?;
11538 if existed {
11539 removed.push(c.worktree.to_string_lossy().into_owned());
11540 }
11541 let handed_over = state.released_branches.contains(&c.branch);
11544 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11545 if !git::branch_delete(&repo, &c.branch).await? {
11546 bail!("candidate branch removal refused");
11547 }
11548 removed.push(c.branch.clone());
11549 }
11550 state.candidates[i].folded = true;
11551 }
11552
11553 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11554 let path = name.path();
11555 let keep = !drop_winner
11556 && winner.is_some_and(|w| {
11557 path.file_name()
11558 .is_some_and(|n| n == format!("cand-{w}").as_str())
11559 });
11560 if keep {
11561 continue;
11562 }
11563 remove_preserved_worktree(&repo, &path).await?;
11564 removed.push(path.to_string_lossy().into_owned());
11565 }
11566
11567 remove_if_empty(&root);
11576
11577 if state.enabled_worktree_config && drop_winner {
11578 git::release_worktree_config(&repo).await.ok();
11582 state.enabled_worktree_config = false;
11583 }
11584 state.save_under(home)?;
11585 Ok(removed)
11586}
11587
11588async fn verify_merged_fold_sources(
11589 state: &RunState,
11590 drop_winner: bool,
11591 home: &Path,
11592) -> Result<Vec<std::fs::File>> {
11593 let winner = state.tally.as_ref().map(|t| t.winner);
11594 let mut heads = BTreeSet::new();
11595 for candidate in &state.candidates {
11596 if Some(candidate.label) == winner && !drop_winner {
11597 continue;
11598 }
11599 anyhow::ensure!(
11600 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11601 "candidate recovery source is outside the inventoried run root: {}",
11602 candidate.worktree.display()
11603 );
11604 match std::fs::symlink_metadata(&candidate.worktree) {
11605 Ok(_) => {}
11606 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11607 Err(error) => return Err(error.into()),
11608 }
11609 if !state.released_branches.contains(&candidate.branch)
11610 && git::branch_exists(&state.repo, &candidate.branch).await?
11611 {
11612 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11613 }
11614 }
11615 let mut cargo_guards = Vec::new();
11616 let entries = match std::fs::read_dir(state.worktree_root()) {
11617 Ok(entries) => Some(entries),
11618 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11619 Err(error) => return Err(error.into()),
11620 };
11621 for entry in entries.into_iter().flatten() {
11622 let path = entry?.path();
11623 let metadata = std::fs::symlink_metadata(&path)?;
11624 anyhow::ensure!(
11625 metadata.is_dir() && !metadata.file_type().is_symlink(),
11626 "non-directory or symlink recovery source retained: {}",
11627 path.display()
11628 );
11629 if !drop_winner
11630 && winner.is_some_and(|w| {
11631 path.file_name()
11632 .is_some_and(|name| name == format!("cand-{w}").as_str())
11633 })
11634 {
11635 continue;
11636 }
11637 anyhow::ensure!(
11638 path.join(".git").is_file(),
11639 "unregistered recovery source retained: {}",
11640 path.display()
11641 );
11642 anyhow::ensure!(
11643 git::git(
11644 &path,
11645 &[
11646 "status",
11647 "--porcelain",
11648 "--untracked-files=all",
11649 "--ignored",
11650 "--",
11651 ".",
11652 ":(exclude)target"
11653 ]
11654 )
11655 .await?
11656 .is_empty(),
11657 "source changes or ignored recovery files retained: {}",
11658 path.display()
11659 );
11660 match std::fs::symlink_metadata(path.join("target")) {
11661 Ok(_) => {
11662 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11663 }
11664 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11665 Err(error) => return Err(error.into()),
11666 }
11667 for checkpoint in [
11668 "rebase-merge",
11669 "rebase-apply",
11670 "sequencer",
11671 "CHERRY_PICK_HEAD",
11672 "MERGE_HEAD",
11673 "REBASE_HEAD",
11674 "REVERT_HEAD",
11675 "BISECT_LOG",
11676 "AUTO_MERGE",
11677 "MERGE_MSG",
11678 ] {
11679 let location = git::git(
11680 &path,
11681 &[
11682 "rev-parse",
11683 "--path-format=absolute",
11684 "--git-path",
11685 checkpoint,
11686 ],
11687 )
11688 .await?;
11689 anyhow::ensure!(
11690 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11691 if e.kind() == std::io::ErrorKind::NotFound),
11692 "Git recovery checkpoint retained: {checkpoint}"
11693 );
11694 }
11695 heads.insert(git::rev_parse(&path, "HEAD").await?);
11696 }
11697 crate::fold_evidence::preserve(state, home, &heads).await?;
11698
11699 Ok(cargo_guards)
11700}
11701
11702fn remove_if_empty(dir: &Path) {
11713 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11714 std::fs::remove_dir(dir).ok();
11715 }
11716}
11717
11718pub fn worst_open(state: &RunState) -> Option<Severity> {
11720 state
11721 .reviews
11722 .last()?
11723 .reviews
11724 .iter()
11725 .flat_map(|r| r.findings.iter())
11726 .map(|f| f.severity)
11727 .max()
11728}
11729
11730async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11734 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11735 git::acquire_worktree_config(repo).await?;
11736 state.enabled_worktree_config = true;
11737 }
11738 Ok(())
11739}
11740
11741#[cfg(test)]
11742mod tests {
11743 #[tokio::test]
11744 async fn launcher_base_fetch_requires_remote_success() {
11745 let tmp = tempfile::tempdir().unwrap();
11746 let repo = tmp.path().join("repo");
11747 let remote = tmp.path().join("remote");
11748 std::fs::create_dir(&repo).unwrap();
11749 std::fs::create_dir(&remote).unwrap();
11750 git::git(&remote, &["init", "--bare", "-b", "main"])
11751 .await
11752 .unwrap();
11753 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11754 git::git(
11755 &repo,
11756 &[
11757 "-c",
11758 "user.name=Fixture",
11759 "-c",
11760 "user.email=fixture@example.invalid",
11761 "commit",
11762 "--allow-empty",
11763 "-m",
11764 "fixture",
11765 ],
11766 )
11767 .await
11768 .unwrap();
11769 git::git(
11770 &repo,
11771 &["remote", "add", "origin", remote.to_str().unwrap()],
11772 )
11773 .await
11774 .unwrap();
11775 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11776 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11777 assert_eq!(
11778 resolve_base(&repo, "main", "origin", None)
11779 .await
11780 .unwrap()
11781 .commit,
11782 head
11783 );
11784 #[cfg(unix)]
11785 {
11786 let policy: crate::config::Coordination = toml::from_str(
11787 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11788 ).unwrap();
11789 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11790 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11791 let held = crate::repo_lock::SeatLease::operation_with(
11792 &repo,
11793 Some(&policy),
11794 &crate::repo_lock::WaitPolicy::default(),
11795 )
11796 .await
11797 .unwrap();
11798 assert!(held.is_some());
11799 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11800 tokio::pin!(fetch);
11801 assert!(
11802 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11803 .await
11804 .is_err()
11805 );
11806 assert!(
11807 !repo.join(".git/FETCH_HEAD").exists(),
11808 "no fetch before admission"
11809 );
11810 drop(held);
11811 let fetched = fetch.await.unwrap();
11812 assert_eq!(fetched.commit, head);
11813 assert_eq!(fetched.remote, "origin");
11814 assert_eq!(fetched.branch, "main");
11815 }
11816 git::git(
11818 &repo,
11819 &[
11820 "remote",
11821 "set-url",
11822 "origin",
11823 tmp.path().join("missing").to_str().unwrap(),
11824 ],
11825 )
11826 .await
11827 .unwrap();
11828 let error = resolve_base(&repo, "main", "origin", None)
11829 .await
11830 .unwrap_err()
11831 .to_string();
11832 assert!(error.contains("cannot read origin/main"));
11833 assert!(error.contains("refusing to branch off"));
11834 }
11835
11836 #[test]
11837 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11838 let leaked = vec!["chore(deps): update a crate".to_owned()];
11839 let own = "fix(daemon): apply a chosen action";
11840 assert!(should_retitle("", own, &leaked));
11841 assert!(should_retitle(
11842 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11843 own,
11844 &leaked
11845 ));
11846 assert!(should_retitle(
11847 "chore: land candidate A of run 1",
11848 own,
11849 &leaked
11850 ));
11851 assert!(should_retitle(
11852 "magi: candidate A (uncommitted work)",
11853 own,
11854 &leaked
11855 ));
11856 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11857 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11858 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11859 }
11860
11861 #[test]
11862 fn pr_merge_plan_creates_adopts_or_stops() {
11863 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11864 assert_eq!(
11865 pr_merge_plan(Ok(land::OpenPr::One {
11866 url: "u".into(),
11867 title: "t".into()
11868 })),
11869 PrPlan::Adopt {
11870 url: "u".into(),
11871 title: "t".into()
11872 }
11873 );
11874 let PrPlan::Stop(many) =
11875 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11876 else {
11877 panic!("many must stop");
11878 };
11879 assert!(many.contains('a') && many.contains('b'));
11880 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11881 panic!("a failed lookup must stop");
11882 };
11883 assert!(err.contains("bad token"));
11884 }
11885
11886 use super::*;
11887 use crate::run::GateStatus;
11888 use std::collections::BTreeMap;
11889 use std::time::Duration;
11890
11891 fn conductor() -> AgentSpec {
11892 AgentSpec {
11893 id: "conductor".to_owned(),
11894 kind: crate::config::AgentKind::Command,
11895 model: None,
11896 command: vec!["true".to_owned()],
11897 extra_args: Vec::new(),
11898 env: BTreeMap::new(),
11899 prompt_delivery: None,
11900 max_turn_secs: None,
11901 }
11902 }
11903
11904 fn spec(id: &str) -> AgentSpec {
11905 AgentSpec {
11906 id: id.to_owned(),
11907 kind: crate::config::AgentKind::Command,
11908 model: None,
11909 command: vec!["true".to_owned()],
11910 extra_args: Vec::new(),
11911 env: BTreeMap::new(),
11912 prompt_delivery: None,
11913 max_turn_secs: None,
11914 }
11915 }
11916
11917 fn ids(xs: &[&str]) -> BTreeSet<String> {
11918 xs.iter().map(|s| (*s).to_owned()).collect()
11919 }
11920
11921 #[test]
11922 fn pick_successor_skips_an_agent_another_seat_holds() {
11923 let roster = [spec("a"), spec("b"), spec("c")];
11925 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11926 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11927 }
11928
11929 #[test]
11930 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11931 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11933 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11934 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11935 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11936 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11937 }
11938
11939 #[test]
11940 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11941 let roster = [spec("a"), spec("b")];
11942 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11943 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11944 let carried = ids(&["b"]);
11945 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11946 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11947 }
11948
11949 #[test]
11950 fn pick_successor_returns_none_without_any_untried_successor() {
11951 let roster = [spec("a"), spec("b")];
11952 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11953 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11954 let carried = ids(&["a"]);
11955 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11956 }
11957
11958 #[test]
11959 fn pick_successor_rescue_avoids_another_seats_occupant() {
11960 let roster = [spec("a"), spec("b"), spec("c")];
11962 let carried = ids(&["b", "c"]);
11963 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11964 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11965 }
11966
11967 #[test]
11968 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11969 let roster = [spec("a"), spec("b"), spec("c")];
11970 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11971 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11972 }
11973
11974 #[test]
11975 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11976 let roster = [spec("a"), spec("b"), spec("c")];
11977 let failed = ids(&["a", "b", "c"]);
11978 let mut tried = ids(&["b"]);
11980 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11981 assert_eq!(first.id, "a");
11982 tried.insert(first.id.clone());
11983 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11984 assert_eq!(second.id, "c");
11985 tried.insert(second.id.clone());
11986 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11987 }
11988
11989 #[test]
11990 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11991 let roster = [spec("a"), spec("b")];
11992 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11993 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11994 }
11995
11996 #[test]
11997 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11998 let roster = [spec("a"), spec("b"), spec("c")];
11999 let h = SeatHistory {
12000 failed: ids(&["a"]),
12001 last_ok: Some("b".to_owned()),
12002 last_fail: None,
12003 };
12004 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12005 let gone = SeatHistory {
12007 last_ok: Some("zzz".to_owned()),
12008 ..h.clone()
12009 };
12010 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12011 let failed = SeatHistory {
12012 failed: ids(&["a", "b"]),
12013 last_ok: Some("b".to_owned()),
12014 last_fail: None,
12015 };
12016 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12017 }
12018
12019 #[test]
12020 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12021 let first = SeatState::new("review-1", "alpha", 7);
12022 let next = handover_seat("review-1", "gamma", 7);
12023 assert_ne!(first.claude_session, next.claude_session);
12024 }
12025
12026 #[test]
12027 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12028 let mut state = state_with_summary("x", "y");
12029 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12030 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12031 assert_ne!(a.claude_session, b.claude_session);
12032 }
12033
12034 #[test]
12035 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12036 let roster = [spec("a"), spec("b")];
12037 let h = SeatHistory {
12038 failed: ids(&["a", "b"]),
12039 ..SeatHistory::default()
12040 };
12041 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12042 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12043 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12044 }
12045
12046 #[test]
12053 fn failure_signature_ignores_numbers_and_paths() {
12054 assert_eq!(
12055 failure_signature("exited with Some(2) and no usable output"),
12056 failure_signature("exited with Some(137) and no usable output")
12057 );
12058 assert_eq!(
12059 failure_signature("process exited with no usable output (exit code 2)"),
12060 failure_signature("process exited with no usable output (exit code 137)")
12061 );
12062 for (old, new) in [
12063 (
12064 "exited with Some(2) and no usable output",
12065 "process exited with no usable output (exit code 2)",
12066 ),
12067 (
12068 "exited with None and no usable output",
12069 "process exited with no usable output (exit code unavailable)",
12070 ),
12071 ] {
12072 let previous = FailClass::Other(failure_signature(old));
12073 let current = FailClass::Other(failure_signature(new));
12074 assert_eq!(previous, current, "resumption retains the failure class");
12075 assert!(!should_hand_over(Some(&previous), ¤t));
12076 }
12077 assert_eq!(
12078 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12079 failure_signature("cannot open /var/x.txt: denied")
12080 );
12081 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12082 }
12083
12084 #[test]
12085 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12086 let other = FailClass::Other("x".into());
12087 assert!(should_hand_over(None, &FailClass::Quota));
12088 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12089 assert!(should_hand_over(
12090 Some(&FailClass::Timeout),
12091 &FailClass::Timeout
12092 ));
12093 assert!(should_hand_over(None, &other));
12094 assert!(!should_hand_over(Some(&other), &other));
12095 assert!(should_hand_over(
12096 Some(&other),
12097 &FailClass::Other("y".into())
12098 ));
12099 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12101 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12102 }
12103
12104 #[test]
12105 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12106 let a = SeatState::new("judge-1", "alpha", 7);
12107 let b = handover_seat("judge-1", "beta", 7);
12108 assert_ne!(a.claude_session, b.claude_session);
12109 assert_eq!(b.turns, 0);
12110 }
12111
12112 #[test]
12113 fn a_timeout_is_classified_apart_from_other_failures() {
12114 assert_eq!(
12115 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12116 Some(FailClass::Timeout)
12117 );
12118 assert!(matches!(
12119 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12120 Some(FailClass::Other(_))
12121 ));
12122 }
12123
12124 #[test]
12125 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12126 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12127 let tried = BTreeSet::from(["beta".to_owned()]);
12128 let next = next_untried_in_roster(&roster, 1, &tried);
12131 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12132 }
12133
12134 #[test]
12135 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12136 let roster = vec![spec("alpha"), spec("beta")];
12137 let tried = BTreeSet::from(["beta".to_owned()]);
12138 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12142 }
12143
12144 #[test]
12145 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12146 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12147 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12148 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12152 }
12153
12154 #[test]
12155 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12156 let roster = vec![spec("a"), spec("a"), spec("b")];
12157 let tried = BTreeSet::from(["a".to_owned()]);
12158 let next = next_untried_in_roster(&roster, 0, &tried);
12159 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12160 }
12161
12162 #[test]
12163 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12164 let roster = vec![spec("a"), spec("b")];
12165 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12166 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12167 }
12168
12169 #[test]
12170 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12171 let dir = tempfile::tempdir().unwrap();
12172 let bay = dir.path().join("ffff");
12173
12174 remove_if_empty(&bay);
12176 assert!(!bay.exists());
12177
12178 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12181 remove_if_empty(&bay);
12182 assert!(bay.exists(), "non-empty directory must survive");
12183
12184 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12186 remove_if_empty(&bay);
12187 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12188 }
12189
12190 #[test]
12199 fn a_full_panel_that_found_nothing_is_clean() {
12200 assert!(round_is_clean(
12201 0,
12202 true,
12203 2,
12204 2,
12205 0,
12206 IncompleteReviewPolicy::Block
12207 ));
12208 }
12209
12210 #[test]
12211 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12212 assert!(!round_is_clean(
12213 0,
12214 true,
12215 1,
12216 2,
12217 0,
12218 IncompleteReviewPolicy::Block
12219 ));
12220 }
12221
12222 #[test]
12223 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12224 assert!(!round_is_clean(
12225 1,
12226 true,
12227 1,
12228 2,
12229 0,
12230 IncompleteReviewPolicy::Warn
12231 ));
12232 }
12233
12234 #[test]
12235 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12236 assert!(round_is_clean(
12237 0,
12238 true,
12239 1,
12240 2,
12241 0,
12242 IncompleteReviewPolicy::Warn
12243 ));
12244 }
12245
12246 #[test]
12247 fn a_full_panel_with_an_open_finding_is_not_clean() {
12248 assert!(!round_is_clean(
12249 1,
12250 true,
12251 2,
12252 2,
12253 0,
12254 IncompleteReviewPolicy::Block
12255 ));
12256 }
12257
12258 #[test]
12259 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12260 assert!(!round_is_clean(
12261 0,
12262 false,
12263 2,
12264 2,
12265 0,
12266 IncompleteReviewPolicy::Block
12267 ));
12268 }
12269
12270 #[test]
12277 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12278 assert!(round_is_clean(
12281 0,
12282 true,
12283 1,
12284 2,
12285 1,
12286 IncompleteReviewPolicy::Block
12287 ));
12288 }
12289
12290 #[test]
12291 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12292 assert!(!round_is_clean(
12295 0,
12296 true,
12297 1,
12298 2,
12299 0,
12300 IncompleteReviewPolicy::Block
12301 ));
12302 }
12303
12304 #[test]
12305 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12306 assert!(!round_is_clean(
12307 1,
12308 true,
12309 1,
12310 2,
12311 1,
12312 IncompleteReviewPolicy::Block
12313 ));
12314 assert!(!round_is_clean(
12315 0,
12316 false,
12317 1,
12318 2,
12319 1,
12320 IncompleteReviewPolicy::Block
12321 ));
12322 }
12323
12324 #[test]
12325 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12326 assert!(!round_is_clean(
12330 0,
12331 true,
12332 0,
12333 2,
12334 2,
12335 IncompleteReviewPolicy::Block
12336 ));
12337 }
12338
12339 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12340 CommandOutcome {
12341 command: "test".to_owned(),
12342 code,
12343 output_tail: String::new(),
12344 duration_ms: 0,
12345 resource_blocked,
12346 }
12347 }
12348
12349 #[test]
12350 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12351 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12352 assert!(
12353 !verify_inconclusive(&[outcome(Some(1), false)]),
12354 "an ordinary failure is still evidence about the patch"
12355 );
12356 assert!(verify_inconclusive(&[outcome(None, true)]));
12357 assert!(
12358 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12359 "one inconclusive outcome taints the whole batch"
12360 );
12361 assert!(!verify_inconclusive(&[]));
12362 }
12363
12364 #[tokio::test]
12365 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12366 let calls = std::sync::atomic::AtomicUsize::new(0);
12370 let started = Instant::now();
12371 wait_for_pids_with(
12372 &[123],
12373 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12374 Duration::from_millis(5),
12375 Duration::from_secs(5),
12376 )
12377 .await;
12378 assert!(
12379 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12380 "must keep checking rather than deciding on the first answer"
12381 );
12382 assert!(
12383 started.elapsed() < Duration::from_secs(1),
12384 "must return the moment it is confirmed dead, not wait out the ceiling"
12385 );
12386 }
12387
12388 #[tokio::test]
12389 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12390 let started = Instant::now();
12391 wait_for_pids_with(
12392 &[123],
12393 |_| true, Duration::from_millis(5),
12395 Duration::from_millis(30),
12396 )
12397 .await;
12398 let elapsed = started.elapsed();
12399 assert!(
12400 elapsed >= Duration::from_millis(30),
12401 "must not give up before its own ceiling: {elapsed:?}"
12402 );
12403 assert!(
12404 elapsed < Duration::from_secs(1),
12405 "must not wait past its own ceiling either: {elapsed:?}"
12406 );
12407 }
12408
12409 #[tokio::test]
12410 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12411 let started = Instant::now();
12412 wait_for_pids_with(
12413 &[],
12414 |_| true,
12415 Duration::from_secs(5),
12416 Duration::from_secs(5),
12417 )
12418 .await;
12419 assert!(
12420 started.elapsed() < Duration::from_millis(200),
12421 "an empty pid list has nothing to confirm"
12422 );
12423 }
12424
12425 fn review_round(
12431 clean: bool,
12432 blocking: usize,
12433 answered: usize,
12434 expected: usize,
12435 progressed: bool,
12436 e2e_ok: bool,
12437 ) -> ReviewRound {
12438 ReviewRound {
12439 round: 1,
12440 head: "h".to_owned(),
12441 verified_head: None,
12442 verified_at: None,
12443 reviews: Vec::new(),
12444 e2e: vec![CommandOutcome {
12445 command: "test".to_owned(),
12446 code: Some(if e2e_ok { 0 } else { 1 }),
12447 output_tail: String::new(),
12448 duration_ms: 0,
12449 resource_blocked: false,
12450 }],
12451 verify_retried: false,
12452 e2e_deferred: false,
12453 e2e_defer_reason: None,
12454 fix: None,
12455 blocking,
12456 answered,
12457 expected,
12458 clean,
12459 progressed,
12460 vote_split: false,
12461 reconsideration: Vec::new(),
12462 verdict: None,
12463 }
12464 }
12465
12466 #[test]
12467 fn review_conclusion_is_none_when_nothing_has_run() {
12468 assert_eq!(review_conclusion(&[], 3), None);
12469 }
12470
12471 #[test]
12472 fn review_conclusion_is_none_while_rounds_remain() {
12473 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12474 assert_eq!(review_conclusion(&rounds, 3), None);
12475 }
12476
12477 #[test]
12478 fn review_conclusion_is_gating_once_a_round_is_clean() {
12479 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12480 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12481 }
12482
12483 #[test]
12484 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12485 let rounds = vec![
12486 review_round(false, 1, 2, 2, true, true),
12487 review_round(false, 1, 2, 2, true, true),
12488 ];
12489 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12490 }
12491
12492 #[test]
12493 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12494 let rounds = vec![
12495 review_round(false, 1, 2, 2, true, true),
12496 review_round(false, 1, 2, 2, true, false),
12497 ];
12498 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12499 }
12500
12501 #[test]
12502 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12503 let mut blocked = review_round(false, 1, 2, 2, true, false);
12510 blocked.e2e[0].resource_blocked = true;
12511 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12512 assert_eq!(review_conclusion(&rounds, 2), None);
12513 }
12514
12515 #[test]
12516 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12517 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12519 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12520 }
12521
12522 #[test]
12523 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12524 let rounds = vec![
12525 review_round(false, 1, 2, 2, false, true),
12526 review_round(false, 1, 2, 2, false, true),
12527 ];
12528 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12529 }
12530
12531 fn secs(n: u64) -> Duration {
12532 Duration::from_secs(n)
12533 }
12534
12535 #[tokio::test]
12538 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12539 ask_test_home();
12540 let tmp = tempfile::tempdir().unwrap();
12541 let repo = tmp.path().join("repo");
12542 std::fs::create_dir_all(&repo).unwrap();
12543 init_repo(&repo);
12544 let g = |args: &[&str]| {
12545 let out = std::process::Command::new("git")
12546 .args(args)
12547 .current_dir(&repo)
12548 .quiet()
12549 .output()
12550 .unwrap();
12551 assert!(
12552 out.status.success(),
12553 "{}",
12554 String::from_utf8_lossy(&out.stderr)
12555 );
12556 };
12557 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12558 g(&["checkout", "-b", "historical"]);
12559 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12560 g(&["add", "."]);
12561 g(&["commit", "-m", "old alternative"]);
12562 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12563 g(&["checkout", "-b", "magi/task-selected", &base]);
12564 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12565 g(&["add", "."]);
12566 g(&["commit", "-m", "measurement infrastructure"]);
12567 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12568 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12569 g(&["add", "."]);
12570 g(&["commit", "-m", "dependent implementation"]);
12571 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12572 g(&["checkout", "main"]);
12573 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12574 g(&["add", "."]);
12575 g(&["commit", "-m", "unrelated base change"]);
12576 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12577 let origin = tmp.path().join("origin.git");
12578 g(&[
12579 "clone",
12580 "--bare",
12581 repo.to_str().unwrap(),
12582 origin.to_str().unwrap(),
12583 ]);
12584 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12585 g(&["fetch", "origin"]);
12586 let selection = crate::pr_recovery::Selection {
12587 url: "https://example.test/project/pull/1".into(),
12588 branch: "magi/task-selected".into(),
12589 head: head.clone(),
12590 source_head: None,
12591 source_run: None,
12592 source_proof: None,
12593 disposition: Some(crate::pr_recovery::Disposition::Review),
12594 instruction: "preserve measured behavior".into(),
12595 approved_at: jiff::Timestamp::now(),
12596 };
12597 let mut template = runner_at(RunStatus::Reviewing);
12598 template.state.repo = repo.clone();
12599 template.state.base_commit = current.clone();
12600 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12601 template.state.origin = Some(crate::run::Origin::queue("selected"));
12602 template.state.pr_head = Some(selection.branch.clone());
12603 template.state.pr_recovery = Some(selection.clone());
12604 let mut runner =
12605 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12606 .await
12607 .unwrap();
12608 let wt = runner.state.winner().unwrap().worktree.clone();
12609 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12610 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12611 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12612 assert!(runner.state.seeds.is_empty());
12613 assert_eq!(runner.state.instruction, "preserve measured behavior");
12614 runner.sync_to_base().await.unwrap();
12615 assert_eq!(
12616 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12617 "prerequisite"
12618 );
12619 assert_eq!(
12620 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12621 "measured implementation"
12622 );
12623 assert_eq!(
12624 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12625 "current main change"
12626 );
12627 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12628 assert_eq!(
12629 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12630 head
12631 );
12632
12633 let mut partial = runner_at(RunStatus::Reviewing);
12635 partial.state.repo = repo.clone();
12636 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12637 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12638 run: Some("exit 1".into()),
12639 ..Default::default()
12640 }];
12641 let selection = runner.state.pr_recovery.clone().unwrap();
12642 partial.state.pr_recovery = Some(selection.clone());
12643 partial.state.pr_head = Some(selection.branch.clone());
12644 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12645 let id = partial.state.id.clone();
12646 let error = match Runner::open_selected(
12647 &repo,
12648 partial.state,
12649 partial.roles,
12650 1,
12651 runner.state.base_commit.clone(),
12652 selection,
12653 )
12654 .await
12655 {
12656 Ok(_) => panic!("setup must fail"),
12657 Err(e) => e,
12658 };
12659 assert_eq!(
12660 error
12661 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12662 .unwrap()
12663 .run,
12664 id
12665 );
12666 let failed = RunState::load(&id).unwrap();
12667 assert_eq!(failed.status, RunStatus::Blocked);
12668 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12669 assert!(Runner::resume(&id).is_err());
12670 runner.state.status = RunStatus::Ready;
12671 runner.state.driver_exited = true;
12672 runner.state.save().unwrap();
12673 let mut retry = runner_at(RunStatus::Reviewing);
12674 retry.state.repo = repo.clone();
12675 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12676 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12677 reselected.approved_at = jiff::Timestamp::now();
12678 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12679 retry.state.pr_head = Some(reselected.branch.clone());
12680 retry.state.pr_recovery = Some(reselected.clone());
12681 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12682 .await
12683 .unwrap();
12684 let restarted = Runner::open_selected(
12685 &repo,
12686 retry.state,
12687 retry.roles,
12688 1,
12689 runner.state.base_commit.clone(),
12690 reselected,
12691 )
12692 .await
12693 .unwrap();
12694 assert_eq!(
12695 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12696 .await
12697 .unwrap(),
12698 head
12699 );
12700 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12701 assert_eq!(
12702 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12703 head
12704 );
12705 }
12706
12707 #[tokio::test]
12708 #[cfg(unix)]
12709 async fn fallback_and_interrupted_restoration_require_coordination() {
12710 use std::os::fd::AsRawFd as _;
12711 ask_test_home();
12712 let tmp = tempfile::tempdir().unwrap();
12713 let mut runner = registered_runner(tmp.path()).await;
12714 runner.state.config.graph.timeout_resource_wait = 1;
12715 let winner = runner.state.candidates[0].clone();
12716 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12717 let leftover = winner.worktree.join("leftover");
12718 std::fs::write(&leftover, "untrusted edits").unwrap();
12719 let next = tmp.path().join("next-fixer");
12720 let mut failed = spec("first");
12721 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12722 let mut successor = spec("second");
12723 successor.command = vec![
12724 "sh".into(),
12725 "-c".into(),
12726 format!("touch '{}'; echo done", next.display()),
12727 ];
12728 let attempts = vec![
12729 (failed.clone(), "first".into()),
12730 (successor.clone(), "second".into()),
12731 ];
12732 let lock_path = runner.state.repo.join(".git/updater.lock");
12733 let external = std::fs::File::create(&lock_path).unwrap();
12734 assert_eq!(
12735 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12736 0
12737 );
12738 let artifacts = tmp.path().join("artifacts");
12739 let build = |spec, seat| SeatJob {
12740 spec,
12741 seat,
12742 cwd: winner.worktree.clone(),
12743 prompt: "fixture".into(),
12744 timeout: Duration::from_secs(10),
12745 allow_write: true,
12746 sessions: false,
12747 artifacts: artifacts.clone(),
12748 stem: "fix".into(),
12749 handover: None,
12750 };
12751 let (_, _, out) = runner
12752 .ask_fixer_among(
12753 attempts.clone(),
12754 "graft",
12755 None,
12756 Some((&winner.worktree, &head)),
12757 build,
12758 )
12759 .await;
12760 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12761 assert!(leftover.exists(), "contention must prevent clean");
12762 assert!(!next.exists());
12763 assert!(runner.state.handovers.is_empty());
12764 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12766 from_run: "earlier".into(),
12767 generation: 1,
12768 mode: crate::recovery::Mode::VerifyRepair,
12769 reasons: Vec::new(),
12770 reviewed_head: None,
12771 rounds: Vec::new(),
12772 open_findings: Vec::new(),
12773 evidence: None,
12774 criteria_inherited: true,
12775 repair: crate::recovery::Repair::Started {
12776 before: head.clone(),
12777 },
12778 });
12779 assert!(
12780 runner
12781 .recovery_repair(&winner)
12782 .await
12783 .unwrap_err()
12784 .is::<crate::repo_lock::ResourceBlocked>()
12785 );
12786 assert!(matches!(
12787 runner.state.recovery.as_ref().unwrap().repair,
12788 crate::recovery::Repair::Started { .. }
12789 ));
12790 assert!(leftover.exists());
12791 drop(external);
12792 runner.recovery_repair(&winner).await.unwrap();
12793 assert!(!leftover.exists());
12794 assert!(matches!(
12795 runner.state.recovery.as_ref().unwrap().repair,
12796 crate::recovery::Repair::Done { .. }
12797 ));
12798 std::fs::remove_file(&lock_path).unwrap();
12800 std::fs::create_dir(&lock_path).unwrap();
12801 std::fs::write(&leftover, "untrusted edits").unwrap();
12802 let (_, _, out) = runner
12803 .ask_fixer_among(
12804 attempts,
12805 "graft",
12806 None,
12807 Some((&winner.worktree, &head)),
12808 build,
12809 )
12810 .await;
12811 assert!(matches!(out, AgentOutcome::Failed(_)));
12812 assert!(leftover.exists());
12813 assert!(!next.exists());
12814 assert_eq!(
12815 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12816 head
12817 );
12818 }
12819
12820 #[cfg(unix)]
12821 async fn registered_runner(root: &Path) -> Runner {
12822 ask_test_home();
12823 let repo = root.join("repo");
12824 std::fs::create_dir_all(&repo).unwrap();
12825 init_repo(&repo);
12826 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12827 let mut runner = runner_at(RunStatus::Prep);
12828 runner.state.repo = repo.clone();
12829 runner.state.config.coordination = crate::config::test_coordination();
12830 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12831 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12832 runner.state.config.graph.advise = false;
12833 runner.roles.implementers = vec![spec("implement")];
12834 runner.roles.judges = vec![spec("judge")];
12835 runner.prep().await.unwrap();
12836 runner
12837 }
12838
12839 #[tokio::test]
12840 #[cfg(unix)]
12841 async fn parked_implementer_persists_completed_work() {
12842 for registered in [false, true] {
12843 let tmp = tempfile::tempdir().unwrap();
12844 let mut runner = registered_runner(tmp.path()).await;
12845 if !registered {
12846 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12847 }
12848 let marker = tmp.path().join("started");
12849 let release = tmp.path().join("release");
12850 let mut implementer = spec("implement");
12851 implementer.command = vec![
12852 "sh".into(),
12853 "-c".into(),
12854 format!(
12855 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12856 marker.display(),
12857 release.display()
12858 ),
12859 ];
12860 runner.roles.implementers = vec![implementer.clone()];
12861 runner.state.config.agents = vec![implementer];
12862 let pause = runner.pause.clone();
12863 let (result, ()) = tokio::join!(runner.implement(), async {
12864 tokio::time::timeout(Duration::from_secs(10), async {
12865 while !marker.exists() {
12866 tokio::time::sleep(Duration::from_millis(10)).await;
12867 }
12868 })
12869 .await
12870 .unwrap();
12871 pause.park();
12872 std::fs::write(&release, "go").unwrap();
12873 });
12874 result.unwrap();
12875 let candidate = &runner.state.candidates[0];
12876 assert!(candidate.commits > 0, "{candidate:?}");
12877 assert!(candidate.summary.contains("saved-result"));
12878 assert_eq!(candidate.files, 1);
12879 let before = candidate.commits;
12880 runner.implement().await.unwrap();
12881 assert_eq!(runner.state.candidates[0].commits, before);
12882 }
12883 }
12884
12885 #[tokio::test]
12886 #[cfg(unix)]
12887 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12888 let tmp = tempfile::tempdir().unwrap();
12889 let mut runner = registered_runner(tmp.path()).await;
12890 let winner = runner.state.candidates[0].clone();
12891 let cache = tmp.path().join("cache");
12892 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12893 runner.state.config.graph.timeout_resource_wait = 0;
12894 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12895 let crate::cache::AcquireOutcome::Acquired(held) =
12896 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12897 else {
12898 panic!("cache must be free")
12899 };
12900 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12901 from_run: "earlier".into(),
12902 generation: 1,
12903 mode: crate::recovery::Mode::VerifyRepair,
12904 reasons: Vec::new(),
12905 reviewed_head: None,
12906 rounds: Vec::new(),
12907 open_findings: Vec::new(),
12908 evidence: None,
12909 criteria_inherited: true,
12910 repair: crate::recovery::Repair::NotStarted,
12911 });
12912 runner.recovery_repair(&winner).await.unwrap();
12913 assert!(matches!(
12914 runner.state.recovery.as_ref().unwrap().repair,
12915 crate::recovery::Repair::WaitingForRestore { .. }
12916 ));
12917 drop(held);
12918 runner.state.resource_block = None;
12919 runner.state.config.graph.timeout_resource_wait = 1;
12920 runner.recovery_repair(&winner).await.unwrap();
12921 assert!(matches!(
12922 runner.state.recovery.as_ref().unwrap().repair,
12923 crate::recovery::Repair::Done { .. }
12924 ));
12925 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12926 }
12927
12928 #[tokio::test]
12929 #[cfg(unix)]
12930 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12931 use std::os::fd::AsRawFd as _;
12932 for graft in [false, true] {
12933 let tmp = tempfile::tempdir().unwrap();
12934 let mut runner = registered_runner(tmp.path()).await;
12935 runner.state.config.graph.timeout_resource_wait = 1;
12936 let winner = runner.state.candidates[0].clone();
12937 let marker = tmp.path().join("fixer-started");
12938 let mut fixer = spec("fixer");
12939 fixer.command = vec![
12940 "sh".into(),
12941 "-c".into(),
12942 format!("echo started >> '{}'; echo done", marker.display()),
12943 ];
12944 runner.roles.fixer = Some(vec![fixer]);
12945 if graft {
12946 let mut loser = winner.clone();
12947 loser.label = 'B';
12948 loser.index = 1;
12949 runner.state.candidates.push(loser);
12950 runner.state.config.graph.graft_runner_up = true;
12951 runner.state.tally = Some(Tally {
12952 first_choice: BTreeMap::new(),
12953 borda: BTreeMap::new(),
12954 winner: winner.label,
12955 rankings: 1,
12956 unanimous_initial: true,
12957 deliberated: false,
12958 changed_votes: 0,
12959 unanimous_final: true,
12960 tie_break: None,
12961 judges: 1,
12962 present: 1,
12963 quorum: 1,
12964 met_quorum: true,
12965 uncontested: None,
12966 });
12967 runner.state.judgements.push(crate::run::Judgement {
12968 judge: 1,
12969 seat: "judge-1".into(),
12970 agent: "judge".into(),
12971 ranking: vec![winner.label, 'B'],
12972 reasons: BTreeMap::new(),
12973 confidence: None,
12974 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12975 order: vec![0, 1],
12976 failed: None,
12977 duration_ms: 0,
12978 });
12979 } else {
12980 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12981 from_run: "earlier".into(),
12982 generation: 1,
12983 mode: crate::recovery::Mode::VerifyRepair,
12984 reasons: Vec::new(),
12985 reviewed_head: None,
12986 rounds: Vec::new(),
12987 open_findings: Vec::new(),
12988 evidence: None,
12989 criteria_inherited: true,
12990 repair: crate::recovery::Repair::NotStarted,
12991 });
12992 }
12993 let reader = crate::repo_lock::SeatLease::for_checkout(
12994 &winner.worktree,
12995 false,
12996 crate::config::test_coordination().as_ref(),
12997 )
12998 .await
12999 .unwrap();
13000 let pause = runner.pause.clone();
13001 let (result, ()) = tokio::join!(
13002 async {
13003 if graft {
13004 runner.graft().await
13005 } else {
13006 runner.recovery_repair(&winner).await
13007 }
13008 },
13009 async {
13010 tokio::time::sleep(Duration::from_millis(100)).await;
13011 pause.park();
13012 }
13013 );
13014 result.unwrap();
13015 assert!(!marker.exists(), "a blocked seat must not spawn");
13016 runner.state = RunState::load(&runner.state.id).unwrap();
13018 if graft {
13019 assert_eq!(
13020 runner.state.graft.as_ref().unwrap().outcome,
13021 GraftOutcome::WaitingForRestore
13022 );
13023 } else {
13024 assert!(matches!(
13025 runner.state.recovery.as_ref().unwrap().repair,
13026 crate::recovery::Repair::WaitingForRestore { .. }
13027 ));
13028 }
13029 drop(reader);
13030 runner.pause = Pause::new();
13031 runner.state.resource_block = None;
13032 let external =
13034 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13035 assert_eq!(
13036 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13037 0
13038 );
13039 let leftover = winner.worktree.join("leftover");
13040 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13041 let result = if graft {
13042 runner.graft().await
13043 } else {
13044 runner.recovery_repair(&winner).await
13045 };
13046 assert!(
13047 result
13048 .unwrap_err()
13049 .is::<crate::repo_lock::ResourceBlocked>()
13050 );
13051 assert!(leftover.exists());
13052 assert!(!marker.exists());
13053 drop(external);
13054 runner.state.resource_block = None;
13055 let result = if graft {
13056 runner.graft().await
13057 } else {
13058 runner.recovery_repair(&winner).await
13059 };
13060 result.unwrap();
13061 assert!(!leftover.exists(), "restore must precede retry");
13062 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13063 if graft {
13064 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13065 } else {
13066 assert!(matches!(
13067 runner.state.recovery.as_ref().unwrap().repair,
13068 crate::recovery::Repair::Done { .. }
13069 ));
13070 }
13071 }
13072 }
13073
13074 #[tokio::test]
13075 #[cfg(unix)]
13076 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13077 use std::os::fd::AsRawFd as _;
13078 let tmp = tempfile::tempdir().unwrap();
13079 let mut runner = registered_runner(tmp.path()).await;
13080 runner.state.config.graph.timeout_resource_wait = 1;
13081 let winner = runner.state.candidates[0].clone();
13082 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13083 runner.state.tally = Some(Tally {
13084 first_choice: BTreeMap::new(),
13085 borda: BTreeMap::new(),
13086 winner: winner.label,
13087 rankings: 0,
13088 unanimous_initial: false,
13089 deliberated: false,
13090 changed_votes: 0,
13091 unanimous_final: false,
13092 tie_break: None,
13093 judges: 0,
13094 present: 0,
13095 quorum: 0,
13096 met_quorum: true,
13097 uncontested: Some("fixture".into()),
13098 });
13099 runner.state.graft = Some(GraftRecord {
13100 pre: pre.clone(),
13101 post: None,
13102 lines: None,
13103 limit: None,
13104 agent: None,
13105 outcome: GraftOutcome::Running,
13106 requested: Vec::new(),
13107 adoption: None,
13108 adoption_error: None,
13109 });
13110 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13111 git::commit_all(&winner.worktree, "unfinished graft")
13112 .await
13113 .unwrap();
13114 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13115 let leftover = winner.worktree.join("leftover");
13116 std::fs::write(&leftover, "untrusted edits").unwrap();
13117 let path = runner.state.repo.join(".git/updater.lock");
13118 let external = std::fs::File::create(&path).unwrap();
13119 assert_eq!(
13120 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13121 0
13122 );
13123 assert!(
13124 runner
13125 .graft()
13126 .await
13127 .unwrap_err()
13128 .is::<crate::repo_lock::ResourceBlocked>()
13129 );
13130 assert!(matches!(
13131 runner.state.graft.as_ref().unwrap().outcome,
13132 GraftOutcome::Running
13133 ));
13134 assert_eq!(
13135 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13136 unfinished
13137 );
13138 assert!(leftover.exists());
13139 drop(external);
13140 std::fs::remove_file(&path).unwrap();
13141 std::fs::create_dir(&path).unwrap();
13142 assert!(runner.graft().await.is_err());
13143 assert!(matches!(
13144 runner.state.graft.as_ref().unwrap().outcome,
13145 GraftOutcome::Running
13146 ));
13147 assert_eq!(
13148 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13149 unfinished
13150 );
13151 assert!(leftover.exists());
13152 std::fs::remove_dir(&path).unwrap();
13153 runner.graft().await.unwrap();
13154 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13155 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13156 assert!(!leftover.exists());
13157 }
13158
13159 #[tokio::test]
13160 #[cfg(unix)]
13161 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13162 use std::os::fd::AsRawFd as _;
13163 let tmp = tempfile::tempdir().unwrap();
13164 let mut runner = registered_runner(tmp.path()).await;
13165 runner.state.prep_complete = Some(false);
13166 runner.state.config.graph.timeout_resource_wait = 1;
13167 runner.state.save().unwrap();
13168 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13169 assert_eq!(
13170 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13171 0
13172 );
13173 runner.execute().await.unwrap();
13174 let disk = RunState::load(&runner.state.id).unwrap();
13175 assert!(disk.driver_exited);
13176 assert_eq!(disk.driver_pid, None);
13177 assert_eq!(runner.state.status, RunStatus::Blocked);
13178 assert_eq!(
13179 runner.state.resource_block.as_ref().unwrap().kind,
13180 "repository coordination"
13181 );
13182 assert!(runner.state.prep_failure.is_none());
13183 assert!(runner.state.seats.is_empty());
13184 runner.pause.park();
13185 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13188 .await
13189 .unwrap()
13190 .unwrap();
13191 assert!(runner.state.parked);
13192 assert!(runner.state.prep_failure.is_none());
13193 let disk = RunState::load(&runner.state.id).unwrap();
13194 assert!(disk.driver_exited);
13195 assert_eq!(disk.driver_pid, None);
13196 }
13197
13198 #[tokio::test]
13199 #[cfg(unix)]
13200 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13201 use std::os::fd::AsRawFd as _;
13202 let tmp = tempfile::tempdir().unwrap();
13203 let mut runner = registered_runner(tmp.path()).await;
13204 runner.state.prep_complete = Some(false);
13205 runner.state.config.graph.timeout_resource_wait = 30;
13206 runner.state.save().unwrap();
13207 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13208 assert_eq!(
13209 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13210 0
13211 );
13212 assert!(
13213 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13214 .await
13215 .is_err()
13216 );
13217 let disk = RunState::load(&runner.state.id).unwrap();
13218 assert!(disk.driver_exited);
13219 assert_eq!(disk.driver_pid, None);
13220 assert!(disk.status.resumable());
13221 assert!(runner.state.driver_exited);
13222 assert_eq!(runner.state.driver_pid, None);
13223 runner.state.save().unwrap();
13224 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13225 }
13226
13227 #[tokio::test]
13228 #[cfg(unix)]
13229 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13230 let tmp = tempfile::tempdir().unwrap();
13231 let mut runner = registered_runner(tmp.path()).await;
13232 let winner = runner.state.candidates[0].clone();
13233 let cache = tmp.path().join("cache");
13234 let started = tmp.path().join("pre-gate-started");
13235 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13236 runner.state.config.verify.gate =
13237 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13238 let home = crate::run::home();
13239 let owner =
13240 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13241 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13242 crate::cache::AcquireOutcome::Acquired(g) => g,
13243 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13244 };
13245 let mut gate = Box::pin(runner.run_gate(&winner));
13246 assert!(
13247 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13248 .await
13249 .is_err()
13250 );
13251 assert!(!started.exists(), "pre-gate must wait for the cache too");
13252 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13256 let editor = crate::repo_lock::SeatLease::checkout_with(
13257 &winner.worktree,
13258 true,
13259 crate::config::test_coordination().as_ref(),
13260 &wait,
13261 )
13262 .await
13263 .unwrap();
13264 drop(editor);
13265 guard.release();
13266 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13267 .await
13268 .unwrap()
13269 .unwrap();
13270 drop(gate);
13271 assert!(outcomes.iter().all(CommandOutcome::ok));
13272 assert!(started.exists());
13273 }
13274
13275 #[tokio::test]
13276 #[cfg(unix)]
13277 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13278 use std::os::fd::AsRawFd as _;
13279
13280 ask_test_home();
13281 let tmp = tempfile::tempdir().unwrap();
13282 let repo = tmp.path().join("repo");
13283 std::fs::create_dir_all(&repo).unwrap();
13284 init_repo(&repo);
13285 let common = repo.join(".git");
13286 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13287 let mut runner = runner_at(RunStatus::Prep);
13288 runner.state.repo = repo.clone();
13289 runner.state.config.coordination = crate::config::test_coordination();
13290 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13291 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13292 runner.state.config.graph.advise = false;
13293 runner.roles.implementers = vec![spec("implement")];
13294 runner.roles.judges = vec![spec("judge")];
13295 runner.prep().await.unwrap();
13296 let winner = runner.state.candidates[0].clone();
13297 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13300 let completed = tmp.path().join("seat-completed");
13301 runner.roles.implementers[0].command = vec![
13302 "sh".into(),
13303 "-c".into(),
13304 format!("touch '{}'; echo completed", completed.display()),
13305 ];
13306 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13307 assert_eq!(
13308 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13309 0
13310 );
13311 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13312 let mut implement = Box::pin(runner.implement());
13313 tokio::select! {
13314 result = &mut implement => panic!("rescue must wait: {result:?}"),
13315 result = tokio::time::timeout(Duration::from_secs(10), async {
13316 while !completed.exists() {
13317 tokio::time::sleep(Duration::from_millis(20)).await;
13318 }
13319 }) => result.unwrap(),
13320 }
13321 assert!(
13322 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13323 .await
13324 .is_err()
13325 );
13326 assert_eq!(
13327 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13328 before
13329 );
13330 assert_eq!(
13331 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13332 0
13333 );
13334 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13335 .await
13336 .unwrap()
13337 .unwrap();
13338 drop(implement);
13339 assert_ne!(
13340 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13341 before
13342 );
13343 assert!(git::is_clean(&winner.worktree).await.unwrap());
13344
13345 let started = tmp.path().join("pre-gate-started");
13346 let proceed = tmp.path().join("pre-gate-proceed");
13347 let checked = tmp.path().join("gate-ran");
13348 runner.state.config.verify.pre_gate = vec![format!(
13349 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13350 started.display(),
13351 proceed.display()
13352 )];
13353 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13354 assert_eq!(
13355 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13356 0
13357 );
13358 let mut gate = Box::pin(runner.run_gate(&winner));
13359 assert!(
13360 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13361 .await
13362 .is_err()
13363 );
13364 assert!(!started.exists());
13365 assert!(!checked.exists());
13366 assert!(git::is_clean(&winner.worktree).await.unwrap());
13367 assert_eq!(
13368 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13369 0
13370 );
13371 tokio::select! {
13372 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13373 result = tokio::time::timeout(Duration::from_secs(10), async {
13374 while !started.exists() {
13375 tokio::time::sleep(Duration::from_millis(20)).await;
13376 }
13377 }) => result.unwrap(),
13378 }
13379 assert_eq!(
13380 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13381 -1,
13382 "mechanical edits must hold the real lock"
13383 );
13384 std::fs::write(proceed, "ready").unwrap();
13385 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13386 .await
13387 .unwrap()
13388 .unwrap();
13389 drop(gate);
13390 assert!(outcomes.iter().all(CommandOutcome::ok));
13391 assert!(checked.exists());
13392 assert!(runner.state.pre_gate_commit.is_some());
13393 assert!(git::is_clean(&winner.worktree).await.unwrap());
13394 assert_eq!(
13395 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13396 0,
13397 "gate completion must release coordination"
13398 );
13399 drop(outsider);
13400 std::fs::remove_file(common.join("updater.lock")).unwrap();
13401 std::fs::create_dir(common.join("updater.lock")).unwrap();
13402 std::fs::remove_file(&started).unwrap();
13403 std::fs::remove_file(&checked).unwrap();
13404 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13405 let error = runner.run_gate(&winner).await.unwrap_err();
13406 assert!(
13407 error
13408 .to_string()
13409 .contains("acquire mandatory repository lock")
13410 );
13411 assert!(
13412 !started.exists(),
13413 "failed acquisition must prevent pre-gate"
13414 );
13415 assert!(!checked.exists(), "failed acquisition must prevent gate");
13416 assert!(git::is_clean(&winner.worktree).await.unwrap());
13417 assert_eq!(
13418 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13419 head
13420 );
13421 }
13422
13423 #[tokio::test]
13424 #[cfg(unix)]
13425 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13426 use std::os::fd::AsRawFd as _;
13427 use std::time::Duration;
13428
13429 ask_test_home();
13430 let tmp = tempfile::tempdir().unwrap();
13431 let repo = tmp.path().join("repo");
13432 std::fs::create_dir_all(&repo).unwrap();
13433 init_repo(&repo);
13434 let common = repo.join(".git");
13435 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13436 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13437 assert_eq!(
13438 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13439 0
13440 );
13441 let mut runner = runner_at(RunStatus::Prep);
13442 runner.state.repo = repo.clone();
13443 runner.state.config.coordination = crate::config::test_coordination();
13444 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13445 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13446 runner.state.config.graph.advise = true;
13447 runner.state.config.graph.advisors = 1;
13448 runner.roles.implementers = vec![spec("implement")];
13449 runner.roles.judges = vec![spec("judge")];
13450 let root = runner.state.worktree_root();
13451 let started = tmp.path().join("setup-started");
13452 let proceed = tmp.path().join("setup-proceed");
13453 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13454 run: Some(format!(
13455 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13456 started.display(),
13457 proceed.display()
13458 )),
13459 ..Default::default()
13460 }];
13461 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13462 let config = git::git(&repo, &["config", "--local", "--list"])
13463 .await
13464 .unwrap();
13465 let mut prep = Box::pin(runner.prep());
13466 assert!(
13467 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13468 .await
13469 .is_err()
13470 );
13471 assert!(
13472 !root.exists(),
13473 "contention must prevent even checkout creation"
13474 );
13475 assert!(!started.exists(), "contention must prevent setup");
13476 assert_eq!(
13477 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13478 branches
13479 );
13480 assert_eq!(
13481 git::git(&repo, &["config", "--local", "--list"])
13482 .await
13483 .unwrap(),
13484 config
13485 );
13486 drop(lock);
13487 tokio::select! {
13488 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13489 result = tokio::time::timeout(Duration::from_secs(10), async {
13490 while !started.exists() {
13491 tokio::time::sleep(Duration::from_millis(20)).await;
13492 }
13493 }) => result.unwrap(),
13494 }
13495 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13496 assert_eq!(
13497 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13498 -1
13499 );
13500 assert!(
13501 tokio::time::timeout(
13502 Duration::from_millis(100),
13503 crate::repo_lock::SeatLease::for_checkout(
13504 &repo,
13505 true,
13506 crate::config::test_coordination().as_ref()
13507 )
13508 )
13509 .await
13510 .is_err(),
13511 "preparation must also exclude local editors"
13512 );
13513 std::fs::write(proceed, "ready").unwrap();
13514 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13515 .await
13516 .unwrap()
13517 .unwrap();
13518 drop(prep);
13519 assert_eq!(runner.state.prep_complete, Some(true));
13520 assert!(root.join("judge-1").exists());
13521 assert!(root.join("advisor-1").exists());
13522 assert_eq!(
13523 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13524 0
13525 );
13526 }
13527
13528 #[tokio::test]
13529 #[cfg(unix)]
13530 async fn preparation_lock_failure_prevents_repository_changes() {
13531 ask_test_home();
13532 let tmp = tempfile::tempdir().unwrap();
13533 let repo = tmp.path().join("repo");
13534 std::fs::create_dir_all(&repo).unwrap();
13535 init_repo(&repo);
13536 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13537 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13540 let mut runner = runner_at(RunStatus::Prep);
13541 runner.state.repo = repo.clone();
13542 runner.state.config.coordination = crate::config::test_coordination();
13543 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13544 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13545 runner.roles.implementers = vec![spec("implement")];
13546 let error = runner.prep().await.unwrap_err();
13547 assert!(
13548 error
13549 .to_string()
13550 .contains("acquire mandatory repository lock")
13551 );
13552 assert!(!runner.state.worktree_root().exists());
13553 assert!(runner.state.candidates.is_empty());
13554 assert!(!repo.join(".git/config.worktree").exists());
13555 assert_eq!(
13556 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13557 "* main"
13558 );
13559 }
13560
13561 #[tokio::test]
13562 async fn preparation_records_creation_intent_before_git_can_fail() {
13563 ask_test_home();
13564 let tmp = tempfile::tempdir().unwrap();
13565 let repo = tmp.path().join("repo");
13566 std::fs::create_dir_all(&repo).unwrap();
13567 init_repo(&repo);
13568 let mut runner = runner_at(RunStatus::Prep);
13569 runner.state.repo = repo.clone();
13570 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13571 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13572 runner.state.config.blind.commit_msg_hook = false;
13573 runner.roles.implementers = vec![spec("one")];
13574 let label = blind::assign_labels(1, runner.state.seed)[0];
13575 let branch = runner.state.branch_for(label);
13576 git::git(&repo, &["branch", &branch]).await.unwrap();
13579 assert!(runner.prep().await.is_err());
13580 let saved = RunState::load(&runner.state.id).unwrap();
13581 assert!(saved.candidates.is_empty());
13582 let event = saved
13583 .events
13584 .iter()
13585 .find(|e| e.node == "prep-intent")
13586 .unwrap();
13587 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13588 assert_eq!(intent["branch"], branch);
13589 assert_eq!(intent["start"], runner.state.base_commit);
13590 assert_eq!(
13591 intent["worktree"],
13592 runner
13593 .state
13594 .worktree_root()
13595 .join(format!("cand-{label}"))
13596 .to_string_lossy()
13597 .as_ref()
13598 );
13599 assert_eq!(intent["picks"], serde_json::json!([]));
13600 }
13601
13602 #[tokio::test]
13603 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13604 ask_test_home();
13605 let tmp = tempfile::tempdir().unwrap();
13606 let repo = tmp.path().join("repo");
13607 std::fs::create_dir_all(&repo).unwrap();
13608 init_repo(&repo);
13609 let mut runner = runner_at(RunStatus::Prep);
13610 runner.state.repo = repo.clone();
13611 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13612 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13613 runner.state.sources.pick_commits.push(refs::Input {
13614 token: "missing".into(),
13615 sha: "0".repeat(40),
13616 provenance: refs::Provenance::ExplicitCli,
13617 });
13618 assert!(runner.prep().await.is_err());
13619 assert!(runner.state.candidates.is_empty());
13620 assert!(!runner.state.worktree_root().exists());
13621 }
13622
13623 #[tokio::test]
13624 async fn prose_and_legacy_inference_never_import_into_candidates() {
13625 ask_test_home();
13626 let tmp = tempfile::tempdir().unwrap();
13627 let repo = tmp.path().join("repo");
13628 std::fs::create_dir_all(&repo).unwrap();
13629 init_repo(&repo);
13630 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13631 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13632 .await
13633 .unwrap();
13634 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13635 git::git(&repo, &["add", "."]).await.unwrap();
13636 git::git(&repo, &["commit", "-m", "historical implementation"])
13637 .await
13638 .unwrap();
13639 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13640 git::git(&repo, &["checkout", "main"]).await.unwrap();
13641 for instruction in [
13642 format!("Example: magi/history/A at {source}"),
13643 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13644 format!("Never import magi/history/A or {source}"),
13645 format!("Conversation answer: I mentioned {source} only as background"),
13646 ] {
13647 let mut runner = runner_at(RunStatus::Prep);
13648 runner.state.repo = repo.clone();
13649 runner.state.base_commit = base.clone();
13650 runner.state.instruction = instruction.clone();
13651 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13652 runner.state.config.blind.commit_msg_hook = false;
13653 runner.state.seeds = vec![refs::Seed {
13654 token: "magi/history/A".into(),
13655 sha: source.clone(),
13656 kind: refs::SeedKind::Unmerged,
13657 branch: true,
13658 detail: String::new(),
13659 }];
13660 runner.roles.implementers = vec![spec("one")];
13661 assert_eq!(seeded_instruction(&runner.state), instruction);
13662 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13664 legacy.as_object_mut().unwrap().remove("sources");
13665 runner.state = serde_json::from_value(legacy).unwrap();
13666 assert_eq!(runner.state.sources, refs::Sources::default());
13667 runner.state.save().unwrap();
13668 runner.state = RunState::load(&runner.state.id).unwrap();
13669 runner.prep().await.unwrap();
13670 for candidate in &runner.state.candidates {
13671 assert_eq!(
13672 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13673 base
13674 );
13675 assert!(!candidate.worktree.join("unwanted").exists());
13676 }
13677 }
13678 }
13679
13680 #[tokio::test]
13681 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13682 ask_test_home();
13683 let tmp = tempfile::tempdir().unwrap();
13684 let repo = tmp.path().join("repo");
13685 std::fs::create_dir_all(&repo).unwrap();
13686 init_repo(&repo);
13687 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13688 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13689 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13690 git::git(&repo, &["add", "."]).await.unwrap();
13691 git::git(&repo, &["commit", "-m", "required seed"])
13692 .await
13693 .unwrap();
13694 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13695 git::git(&repo, &["checkout", "main"]).await.unwrap();
13696 let mut runner = runner_at(RunStatus::Prep);
13697 runner.state.repo = repo.clone();
13698 runner.state.base_commit = base.clone();
13699 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13700 runner.state.config.graph.advise = false;
13701 runner.state.config.blind.commit_msg_hook = false;
13702 runner.state.sources =
13703 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13704 .await
13705 .unwrap();
13706 runner.roles.implementers = vec![spec("one"), spec("two")];
13707 runner.roles.judges = vec![spec("judge")];
13708 let label = blind::assign_labels(2, runner.state.seed)[0];
13709 let branch = runner.state.branch_for(label);
13710 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13711 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13712 .await
13713 .unwrap();
13714 runner.state.candidates.push(Candidate {
13715 index: 0,
13716 label,
13717 agent: "one".into(),
13718 branch,
13719 worktree,
13720 summary: String::new(),
13721 stat: String::new(),
13722 files: 0,
13723 commits: 0,
13724 empty: false,
13725 failed: None,
13726 verified_noop: None,
13727 duration_ms: 0,
13728 folded: false,
13729 });
13730 runner.state.save().unwrap();
13731 runner.state = RunState::load(&runner.state.id).unwrap();
13733 assert_eq!(runner.state.prep_complete, Some(false));
13734 runner.prep().await.unwrap();
13735 assert_eq!(runner.state.prep_complete, Some(true));
13736 assert_eq!(runner.state.candidates.len(), 2);
13737 for candidate in &runner.state.candidates {
13738 assert_eq!(
13739 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13740 git::tree_of(&repo, &seed).await.unwrap()
13741 );
13742 assert_eq!(
13743 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13744 "required commit"
13745 );
13746 }
13747 assert!(runner.state.worktree_root().join("judge-1").exists());
13748 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13749 runner.prep().await.unwrap();
13750 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13751
13752 runner.state.prep_complete = Some(false);
13753 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13754 run: Some(format!(
13755 "test -f '{}'",
13756 tmp.path().join("dependency-ready").display()
13757 )),
13758 ..Default::default()
13759 }];
13760 assert!(runner.prep().await.is_err());
13761 assert!(runner.state.prep_failure.is_some());
13762 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13763 runner.state = RunState::load(&runner.state.id).unwrap();
13764 assert!(
13765 runner.prep().await.is_err(),
13766 "correcting the cause alone must not auto-retry"
13767 );
13768 let failed_at = runner.state.events.last().unwrap().at;
13769 assert!(
13770 runner.retry_preparation(failed_at).is_err(),
13771 "old approvals cannot unlock the hold"
13772 );
13773 let mut task = crate::queue::Task::new(
13774 "retry".into(),
13775 "retry setup".into(),
13776 repo.clone(),
13777 crate::queue::Source::Human,
13778 );
13779 task.start(runner.state.id.clone());
13780 task.hold_machine(Some("setup failed".into()));
13781 task.release_operator("cli", None);
13782 runner
13783 .retry_preparation(task.resumption.as_ref().unwrap().at)
13784 .unwrap();
13785 runner.prep().await.unwrap();
13786 assert!(runner.state.prep_failure.is_none());
13787 assert_eq!(runner.state.prep_complete, Some(true));
13788 assert_eq!(runner.state.candidates.len(), 2);
13789 assert!(runner.state.worktree_root().join("judge-1").exists());
13790 for candidate in &runner.state.candidates {
13791 assert_eq!(
13792 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13793 git::tree_of(&repo, &seed).await.unwrap()
13794 );
13795 }
13796 }
13797
13798 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13799 let mut heads = Vec::new();
13800 for candidate in candidates {
13801 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13802 }
13803 heads
13804 }
13805
13806 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13807 ask_test_home();
13808 let tmp = tempfile::tempdir().unwrap();
13809 let repo = tmp.path().join("repo");
13810 std::fs::create_dir(&repo).unwrap();
13811 init_repo(&repo);
13812 let mut runner = runner_at(RunStatus::Reviewing);
13813 runner.state.repo = repo.clone();
13814 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13815 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13816 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13817 let branch = format!("candidate-{label}");
13818 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13819 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13820 .await
13821 .unwrap();
13822 if label == 'B' {
13823 std::fs::write(worktree.join("change"), "winner").unwrap();
13824 git::rescue_commit(&worktree, "candidate change")
13825 .await
13826 .unwrap();
13827 }
13828 runner.state.candidates.push(Candidate {
13829 index,
13830 label,
13831 agent: "alpha".into(),
13832 branch,
13833 worktree,
13834 summary: "completed reply".into(),
13835 stat: String::new(),
13836 files: usize::from(label == 'B'),
13837 commits: usize::from(label == 'B'),
13838 empty: label == 'A',
13839 failed: None,
13840 verified_noop: None,
13841 duration_ms: 1,
13842 folded: false,
13843 });
13844 }
13845 runner.state.judge_skipped = true;
13846 runner.state.tally = Some(Tally {
13847 first_choice: BTreeMap::new(),
13848 borda: BTreeMap::new(),
13849 winner: 'B',
13850 rankings: 0,
13851 unanimous_initial: false,
13852 deliberated: false,
13853 changed_votes: 0,
13854 unanimous_final: false,
13855 tie_break: None,
13856 judges: 0,
13857 present: 0,
13858 quorum: 0,
13859 met_quorum: true,
13860 uncontested: Some("only B produced a change".into()),
13861 });
13862 runner.state.save().unwrap();
13863 (tmp, runner)
13864 }
13865
13866 #[tokio::test]
13867 #[cfg(unix)]
13868 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13869 for (solo, failed_checkpoint) in
13870 [(true, false), (false, false), (true, true), (false, true)]
13871 {
13872 let (_tmp, mut runner) = preservation_resume_fixture().await;
13873 if solo {
13874 runner.state.candidates.retain(|c| c.label == 'A');
13875 }
13876 runner.state.candidates[0].empty = false;
13877 runner.state.candidates[0].summary.clear();
13878 runner.state.tally = None;
13879 runner.state.judge_skipped = false;
13880 runner.state.status = RunStatus::Implementing;
13881 runner.state.config.disk.min_free_bytes = u64::MAX;
13882 let mut implementer = spec("alpha");
13883 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13884 runner.roles.implementers = vec![implementer];
13885 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13886 if failed_checkpoint {
13887 std::fs::create_dir_all(&patch).unwrap();
13888 }
13889 runner.state.save().unwrap();
13890 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13891 runner.state = RunState::load(&runner.state.id).unwrap();
13892 assert_eq!(runner.state.status, RunStatus::Blocked);
13893 assert!(runner.state.interrupted_implementations.contains(&'A'));
13894 assert_eq!(runner.state.candidates[0].commits, 0);
13895 assert!(!runner.state.candidates[0].empty);
13896 assert!(runner.state.candidates[0].failed.is_none());
13897 if failed_checkpoint {
13898 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13899 std::fs::remove_dir(patch).unwrap();
13900 }
13901 runner.state.resource_block = None;
13902 runner.state.config.disk.min_free_bytes = 0;
13903 runner.roles.implementers[0].command = vec![
13904 "sh".into(),
13905 "-c".into(),
13906 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13907 ];
13908 runner.implement().await.unwrap();
13909 assert!(runner.state.interrupted_implementations.is_empty());
13910 assert!(runner.state.candidate_preservation.is_empty());
13911 assert_eq!(runner.state.candidates[0].commits, 1);
13912 assert!(!runner.state.candidates[0].empty);
13913 assert!(
13914 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13915 .unwrap()
13916 .contains("retried empty interrupted seat")
13917 );
13918 if !solo {
13919 assert_eq!(runner.state.candidates[1].commits, 1);
13920 }
13921 }
13922 }
13923
13924 #[tokio::test]
13925 #[cfg(unix)]
13926 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13927 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13928 let (_tmp, mut runner) = preservation_resume_fixture().await;
13929 runner.state.candidates.retain(|c| c.label == 'A');
13930 runner.state.candidates[0].empty = false;
13931 runner.state.candidates[0].summary.clear();
13932 runner.state.tally = None;
13933 runner.state.judge_skipped = false;
13934 runner.state.status = RunStatus::Implementing;
13935 runner.state.config.disk.min_free_bytes = u64::MAX;
13938 let wt = runner.state.candidates[0].worktree.clone();
13939 std::fs::write(
13940 wt.join("interrupted-source"),
13941 "retain implementation work\n",
13942 )
13943 .unwrap();
13944 if committed {
13945 git::rescue_commit(&wt, "implementation checkpoint")
13946 .await
13947 .unwrap();
13948 }
13949 let lock = git::git(
13950 &wt,
13951 &[
13952 "rev-parse",
13953 "--path-format=absolute",
13954 "--git-path",
13955 "index.lock",
13956 ],
13957 )
13958 .await
13959 .unwrap();
13960 if block_rescue {
13961 std::fs::write(&lock, "fixture-owned lock").unwrap();
13962 }
13963 let mut implementer = spec("alpha");
13964 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13965 runner.roles.implementers = vec![implementer];
13966 runner.state.save().unwrap();
13967 let result = runner.implement().await;
13968 assert_eq!(result.is_err(), block_rescue);
13969 assert_eq!(
13970 runner.state.resource_block.as_ref().unwrap().kind,
13971 "disk pressure"
13972 );
13973 assert!(
13974 runner
13975 .state
13976 .events
13977 .iter()
13978 .any(|e| e.message.contains("agent execution interrupted"))
13979 );
13980 assert!(
13981 runner
13982 .state
13983 .dir()
13984 .join("artifacts/candidate-A-recovery/staged.patch")
13985 .exists()
13986 );
13987 if block_rescue {
13988 let saved = RunState::load(&runner.state.id).unwrap();
13989 assert!(saved.candidate_preservation.contains_key(&'A'));
13990 assert!(!saved.candidates[0].empty);
13991 assert_eq!(
13992 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13993 "retain implementation work\n"
13994 );
13995 std::fs::remove_file(lock).unwrap();
13996 runner.state = saved;
13997 } else {
13998 assert_eq!(runner.state.status, RunStatus::Blocked);
13999 assert_eq!(runner.state.candidates[0].commits, 1);
14000 }
14001 runner.roles.implementers.clear();
14004 runner.state.config.disk.min_free_bytes = 0;
14005 runner.state.resource_block = None;
14006 runner.implement().await.unwrap();
14007 assert!(runner.state.candidate_preservation.is_empty());
14008 assert_eq!(runner.state.candidates[0].commits, 1);
14009 assert!(!runner.state.candidates[0].empty);
14010 assert!(git::is_clean(&wt).await.unwrap());
14011 assert!(
14012 crate::run::read_artifact(&runner.state, "cand-A.patch")
14013 .unwrap()
14014 .contains("retain implementation work")
14015 );
14016 }
14017 }
14018
14019 #[tokio::test]
14020 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14021 let (_tmp, mut runner) = preservation_resume_fixture().await;
14022 runner.implement().await.unwrap();
14024 assert!(runner.state.candidate_preservation.is_empty());
14025 assert!(
14026 !runner
14027 .state
14028 .dir()
14029 .join("artifacts/candidate-A-recovery")
14030 .exists()
14031 );
14032 runner.fold_losers().await.unwrap();
14033 assert!(!runner.state.candidates[0].worktree.exists());
14034 for schema in [14, crate::run::SCHEMA] {
14035 runner.state.schema = schema;
14036 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14037 runner.state.save().unwrap();
14038 runner.state = RunState::load(&runner.state.id).unwrap();
14039 runner.implement().await.unwrap();
14040 assert!(runner.state.candidate_preservation.is_empty());
14041 assert!(runner.state.candidates[0].folded);
14042 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14043 }
14044 }
14045
14046 #[tokio::test]
14047 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14048 let (_tmp, mut runner) = preservation_resume_fixture().await;
14049 let wt = runner.state.candidates[0].worktree.clone();
14050 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14051 runner.state.schema = 14;
14052 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14053 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14054 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14055 for _ in 0..2 {
14056 assert!(runner.implement().await.is_err());
14057 assert_eq!(runner.state.status, RunStatus::Blocked);
14058 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14059 assert!(
14060 runner.state.candidates[0].empty,
14061 "old verdict must remain unusable"
14062 );
14063 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14064 assert!(runner.state.judge_skipped);
14065 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14066 assert!(
14067 crate::run::read_artifact(&runner.state, "cand-A.patch")
14068 .unwrap()
14069 .contains("owned edits")
14070 );
14071 assert!(wt.exists());
14072 assert_eq!(
14073 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14074 .await
14075 .unwrap(),
14076 1
14077 );
14078 runner.state = RunState::load(&runner.state.id).unwrap();
14079 }
14080 }
14081
14082 #[tokio::test]
14083 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14084 let (_tmp, mut runner) = preservation_resume_fixture().await;
14085 let wt = runner.state.candidates[0].worktree.clone();
14086 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14087 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14088 let index_lock = PathBuf::from(
14089 git::git(
14090 &wt,
14091 &[
14092 "rev-parse",
14093 "--path-format=absolute",
14094 "--git-path",
14095 "index.lock",
14096 ],
14097 )
14098 .await
14099 .unwrap(),
14100 );
14101 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14102 runner.state.schema = 14;
14103 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14104 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14105 assert!(runner.implement().await.is_err());
14106 assert!(runner.after_implement().is_err());
14107 assert!(runner.fold_losers().await.is_err());
14108 assert_eq!(
14109 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14110 "failed seat edits"
14111 );
14112 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14113 runner.state = RunState::load(&runner.state.id).unwrap();
14114 std::fs::remove_file(index_lock).unwrap();
14115 assert!(runner.implement().await.is_err());
14117 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14118 assert!(runner.state.candidates[0].empty);
14119 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14120 assert!(
14121 crate::run::read_artifact(&runner.state, "cand-A.patch")
14122 .unwrap()
14123 .contains("failed seat edits")
14124 );
14125 assert!(wt.exists());
14126 }
14127
14128 #[tokio::test]
14129 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14130 let (_tmp, mut runner) = preservation_resume_fixture().await;
14131 let repo = runner.state.repo.clone();
14132 let common = git::git(
14133 &repo,
14134 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14135 )
14136 .await
14137 .unwrap();
14138 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14139 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14140 assert!(runner.fold_losers().await.is_err());
14141 assert!(!runner.state.candidates[0].worktree.exists());
14142 assert!(!runner.state.candidates[0].folded);
14143 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14144 runner.state = RunState::load(&runner.state.id).unwrap();
14145 std::fs::remove_file(branch_lock).unwrap();
14146 let judge = runner.state.worktree_root().join("judge-1");
14147 git::worktree_add_detached(&repo, &judge, "main")
14148 .await
14149 .unwrap();
14150 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14151 runner.roles.judges = vec![spec("judge")];
14152 assert!(runner.fold_losers().await.is_err());
14153 runner.state = RunState::load(&runner.state.id).unwrap();
14154 assert!(
14155 runner.state.candidates[0].folded,
14156 "candidate progress saved before judge failure"
14157 );
14158 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14159 assert!(judge.join("uncommitted").exists());
14160 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14161 runner.fold_losers().await.unwrap();
14162 assert!(!judge.exists());
14163 runner.fold_losers().await.unwrap();
14164 }
14165
14166 #[tokio::test]
14167 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14168 let (_tmp, mut runner) = preservation_resume_fixture().await;
14169 runner.state.candidates.retain(|c| c.label == 'B');
14170 runner.state.tally = None;
14171 runner.state.judge_skipped = false;
14172 runner.state.prep_complete = Some(true);
14173 runner.state.config.graph.advise = false;
14174 runner.state.config.graph.selected_implementer = Some("alpha".into());
14175 let wt = runner.state.candidates[0].worktree.clone();
14176 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14177 runner
14178 .state
14179 .candidate_preservation
14180 .insert('B', "awaiting persistence".into());
14181 let index_lock = git::git(
14182 &wt,
14183 &[
14184 "rev-parse",
14185 "--path-format=absolute",
14186 "--git-path",
14187 "index.lock",
14188 ],
14189 )
14190 .await
14191 .unwrap();
14192 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14193 runner.state.save().unwrap();
14194 assert!(runner.execute_graph().await.is_err());
14195 assert_eq!(runner.state.status, RunStatus::Blocked);
14196 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14197 std::fs::remove_file(index_lock).unwrap();
14198 for _ in 0..2 {
14201 runner.state = RunState::load(&runner.state.id).unwrap();
14202 runner.pause.park();
14203 runner.execute_graph().await.unwrap();
14204 assert_eq!(runner.state.status, RunStatus::Implementing);
14205 assert!(runner.state.candidate_preservation.is_empty());
14206 assert_eq!(runner.state.candidates[0].commits, 2);
14207 assert!(runner.state.tally.is_none());
14208 }
14209 runner.implement().await.unwrap();
14210 assert_eq!(runner.state.status, RunStatus::Judging);
14211 runner.state.status = RunStatus::Blocked;
14213 runner.state.candidates[0].failed = Some("implementation failed".into());
14214 runner.state.save().unwrap();
14215 let error = runner.execute_graph().await.unwrap_err();
14216 assert!(
14217 error
14218 .to_string()
14219 .contains("selected implementation is held")
14220 );
14221 }
14222
14223 #[tokio::test]
14224 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14225 let (_tmp, mut runner) = preservation_resume_fixture().await;
14226 let repo = runner.state.repo.clone();
14227 let candidate = runner.state.candidates[0].clone();
14228 let registered = git::worktree_holding(&repo, &candidate.branch)
14231 .await
14232 .unwrap()
14233 .unwrap();
14234 assert_eq!(
14235 registered.canonicalize().unwrap(),
14236 candidate.worktree.canonicalize().unwrap()
14237 );
14238 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14239 assert_eq!(
14240 git::worktree_holding(&repo, &candidate.branch)
14241 .await
14242 .unwrap(),
14243 Some(registered)
14244 );
14245 assert!(
14246 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14247 "the stale registration initially keeps the branch checked out"
14248 );
14249 runner.fold_losers().await.unwrap();
14250 assert!(runner.state.candidates[0].folded);
14251 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14252 assert_eq!(
14253 git::worktree_holding(&repo, &candidate.branch)
14254 .await
14255 .unwrap(),
14256 None
14257 );
14258 runner.state = RunState::load(&runner.state.id).unwrap();
14259 runner.fold_losers().await.unwrap();
14260 }
14261
14262 #[tokio::test]
14263 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14264 let (_tmp, mut runner) = preservation_resume_fixture().await;
14265 let repo = runner.state.repo.clone();
14266 let candidate = runner.state.candidates[0].clone();
14267 let winner = runner.state.candidates[1].clone();
14268 let registered = git::worktree_holding(&repo, &candidate.branch)
14269 .await
14270 .unwrap()
14271 .unwrap();
14272 assert_eq!(
14273 registered.canonicalize().unwrap(),
14274 candidate.worktree.canonicalize().unwrap()
14275 );
14276 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14277 assert_eq!(
14278 git::worktree_holding(&repo, &candidate.branch)
14279 .await
14280 .unwrap(),
14281 Some(registered)
14282 );
14283 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14284
14285 for _ in 0..2 {
14286 fold_run(&mut runner.state, false, &crate::run::home())
14287 .await
14288 .unwrap();
14289 runner.state = RunState::load(&runner.state.id).unwrap();
14290 assert!(runner.state.candidates[0].folded);
14291 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14292 assert_eq!(
14293 git::worktree_holding(&repo, &candidate.branch)
14294 .await
14295 .unwrap(),
14296 None
14297 );
14298 assert!(!runner.state.candidates[1].folded);
14299 assert!(winner.worktree.exists());
14300 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14301 }
14302 }
14303
14304 #[tokio::test]
14305 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14306 ask_test_home();
14307 let tmp = tempfile::tempdir().unwrap();
14308 let repo = tmp.path().join("repo");
14309 std::fs::create_dir(&repo).unwrap();
14310 init_repo(&repo);
14311 let mut runner = runner_at(RunStatus::Implementing);
14312 runner.state.repo = repo.clone();
14313 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14314 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14315 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14316 let branch = format!("candidate-{label}");
14317 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14318 git::worktree_add_branch(&repo, &wt, &branch, "main")
14319 .await
14320 .unwrap();
14321 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14322 git::git(&wt, &["add", "README.md"]).await.unwrap();
14323 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14324 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14325 runner.state.candidates.push(Candidate {
14326 index,
14327 label,
14328 agent: "alpha".to_owned(),
14329 branch,
14330 worktree: wt,
14331 summary: "completed implementation".to_owned(),
14332 stat: String::new(),
14333 files: 0,
14334 commits: 0,
14335 empty: false,
14336 failed: None,
14337 verified_noop: None,
14338 duration_ms: 42,
14339 folded: false,
14340 });
14341 runner
14342 .state
14343 .candidate_preservation
14344 .insert(label, "awaiting persistence".to_owned());
14345 }
14346 let wt = runner.state.candidates[0].worktree.clone();
14347 let index_lock = git::git(
14348 &wt,
14349 &[
14350 "rev-parse",
14351 "--path-format=absolute",
14352 "--git-path",
14353 "index.lock",
14354 ],
14355 )
14356 .await
14357 .unwrap();
14358 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14359 runner.state.save().unwrap();
14360 assert!(runner.persist_candidates().await.is_err());
14361 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14362 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14363 assert_eq!(
14364 runner.state.candidates[1].commits, 1,
14365 "other results are still collected"
14366 );
14367 assert!(
14368 !runner.state.candidates[0].empty,
14369 "failed rescue is not no change"
14370 );
14371 assert_eq!(
14372 runner.state.candidates[0].summary,
14373 "completed implementation"
14374 );
14375 assert!(runner.after_implement().is_err());
14376 let home = tmp.path().join("fold-home");
14377 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14378 assert!(wt.join("new-file").exists());
14379 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14380 assert!(recovery.join("staged.patch").exists());
14381 assert!(recovery.join("unstaged.patch").exists());
14382 assert_eq!(
14383 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14384 "untracked edits\n"
14385 );
14386 runner.state.begin_pass();
14387 runner.state.end_pass_parked(ParkCause::Upgrade {
14388 from: "1.0".into(),
14389 to: Some("1.1".into()),
14390 });
14391 runner.state.save().unwrap();
14392 runner.state = RunState::load(&runner.state.id).unwrap();
14393 assert!(matches!(
14394 runner.state.passes.last().unwrap().end,
14395 Some(crate::run::PassEnd::Parked {
14396 cause: ParkCause::Upgrade { .. }
14397 })
14398 ));
14399 std::fs::remove_file(index_lock).unwrap();
14400 runner.implement().await.unwrap();
14402 assert!(runner.state.candidate_preservation.is_empty());
14403 assert!(
14404 runner
14405 .state
14406 .candidates
14407 .iter()
14408 .all(|c| c.commits == 1 && !c.empty)
14409 );
14410 assert!(git::is_clean(&wt).await.unwrap());
14411 assert!(
14412 crate::run::read_artifact(&runner.state, "cand-A.patch")
14413 .unwrap()
14414 .contains("later tracked edits")
14415 );
14416 }
14417
14418 #[tokio::test]
14419 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14420 ask_test_home();
14421 let tmp = tempfile::tempdir().unwrap();
14422 let repo = tmp.path().join("repo");
14423 std::fs::create_dir(&repo).unwrap();
14424 init_repo(&repo);
14425 let wt = tmp.path().join("candidate");
14426 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14427 .await
14428 .unwrap();
14429 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14430 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14431 assert!(wt.join("uncommitted").exists());
14432 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14433 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14434 let mut runner = runner_at(RunStatus::Implementing);
14435 runner.state.repo = repo.clone();
14436 runner.state.base_commit = "missing-base".to_owned();
14437 runner.state.candidates.push(Candidate {
14438 index: 0,
14439 label: 'A',
14440 agent: "alpha".into(),
14441 branch: "candidate".into(),
14442 worktree: wt.clone(),
14443 summary: "completed".into(),
14444 stat: String::new(),
14445 files: 0,
14446 commits: 0,
14447 empty: false,
14448 failed: None,
14449 verified_noop: None,
14450 duration_ms: 1,
14451 folded: false,
14452 });
14453 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14454 assert!(
14455 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14456 .await
14457 .is_err()
14458 );
14459 assert!(!runner.state.candidates[0].folded);
14460 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14461 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14462 runner
14463 .state
14464 .candidate_preservation
14465 .insert('A', "pending".into());
14466 assert!(runner.persist_candidates().await.is_err());
14467 assert!(!runner.state.candidates[0].empty);
14468 assert!(!runner.state.candidates[0].folded);
14469 assert!(wt.exists());
14470 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14471 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14472 std::fs::create_dir_all(&patch_path).unwrap();
14473 assert!(
14474 runner.persist_candidates().await.is_err(),
14475 "artifact failure stays pending"
14476 );
14477 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14478 assert!(!runner.state.candidates[0].empty);
14479 std::fs::remove_dir(patch_path).unwrap();
14480 runner.persist_candidates().await.unwrap();
14481 assert!(
14482 runner.state.candidates[0].empty,
14483 "only successful inspection confirms no change"
14484 );
14485 }
14486
14487 fn init_repo(dir: &Path) {
14488 let run = |args: &[&str]| {
14489 let out = std::process::Command::new("git")
14490 .args(args)
14491 .current_dir(dir)
14492 .quiet()
14493 .output()
14494 .expect("spawn git");
14495 assert!(
14496 out.status.success(),
14497 "git {args:?} failed: {}",
14498 String::from_utf8_lossy(&out.stderr)
14499 );
14500 };
14501 run(&["init", "-b", "main"]);
14502 run(&["config", "user.name", "magi test"]);
14503 run(&["config", "user.email", "magi@example.com"]);
14504 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14505 run(&["add", "-A"]);
14506 run(&["commit", "-m", "init"]);
14507 }
14508
14509 fn ask_test_home() {
14517 crate::run::pin_test_home();
14518 }
14519
14520 fn runner_at(status: RunStatus) -> Runner {
14523 let mut state = RunState::new(
14524 PathBuf::from("/nonexistent/repo"),
14525 "main".to_owned(),
14526 "deadbeef".to_owned(),
14527 "task".to_owned(),
14528 Config::default(),
14529 );
14530 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14533 state.id = format!(
14534 "20000101-000000-{:04x}",
14535 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14536 );
14537 state.status = status;
14538 Runner {
14539 state,
14540 roles: ResolvedRoles {
14541 implementers: Vec::new(),
14542 judges: Vec::new(),
14543 reviewers: Vec::new(),
14544 fixer: None,
14545 conductor: conductor(),
14546 implementer_roster: Vec::new(),
14547 judge_roster: Vec::new(),
14548 reviewer_roster: Vec::new(),
14549 },
14550 sem: Arc::new(Semaphore::new(1)),
14551 pause: Pause::new(),
14552 interrupt: Pause::new(),
14553 }
14554 }
14555
14556 #[test]
14557 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14558 crate::run::pin_test_home();
14559 let mut runner = runner_at(RunStatus::Gating);
14560 runner.state.save().unwrap();
14561 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14564 assert!(runner.claim_driver().is_err());
14565 assert!(runner.state.driver_exited);
14566 assert_eq!(runner.state.driver_pid, None);
14567 let disk = RunState::load(&runner.state.id).unwrap();
14568 assert!(disk.driver_exited);
14569 assert_eq!(disk.driver_pid, None);
14570 assert_eq!(disk.status, RunStatus::Gating);
14571 }
14572
14573 #[tokio::test]
14574 async fn an_early_graph_error_releases_its_driver() {
14575 crate::run::pin_test_home();
14576 let mut runner = runner_at(RunStatus::Blocked);
14577 runner.state.config.graph.selected_implementer = Some("selected".into());
14578 runner.state.save().unwrap();
14579 assert!(runner.execute().await.is_err());
14580 let disk = RunState::load(&runner.state.id).unwrap();
14581 assert!(disk.driver_exited);
14582 assert_eq!(disk.driver_pid, None);
14583 assert_eq!(disk.status, RunStatus::Blocked);
14584 }
14585
14586 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14587 crate::run::MergeRepair {
14588 question: "repair-question".into(),
14589 head: "old-head".into(),
14590 pr: "https://github.com/o/r/pull/42".into(),
14591 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14592 gate: vec![outcome(Some(0), false)],
14593 gate_head: Some("old-head".into()),
14594 gate_fixes: Vec::new(),
14595 pre_gate: Vec::new(),
14596 phase,
14597 failure: None,
14598 }
14599 }
14600
14601 #[tokio::test]
14602 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14603 crate::run::pin_test_home();
14604 let mut runner = runner_at(RunStatus::Landing);
14605 runner.state.merge_repairs.push(merge_repair_request(
14606 crate::run::MergeRepairPhase::Requested,
14607 ));
14608 runner.state.save().unwrap();
14609 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14610 assert!(
14611 runner
14612 .execute()
14613 .await
14614 .unwrap_err()
14615 .to_string()
14616 .contains("already running")
14617 );
14618 assert_eq!(
14619 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14620 crate::run::MergeRepairPhase::Requested
14621 );
14622 drop(claim);
14623 let mut disk = runner.state.clone();
14624 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14625 disk.save().unwrap();
14626 assert!(
14627 runner
14628 .execute()
14629 .await
14630 .unwrap_err()
14631 .to_string()
14632 .contains("decision changed")
14633 );
14634 assert_eq!(
14635 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14636 crate::run::MergeRepairPhase::Complete
14637 );
14638 }
14639
14640 #[test]
14641 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14642 crate::run::pin_test_home();
14643 let mut runner = runner_at(RunStatus::Landing);
14644 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14645 runner.state.reviews = request.reviews.clone();
14646 runner.state.gate = request.gate.clone();
14647 runner.state.gate_ran = true;
14648 runner.state.land_approval = Some(crate::run::LandApproval {
14649 question: "old".into(),
14650 head: "old-head".into(),
14651 });
14652 runner.state.merge_repairs.push(request);
14653 runner.begin_merge_repair_review(0).unwrap();
14654 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14655 assert!(!runner.state.gate_status().ok());
14656 assert!(runner.state.land_approval.is_none());
14657 let disk = RunState::load(&runner.state.id).unwrap();
14658 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14659 assert!(disk.merge_repairs[0].gate[0].ok());
14660 assert_eq!(
14661 disk.merge_repairs[0].phase,
14662 crate::run::MergeRepairPhase::Reviewing
14663 );
14664 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14665 assert!(disk.reviews.is_empty());
14666 }
14667
14668 #[tokio::test]
14669 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14670 crate::run::pin_test_home();
14671 for phase in [
14672 crate::run::MergeRepairPhase::Fixing,
14673 crate::run::MergeRepairPhase::Requested,
14674 ] {
14675 let mut runner = runner_at(RunStatus::Landing);
14676 runner.state.config.graph.land_rounds = 0;
14677 runner.state.merge_repairs.push(merge_repair_request(phase));
14678 runner.state.save().unwrap();
14679 assert!(!runner.repair_merge_review().await.unwrap());
14680 assert_eq!(runner.state.status, RunStatus::Blocked);
14681 assert_eq!(
14682 runner.state.merge_repairs[0].phase,
14683 crate::run::MergeRepairPhase::Failed
14684 );
14685 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14686 assert!(
14687 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14688 "interrupted"
14689 } else {
14690 "budget exhausted"
14691 })
14692 );
14693 assert!(runner.state.merge.is_none());
14694 }
14695 }
14696
14697 #[test]
14701 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14702 crate::run::pin_test_home();
14703 let mut runner = runner_at(RunStatus::Implementing);
14704 let interrupt = Pause::new();
14705 runner.watch_interrupt(interrupt.clone());
14706
14707 interrupt.park_because("task a1b2 asked to run first");
14708
14709 assert!(runner.park_here().expect("park_here"));
14710 assert!(runner.state.parked);
14711 let last = runner.state.events.last().expect("a park event");
14712 assert_eq!(last.node, "park");
14713 assert!(
14714 last.message.contains("task a1b2 asked to run first"),
14715 "expected the interrupt reason in {:?}",
14716 last.message
14717 );
14718 }
14719
14720 #[test]
14723 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14724 crate::run::pin_test_home();
14725 let mut runner = runner_at(RunStatus::Implementing);
14726 runner.state.begin_pass();
14727 let shutdown = Pause::new();
14728 runner.on_pause(shutdown.clone());
14729 let upgrade = ParkCause::Upgrade {
14730 from: "0.121.0".to_owned(),
14731 to: Some("0.121.1".to_owned()),
14732 };
14733 shutdown.park_for(upgrade.clone());
14734 shutdown.park_for(ParkCause::Operator);
14735
14736 assert!(runner.park_here().expect("park_here"));
14737 assert_eq!(
14738 runner.state.passes[0].end,
14739 Some(crate::run::PassEnd::Parked { cause: upgrade })
14740 );
14741 let last = runner.state.events.last().expect("a park event");
14742 assert!(last.message.contains("0.121.0"), "{}", last.message);
14743 }
14744
14745 #[test]
14753 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14754 crate::run::pin_test_home();
14755 let mut runner = runner_at(RunStatus::Implementing);
14756 let shutdown = Pause::new();
14757 runner.on_pause(shutdown.clone());
14758 let interrupt = Pause::new();
14759 runner.watch_interrupt(interrupt.clone());
14760
14761 assert!(!runner.park_here().expect("park_here"));
14763 assert!(!runner.state.parked);
14764
14765 interrupt.park_because("test");
14767 assert!(!shutdown.parked());
14768 assert!(runner.park_here().expect("park_here"));
14769 }
14770
14771 #[tokio::test]
14785 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14786 crate::run::pin_test_home();
14787 let mut runner = runner_at(RunStatus::Implementing);
14788 let interrupt = Pause::new();
14789 runner.watch_interrupt(interrupt.clone());
14790
14791 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14792 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14793
14794 let node = async move {
14798 started_tx.send(()).expect("send started");
14799 finish_rx.await.expect("recv finish");
14800 "node finished"
14801 };
14802
14803 let interrupter = async move {
14804 started_rx.await.expect("recv started");
14805 interrupt.park_because("higher-priority task waiting");
14807 tokio::task::yield_now().await;
14811 finish_tx.send(()).expect("send finish");
14812 };
14813
14814 let (node_result, ()) = tokio::join!(node, interrupter);
14815 assert_eq!(
14816 node_result, "node finished",
14817 "the in-flight call ran to completion"
14818 );
14819
14820 assert!(runner.park_here().expect("park_here"));
14823 assert!(runner.state.parked);
14824 }
14825
14826 #[test]
14832 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14833 crate::run::pin_test_home();
14834 let mut runner = runner_at(RunStatus::Judging);
14835 runner.state.config.agents = vec![conductor()];
14839 runner.state.candidates = vec![Candidate {
14840 index: 0,
14841 label: 'A',
14842 agent: "alpha".to_owned(),
14843 branch: "magi/x/A".to_owned(),
14844 worktree: PathBuf::from("/nonexistent/worktree"),
14845 summary: "did the thing".to_owned(),
14846 stat: "1 file changed".to_owned(),
14847 files: 1,
14848 commits: 1,
14849 empty: false,
14850 failed: None,
14851 verified_noop: None,
14852 duration_ms: 1234,
14853 folded: false,
14854 }];
14855 let run_id = runner.state.id.clone();
14856
14857 let interrupt = Pause::new();
14858 runner.watch_interrupt(interrupt.clone());
14859 interrupt.park_because("task c3d4 asked to run first");
14860 assert!(runner.park_here().expect("park_here"));
14861
14862 let resumed = Runner::resume(&run_id).expect("resume");
14863 assert_eq!(resumed.state.candidates.len(), 1);
14864 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14865 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14866 assert_eq!(resumed.state.status, runner.state.status);
14867 assert!(
14868 resumed.state.parked,
14869 "still parked until `execute` actually walks the graph again"
14870 );
14871 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14872 }
14873
14874 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14876 let mut q = ask::Question::new(
14877 run.to_owned(),
14878 "implement".to_owned(),
14879 "impl-A".to_owned(),
14880 "Which storage backend should the cache use?".to_owned(),
14881 String::new(),
14882 vec!["SQLite".to_owned(), "Redis".to_owned()],
14883 );
14884 store.put(&mut q).unwrap();
14885 q
14886 }
14887
14888 #[test]
14889 fn a_failed_runs_open_question_is_abandoned() {
14890 ask_test_home();
14891 let store = ask::Questions::open();
14892 let mut runner = runner_at(RunStatus::Failed);
14893 let run = runner.state.id.clone();
14894 let q = ask_open_question(&store, &run);
14895
14896 runner.settle_questions();
14897
14898 let back = store.get(&q.id).unwrap();
14899 assert!(
14900 !back.status.open(),
14901 "the seat that asked died with the run; nobody is left to read an answer"
14902 );
14903 assert!(
14904 back.detail.contains(&run) && back.detail.contains("failed"),
14905 "the reason names what the run became, not just that it is gone: {}",
14906 back.detail
14907 );
14908 }
14909
14910 #[test]
14911 fn a_merged_runs_open_question_is_abandoned_too() {
14912 ask_test_home();
14913 let store = ask::Questions::open();
14914 for status in [RunStatus::Merged, RunStatus::Ready] {
14917 let mut runner = runner_at(status);
14918 let run = runner.state.id.clone();
14919 let q = ask_open_question(&store, &run);
14920
14921 runner.settle_questions();
14922
14923 let back = store.get(&q.id).unwrap();
14924 assert!(
14925 !back.status.open(),
14926 "{status:?} run's question must not outlive the run"
14927 );
14928 }
14929 }
14930
14931 #[test]
14932 fn a_still_resumable_runs_open_question_is_left_alone() {
14933 ask_test_home();
14934 let store = ask::Questions::open();
14935 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14941 let mut runner = runner_at(status);
14942 let run = runner.state.id.clone();
14943 let q = ask_open_question(&store, &run);
14944
14945 runner.settle_questions();
14946
14947 let back = store.get(&q.id).unwrap();
14948 assert!(
14949 back.status.open(),
14950 "{status:?} is still alive; the question must still be waiting"
14951 );
14952 }
14953 }
14954
14955 #[test]
14956 fn settle_questions_never_touches_an_already_answered_question() {
14957 ask_test_home();
14958 let store = ask::Questions::open();
14959 let mut runner = runner_at(RunStatus::Failed);
14960 let run = runner.state.id.clone();
14961 let mut q = ask_open_question(&store, &run);
14962 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14963 .unwrap();
14964 store.put(&mut q).unwrap();
14965
14966 runner.settle_questions();
14971 runner.settle_questions();
14972
14973 let back = store.get(&q.id).unwrap();
14974 assert_eq!(
14975 back.status,
14976 ask::QuestionStatus::Answered,
14977 "a real answer is a decision on record, never overwritten by a sweep"
14978 );
14979 }
14980
14981 #[tokio::test]
14992 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14993 crate::run::pin_test_home();
14994 let tmp = tempfile::tempdir().expect("tempdir");
14995 let repo = tmp.path().join("repo");
14996 std::fs::create_dir_all(&repo).unwrap();
14997 init_repo(&repo);
14998
14999 let mut config = Config::default();
15000 config.graph.worktree_root = Some(tmp.path().join("wt"));
15001
15002 let mut state = RunState::new(
15003 repo.clone(),
15004 "main".to_owned(),
15005 "deadbeef".to_owned(),
15006 "task".to_owned(),
15007 config,
15008 );
15009 let root = state.worktree_root();
15010 let wt_a = root.join("cand-A");
15011 let wt_b = root.join("cand-B");
15012 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15013 .await
15014 .expect("worktree A");
15015 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15016 .await
15017 .expect("worktree B");
15018
15019 state.candidates = vec![
15020 Candidate {
15021 index: 0,
15022 label: 'A',
15023 agent: "alpha".to_owned(),
15024 branch: "magi/x/A".to_owned(),
15025 worktree: wt_a.clone(),
15026 summary: String::new(),
15027 stat: String::new(),
15028 files: 0,
15029 commits: 0,
15030 empty: false,
15031 failed: None,
15032 verified_noop: None,
15033 duration_ms: 0,
15034 folded: false,
15035 },
15036 Candidate {
15037 index: 1,
15038 label: 'B',
15039 agent: "beta".to_owned(),
15040 branch: "magi/x/B".to_owned(),
15041 worktree: wt_b.clone(),
15042 summary: String::new(),
15043 stat: String::new(),
15044 files: 0,
15045 commits: 0,
15046 empty: false,
15047 failed: None,
15048 verified_noop: None,
15049 duration_ms: 0,
15050 folded: false,
15051 },
15052 ];
15053 state.tally = Some(Tally {
15054 first_choice: BTreeMap::from([('A', 1)]),
15055 borda: BTreeMap::new(),
15056 winner: 'A',
15057 rankings: 1,
15058 unanimous_initial: true,
15059 deliberated: false,
15060 changed_votes: 0,
15061 unanimous_final: true,
15062 tie_break: None,
15063 judges: 1,
15064 present: 1,
15065 quorum: 1,
15066 met_quorum: true,
15067 uncontested: None,
15068 });
15069 state.status = RunStatus::Ready;
15070
15071 fold_run(&mut state, false, &crate::run::home())
15072 .await
15073 .expect("fold_run");
15074
15075 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15076 assert!(
15077 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15078 "the unmerged winner's branch survives"
15079 );
15080 assert!(
15081 !state.candidates[0].folded,
15082 "the winner is not marked folded"
15083 );
15084
15085 assert!(!wt_b.exists(), "the loser's worktree is removed");
15086 assert!(
15087 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15088 "the loser's branch is removed"
15089 );
15090 assert!(state.candidates[1].folded, "the loser is marked folded");
15091 }
15092
15093 #[tokio::test]
15094 #[cfg(unix)]
15095 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15096 use std::os::fd::AsRawFd as _;
15097 use std::time::Duration;
15098
15099 let tmp = tempfile::tempdir().unwrap();
15100 let repo = tmp.path().join("repo");
15101 std::fs::create_dir_all(&repo).unwrap();
15102 init_repo(&repo);
15103 let common = repo.join(".git");
15104 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15105 let home = tmp.path().join("home");
15106 let mut config = Config::default();
15107 config.coordination = crate::config::test_coordination();
15108 config.graph.worktree_root = Some(tmp.path().join("wt"));
15109 config.graph.timeout_resource_wait = 1;
15110 let mut state = RunState::new(
15111 repo.clone(),
15112 "main".into(),
15113 "deadbeef".into(),
15114 "task".into(),
15115 config,
15116 );
15117 let root = state.worktree_root();
15118 let candidate = root.join("cand-A");
15119 let reviewer = root.join("review-1");
15120 let branch = state.branch_for('A');
15121 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15122 .await
15123 .unwrap();
15124 git::worktree_add_detached(&repo, &reviewer, "main")
15125 .await
15126 .unwrap();
15127 git::acquire_worktree_config(&repo).await.unwrap();
15128 state.enabled_worktree_config = true;
15129 state.status = RunStatus::Ready;
15130 state.candidates.push(Candidate {
15131 index: 0,
15132 label: 'A',
15133 agent: "implement".into(),
15134 branch: branch.clone(),
15135 worktree: candidate.clone(),
15136 summary: String::new(),
15137 stat: String::new(),
15138 files: 0,
15139 commits: 0,
15140 empty: false,
15141 failed: None,
15142 verified_noop: None,
15143 duration_ms: 0,
15144 folded: false,
15145 });
15146 state.save_under(&home).unwrap();
15147 let config_before = std::fs::read(common.join("config")).unwrap();
15148 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15149 .await
15150 .unwrap();
15151 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15154 assert_eq!(
15155 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15156 0
15157 );
15158 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15159 assert!(
15160 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15161 .await
15162 .is_err()
15163 );
15164 assert!(candidate.exists());
15165 assert!(reviewer.exists());
15166 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15167 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15168 assert_eq!(
15169 git::git(&repo, &["worktree", "list", "--porcelain"])
15170 .await
15171 .unwrap(),
15172 worktrees_before
15173 );
15174 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15175 .await
15176 .unwrap()
15177 .unwrap_err();
15178 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15179 drop(fold);
15180 assert!(candidate.exists() && reviewer.exists());
15181 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15182 assert_eq!(
15183 state.resource_block.as_ref().unwrap().kind,
15184 "repository coordination"
15185 );
15186 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15187 drop(external);
15188 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15189 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15190 .await
15191 .unwrap()
15192 .unwrap();
15193 drop(fold);
15194 assert!(removed.contains(&branch));
15195 assert!(!root.exists());
15196 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15197 assert!(state.candidates[0].folded);
15198 assert!(!state.enabled_worktree_config);
15199 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15200 let enabled = git::git_raw(
15201 &repo,
15202 &["config", "--local", "--get", "extensions.worktreeConfig"],
15203 )
15204 .await
15205 .unwrap();
15206 assert!(!enabled.ok(), "the last config reference was released");
15207 }
15208
15209 #[tokio::test]
15212 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15213 let tmp = tempfile::tempdir().expect("tempdir");
15214 let repo = tmp.path().join("repo");
15215 std::fs::create_dir_all(&repo).unwrap();
15216 init_repo(&repo);
15217 let home = tmp.path().join("home");
15218
15219 let mut config = Config::default();
15220 config.graph.worktree_root = Some(tmp.path().join("wt"));
15221 let mut state = RunState::new(
15222 repo.clone(),
15223 "main".to_owned(),
15224 "deadbeef".to_owned(),
15225 "task".to_owned(),
15226 config,
15227 );
15228 git::git(&repo, &["branch", "magi/x/A", "main"])
15230 .await
15231 .expect("branch");
15232 state.candidates = vec![Candidate {
15233 index: 0,
15234 label: 'A',
15235 agent: "alpha".to_owned(),
15236 branch: "magi/x/A".to_owned(),
15237 worktree: state.worktree_root().join("cand-A"),
15238 summary: String::new(),
15239 stat: String::new(),
15240 files: 0,
15241 commits: 0,
15242 empty: false,
15243 failed: None,
15244 verified_noop: None,
15245 duration_ms: 0,
15246 folded: true,
15247 }];
15248 state.released_to = Some("20260901-000000-new1".to_owned());
15249 state.released_branches = vec!["magi/x/A".to_owned()];
15250
15251 fold_run(&mut state, true, &home).await.expect("fold_run");
15252 state.status = RunStatus::Merged;
15253 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15254 state.merge = Some(crate::run::MergeOutcome {
15255 mode: MergeMode::Pr,
15256 ok: true,
15257 detail: "confirmed merged".into(),
15258 empty: false,
15259 });
15260 fold_run(&mut state, true, &home)
15261 .await
15262 .expect("merged fold");
15263
15264 assert!(
15265 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15266 "the handed-over branch survives a fold"
15267 );
15268 }
15269
15270 #[tokio::test]
15274 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15275 let tmp = tempfile::tempdir().expect("tempdir");
15276 let repo = tmp.path().join("repo");
15277 std::fs::create_dir_all(&repo).unwrap();
15278 init_repo(&repo);
15279 let run = |args: &[&str]| {
15280 let out = std::process::Command::new("git")
15281 .quiet()
15282 .args(args)
15283 .current_dir(&repo)
15284 .output()
15285 .expect("spawn git");
15286 assert!(out.status.success(), "git {args:?}");
15287 };
15288 run(&["branch", "magi/x/A"]);
15289 run(&["checkout", "-q", "-b", "magi/x/B"]);
15290 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15291 run(&["add", "-A"]);
15292 run(&["commit", "-q", "-m", "work"]);
15293 run(&["checkout", "-q", "main"]);
15294
15295 let probe = RunState::new(
15299 repo.clone(),
15300 "main".to_owned(),
15301 "deadbeef".to_owned(),
15302 "task".to_owned(),
15303 Config::default(),
15304 );
15305 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15306 let bare = tmp.path().join("origin.git");
15307 let out = std::process::Command::new("git")
15308 .quiet()
15309 .args(["init", "-q", "--bare"])
15310 .arg(&bare)
15311 .output()
15312 .expect("spawn git");
15313 assert!(out.status.success(), "git init --bare");
15314 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15315 run(&["push", "-q", "origin", "main"]);
15316
15317 let mut state = RunState::new(
15318 repo.clone(),
15319 "main".to_owned(),
15320 "deadbeef".to_owned(),
15321 "task".to_owned(),
15322 Config::default(),
15323 );
15324 state.seeds = vec![refs::Seed {
15325 token: "magi/27b2/A".to_owned(),
15326 kind: refs::SeedKind::Unresolved,
15327 sha: String::new(),
15328 branch: true,
15329 detail: "no branch or commit named magi/27b2/A".to_owned(),
15330 }];
15331
15332 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15333 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15334 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15335 let detail = empty_candidate_detail(&state, "main");
15336 assert!(detail.starts_with("empty candidate"), "{detail}");
15337 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15338 assert!(detail.contains("--start-from"), "{detail}");
15339 }
15340
15341 #[tokio::test]
15350 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15351 let tmp = tempfile::tempdir().expect("tempdir");
15352 let repo = tmp.path().join("repo");
15353 std::fs::create_dir_all(&repo).unwrap();
15354 init_repo(&repo);
15355
15356 let mut config = Config::default();
15357 config.merge.mode = MergeMode::Local;
15358
15359 let mut state = RunState::new(
15360 repo.clone(),
15361 "main".to_owned(),
15362 "deadbeef".to_owned(),
15363 "task".to_owned(),
15364 config,
15365 );
15366 state.candidates = vec![Candidate {
15367 index: 0,
15368 label: 'A',
15369 agent: "alpha".to_owned(),
15370 branch: "does-not-exist".to_owned(),
15371 worktree: repo.clone(),
15372 summary: String::new(),
15373 stat: String::new(),
15374 files: 0,
15375 commits: 0,
15376 empty: false,
15377 failed: None,
15378 verified_noop: None,
15379 duration_ms: 0,
15380 folded: false,
15381 }];
15382 state.tally = Some(Tally {
15383 first_choice: BTreeMap::from([('A', 1)]),
15384 borda: BTreeMap::new(),
15385 winner: 'A',
15386 rankings: 1,
15387 unanimous_initial: true,
15388 deliberated: false,
15389 changed_votes: 0,
15390 unanimous_final: true,
15391 tie_break: None,
15392 judges: 0,
15393 present: 0,
15394 quorum: 0,
15395 met_quorum: true,
15396 uncontested: Some("only candidate A produced a change".to_owned()),
15397 });
15398 state.reviews = vec![ReviewRound {
15399 round: 1,
15400 head: "deadbeef".to_owned(),
15401 verified_head: None,
15402 verified_at: None,
15403 reviews: Vec::new(),
15404 e2e: Vec::new(),
15405 fix: None,
15406 blocking: 0,
15407 answered: 0,
15408 expected: 0,
15409 clean: true,
15410 verify_retried: false,
15411 e2e_deferred: false,
15412 e2e_defer_reason: None,
15413 progressed: false,
15414 vote_split: false,
15415 reconsideration: Vec::new(),
15416 verdict: None,
15417 }];
15418 state.gate = vec![CommandOutcome {
15419 command: "test".to_owned(),
15420 code: Some(0),
15421 output_tail: String::new(),
15422 duration_ms: 0,
15423 resource_blocked: false,
15424 }];
15425 state.gate_ran = true;
15426 state.status = RunStatus::Ready;
15431 state.merge = Some(MergeOutcome {
15432 mode: MergeMode::Local,
15433 ok: false,
15434 detail: "already concluded".to_owned(),
15435 empty: false,
15436 });
15437
15438 let mut runner = Runner {
15439 state,
15440 roles: ResolvedRoles {
15441 implementers: Vec::new(),
15442 judges: Vec::new(),
15443 reviewers: Vec::new(),
15444 fixer: None,
15445 conductor: conductor(),
15446 implementer_roster: Vec::new(),
15447 judge_roster: Vec::new(),
15448 reviewer_roster: Vec::new(),
15449 },
15450 sem: Arc::new(Semaphore::new(1)),
15451 pause: Pause::new(),
15452 interrupt: Pause::new(),
15453 };
15454
15455 runner.merge().await.expect("merge");
15456
15457 assert_eq!(
15458 runner.state.status,
15459 RunStatus::Ready,
15460 "a concluded run's status must not change on reentry"
15461 );
15462 assert_eq!(
15463 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15464 Some("already concluded"),
15465 "merge must not run again once the node already recorded an outcome"
15466 );
15467 }
15468
15469 #[tokio::test]
15478 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15479 let tmp = tempfile::tempdir().expect("tempdir");
15480 let repo = tmp.path().join("repo");
15481 std::fs::create_dir_all(&repo).unwrap();
15482 init_repo(&repo);
15483
15484 let mut config = Config::default();
15485 config.merge.mode = MergeMode::Local;
15486
15487 let mut state = RunState::new(
15488 repo.clone(),
15489 "main".to_owned(),
15490 "deadbeef".to_owned(),
15491 "task".to_owned(),
15492 config,
15493 );
15494 state.candidates = vec![Candidate {
15495 index: 0,
15496 label: 'A',
15497 agent: "alpha".to_owned(),
15498 branch: "does-not-exist".to_owned(),
15499 worktree: repo.clone(),
15500 summary: String::new(),
15501 stat: String::new(),
15502 files: 0,
15503 commits: 0,
15504 empty: false,
15505 failed: None,
15506 verified_noop: None,
15507 duration_ms: 0,
15508 folded: false,
15509 }];
15510 state.tally = Some(Tally {
15511 first_choice: BTreeMap::from([('A', 1)]),
15512 borda: BTreeMap::new(),
15513 winner: 'A',
15514 rankings: 1,
15515 unanimous_initial: true,
15516 deliberated: false,
15517 changed_votes: 0,
15518 unanimous_final: true,
15519 tie_break: None,
15520 judges: 0,
15521 present: 0,
15522 quorum: 0,
15523 met_quorum: true,
15524 uncontested: Some("only candidate A produced a change".to_owned()),
15525 });
15526 state.reviews = vec![ReviewRound {
15527 round: 1,
15528 head: "deadbeef".to_owned(),
15529 verified_head: None,
15530 verified_at: None,
15531 reviews: Vec::new(),
15532 e2e: Vec::new(),
15533 fix: None,
15534 blocking: 0,
15535 answered: 0,
15536 expected: 0,
15537 clean: true,
15538 verify_retried: false,
15539 e2e_deferred: false,
15540 e2e_defer_reason: None,
15541 progressed: false,
15542 vote_split: false,
15543 reconsideration: Vec::new(),
15544 verdict: None,
15545 }];
15546 state.gate = Vec::new();
15548 state.gate_ran = false;
15549 state.status = RunStatus::Gating;
15550
15551 let mut runner = Runner {
15552 state,
15553 roles: ResolvedRoles {
15554 implementers: Vec::new(),
15555 judges: Vec::new(),
15556 reviewers: Vec::new(),
15557 fixer: None,
15558 conductor: conductor(),
15559 implementer_roster: Vec::new(),
15560 judge_roster: Vec::new(),
15561 reviewer_roster: Vec::new(),
15562 },
15563 sem: Arc::new(Semaphore::new(1)),
15564 pause: Pause::new(),
15565 interrupt: Pause::new(),
15566 };
15567
15568 runner.merge().await.expect("merge");
15569
15570 assert!(
15571 runner.state.merge.is_none(),
15572 "an empty gate must never be read as a passing one: {:?}",
15573 runner.state.merge
15574 );
15575 }
15576
15577 #[tokio::test]
15584 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15585 let tmp = tempfile::tempdir().expect("tempdir");
15586 let repo = tmp.path().join("repo");
15587 std::fs::create_dir_all(&repo).unwrap();
15588 init_repo(&repo);
15589
15590 let config = Config::default();
15592
15593 let mut state = RunState::new(
15594 repo.clone(),
15595 "main".to_owned(),
15596 "deadbeef".to_owned(),
15597 "task".to_owned(),
15598 config,
15599 );
15600 state.candidates = vec![Candidate {
15601 index: 0,
15602 label: 'A',
15603 agent: "alpha".to_owned(),
15604 branch: "does-not-exist".to_owned(),
15605 worktree: repo.clone(),
15606 summary: String::new(),
15607 stat: String::new(),
15608 files: 0,
15609 commits: 0,
15610 empty: false,
15611 failed: None,
15612 verified_noop: None,
15613 duration_ms: 0,
15614 folded: false,
15615 }];
15616 state.tally = Some(Tally {
15617 first_choice: BTreeMap::from([('A', 1)]),
15618 borda: BTreeMap::new(),
15619 winner: 'A',
15620 rankings: 1,
15621 unanimous_initial: true,
15622 deliberated: false,
15623 changed_votes: 0,
15624 unanimous_final: true,
15625 tie_break: None,
15626 judges: 0,
15627 present: 0,
15628 quorum: 0,
15629 met_quorum: true,
15630 uncontested: Some("only candidate A produced a change".to_owned()),
15631 });
15632 state.reviews = vec![ReviewRound {
15633 round: 1,
15634 head: "deadbeef".to_owned(),
15635 verified_head: None,
15636 verified_at: None,
15637 reviews: Vec::new(),
15638 e2e: Vec::new(),
15639 fix: None,
15640 blocking: 0,
15641 answered: 0,
15642 expected: 0,
15643 clean: true,
15644 verify_retried: false,
15645 e2e_deferred: false,
15646 e2e_defer_reason: None,
15647 progressed: false,
15648 vote_split: false,
15649 reconsideration: Vec::new(),
15650 verdict: None,
15651 }];
15652
15653 let mut runner = Runner {
15654 state,
15655 roles: ResolvedRoles {
15656 implementers: Vec::new(),
15657 judges: Vec::new(),
15658 reviewers: Vec::new(),
15659 fixer: None,
15660 conductor: conductor(),
15661 implementer_roster: Vec::new(),
15662 judge_roster: Vec::new(),
15663 reviewer_roster: Vec::new(),
15664 },
15665 sem: Arc::new(Semaphore::new(1)),
15666 pause: Pause::new(),
15667 interrupt: Pause::new(),
15668 };
15669
15670 runner.gate().await.expect("gate");
15671 assert!(
15672 runner.state.gate_ran,
15673 "zero configured commands is still a real attempt, not an unrun gate"
15674 );
15675 assert!(runner.state.gate.is_empty());
15676 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15677 assert_ne!(
15678 runner.state.status,
15679 RunStatus::Blocked,
15680 "a gate with nothing to check must not read as failed"
15681 );
15682
15683 runner.merge().await.expect("merge");
15684 assert_eq!(
15685 runner.state.status,
15686 RunStatus::Ready,
15687 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15688 );
15689 }
15690
15691 #[tokio::test]
15702 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15703 crate::run::pin_test_home();
15704 let home = crate::run::home();
15705
15706 let tmp = tempfile::tempdir().expect("tempdir");
15707 let repo = tmp.path().join("repo");
15708 std::fs::create_dir_all(&repo).unwrap();
15709 init_repo(&repo);
15710 let cache_dir = tmp.path().join("target");
15713
15714 let mut config = Config::default();
15715 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15716 config.graph.timeout_verify = Some(2);
15719 config.graph.timeout_resource_wait = 2;
15720
15721 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15722 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15723 .expect("no io error acquiring directly")
15724 {
15725 crate::cache::AcquireOutcome::Acquired(g) => g,
15726 crate::cache::AcquireOutcome::Busy(b) => {
15727 panic!("expected the direct acquire to win the lease first: {b:?}")
15728 }
15729 };
15730
15731 let mut state = RunState::new(
15732 repo.clone(),
15733 "main".to_owned(),
15734 "deadbeef".to_owned(),
15735 "task".to_owned(),
15736 config,
15737 );
15738 state.candidates = vec![Candidate {
15739 index: 0,
15740 label: 'A',
15741 agent: "alpha".to_owned(),
15742 branch: "does-not-exist".to_owned(),
15743 worktree: repo.clone(),
15744 summary: String::new(),
15745 stat: String::new(),
15746 files: 0,
15747 commits: 0,
15748 empty: false,
15749 failed: None,
15750 verified_noop: None,
15751 duration_ms: 0,
15752 folded: false,
15753 }];
15754 state.tally = Some(Tally {
15755 first_choice: BTreeMap::from([('A', 1)]),
15756 borda: BTreeMap::new(),
15757 winner: 'A',
15758 rankings: 1,
15759 unanimous_initial: true,
15760 deliberated: false,
15761 changed_votes: 0,
15762 unanimous_final: true,
15763 tie_break: None,
15764 judges: 0,
15765 present: 0,
15766 quorum: 0,
15767 met_quorum: true,
15768 uncontested: Some("only candidate A produced a change".to_owned()),
15769 });
15770 state.reviews = vec![ReviewRound {
15771 round: 1,
15772 head: "deadbeef".to_owned(),
15773 verified_head: None,
15774 verified_at: None,
15775 reviews: Vec::new(),
15776 e2e: Vec::new(),
15777 fix: None,
15778 blocking: 0,
15779 answered: 0,
15780 expected: 0,
15781 clean: true,
15782 verify_retried: false,
15783 e2e_deferred: false,
15784 e2e_defer_reason: None,
15785 progressed: false,
15786 vote_split: false,
15787 reconsideration: Vec::new(),
15788 verdict: None,
15789 }];
15790
15791 let mut runner = Runner {
15792 state,
15793 roles: ResolvedRoles {
15794 implementers: Vec::new(),
15795 judges: Vec::new(),
15796 reviewers: Vec::new(),
15797 fixer: None,
15798 conductor: conductor(),
15799 implementer_roster: Vec::new(),
15800 judge_roster: Vec::new(),
15801 reviewer_roster: Vec::new(),
15802 },
15803 sem: Arc::new(Semaphore::new(1)),
15804 pause: Pause::new(),
15805 interrupt: Pause::new(),
15806 };
15807
15808 let started = std::time::Instant::now();
15809 runner.gate().await.expect("gate");
15810 assert!(
15811 started.elapsed() < Duration::from_secs(1),
15812 "a gate with nothing to run must never wait on a lease it never needed"
15813 );
15814 assert!(
15815 runner.state.gate_ran,
15816 "zero commands is still a real, immediate attempt"
15817 );
15818 assert!(runner.state.gate.is_empty());
15819 assert_ne!(
15820 runner.state.status,
15821 RunStatus::Blocked,
15822 "must not read as resource-blocked on a lease it never asked for"
15823 );
15824 }
15825
15826 #[cfg(unix)]
15827 #[tokio::test]
15828 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15829 crate::run::pin_test_home();
15830 let tmp = tempfile::tempdir().unwrap();
15831 let mut config = Config::default();
15832 config.disk.min_free_bytes = 0;
15833 let mut state = RunState::new(
15834 tmp.path().to_owned(),
15835 "main".into(),
15836 "base".into(),
15837 "verification evidence".into(),
15838 config,
15839 );
15840 let (outcomes, pids) = run_commands(
15841 &mut state,
15842 "gate",
15843 "gate",
15844 1,
15845 &["sh".into(), "-c".into()],
15846 &[
15847 "printf before-timeout; exec sleep 30".into(),
15848 "kill -TERM $$".into(),
15849 "exit 7".into(),
15850 "printf success".into(),
15851 ],
15852 tmp.path(),
15853 Duration::from_secs(1),
15854 )
15855 .await;
15856 wait_for_timed_out_children_to_die(&pids).await;
15857 assert_eq!(
15858 pids.len(),
15859 1,
15860 "unleased timeout retains its death-check PID"
15861 );
15862 assert_eq!(outcomes.len(), 4);
15863 assert_eq!(outcomes[0].code, None);
15864 assert!(outcomes[0].output_tail.contains("before-timeout"));
15865 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15866 assert_eq!(outcomes[1].code, None);
15867 assert!(outcomes[1].output_tail.contains("command terminated:"));
15868 assert!(!outcomes[1].output_tail.contains("timed out"));
15869 assert_eq!(outcomes[2].code, Some(7));
15870 assert!(!outcomes[2].output_tail.contains("timed out"));
15871 assert_eq!(outcomes[3].code, Some(0));
15872 assert_eq!(outcomes[3].output_tail, "success");
15873 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15874 }
15875
15876 #[tokio::test]
15886 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15887 crate::run::pin_test_home();
15888
15889 let tmp = tempfile::tempdir().expect("tempdir");
15890 let repo = tmp.path().join("repo");
15891 std::fs::create_dir_all(&repo).unwrap();
15892 init_repo(&repo);
15893
15894 let mut config = Config::default();
15895 config.verify.gate = vec![
15896 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15897 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15898 .to_owned(),
15899 ];
15900
15901 let mut state = RunState::new(
15902 repo.clone(),
15903 "main".to_owned(),
15904 "deadbeef".to_owned(),
15905 "task".to_owned(),
15906 config,
15907 );
15908 let run_id = state.id.clone();
15909 state.candidates = vec![Candidate {
15910 index: 0,
15911 label: 'A',
15912 agent: "alpha".to_owned(),
15913 branch: "does-not-exist".to_owned(),
15914 worktree: repo.clone(),
15915 summary: String::new(),
15916 stat: String::new(),
15917 files: 0,
15918 commits: 0,
15919 empty: false,
15920 failed: None,
15921 verified_noop: None,
15922 duration_ms: 0,
15923 folded: false,
15924 }];
15925 state.tally = Some(Tally {
15926 first_choice: BTreeMap::from([('A', 1)]),
15927 borda: BTreeMap::new(),
15928 winner: 'A',
15929 rankings: 1,
15930 unanimous_initial: true,
15931 deliberated: false,
15932 changed_votes: 0,
15933 unanimous_final: true,
15934 tie_break: None,
15935 judges: 0,
15936 present: 0,
15937 quorum: 0,
15938 met_quorum: true,
15939 uncontested: Some("only candidate A produced a change".to_owned()),
15940 });
15941 state.reviews = vec![ReviewRound {
15942 round: 1,
15943 head: "deadbeef".to_owned(),
15944 verified_head: None,
15945 verified_at: None,
15946 reviews: Vec::new(),
15947 e2e: Vec::new(),
15948 fix: None,
15949 blocking: 0,
15950 answered: 0,
15951 expected: 0,
15952 clean: true,
15953 verify_retried: false,
15954 e2e_deferred: false,
15955 e2e_defer_reason: None,
15956 progressed: false,
15957 vote_split: false,
15958 reconsideration: Vec::new(),
15959 verdict: None,
15960 }];
15961
15962 let mut runner = Runner {
15963 state,
15964 roles: ResolvedRoles {
15965 implementers: Vec::new(),
15966 judges: Vec::new(),
15967 reviewers: Vec::new(),
15968 fixer: None,
15969 conductor: conductor(),
15970 implementer_roster: Vec::new(),
15971 judge_roster: Vec::new(),
15972 reviewer_roster: Vec::new(),
15973 },
15974 sem: Arc::new(Semaphore::new(1)),
15975 pause: Pause::new(),
15976 interrupt: Pause::new(),
15977 };
15978
15979 let started_marker = repo.join("started.marker");
15980 let release_marker = repo.join("release.marker");
15981 let poller = tokio::spawn(async move {
15982 for _ in 0..100 {
15987 if started_marker.exists()
15988 && let Ok(s) = crate::run::RunState::load(&run_id)
15989 && let Some(a) = s.active.get("gate")
15990 {
15991 std::fs::write(&release_marker, b"go").expect("release marker");
15992 return Some(a.clone());
15993 }
15994 tokio::time::sleep(Duration::from_millis(50)).await;
15995 }
15996 None
15997 });
15998
15999 runner.gate().await.expect("gate");
16000 let captured = poller.await.expect("poller task");
16001 let captured = captured.expect(
16002 "the poller never saw a `gate` task entry in run.json while the command was \
16003 still blocked on its own release marker",
16004 );
16005
16006 assert_eq!(captured.task.as_deref(), Some("gate"));
16007 assert_eq!(captured.node, "gate");
16008 assert_eq!(captured.index, Some(1));
16009 assert_eq!(captured.total, Some(1));
16010 assert!(
16011 captured
16012 .command
16013 .as_deref()
16014 .is_some_and(|c| c.contains("started.marker")),
16015 "{captured:?}"
16016 );
16017
16018 assert!(
16019 runner.state.active.is_empty(),
16020 "the entry must be cleared once the command actually finished: {:?}",
16021 runner.state.active
16022 );
16023 assert!(runner.state.gate_ran);
16024 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16025 }
16026
16027 #[tokio::test]
16030 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16031 use crate::verdict::{Finding, ReviewVote, Severity};
16032 crate::run::pin_test_home();
16033 let tmp = tempfile::tempdir().expect("tempdir");
16034 let repo = tmp.path().join("repo");
16035 std::fs::create_dir_all(&repo).unwrap();
16036 init_repo(&repo);
16037
16038 for (severity, vote, expect) in [
16039 (Severity::Major, ReviewVote::Reject, true),
16040 (Severity::Minor, ReviewVote::Reject, false),
16041 (Severity::Major, ReviewVote::Approve, false),
16042 ] {
16043 let mut round = review_round(false, 1, 1, 1, false, true);
16044 round.reviews = vec![ReviewRecord {
16045 reviewer: 1,
16046 agent: "alpha".to_owned(),
16047 summary: String::new(),
16048 findings: vec![Finding {
16049 id: "R1-1-1".to_owned(),
16050 severity,
16051 file: None,
16052 line: None,
16053 title: "t".to_owned(),
16054 detail: String::new(),
16055 }],
16056 vote: Some(vote),
16057 failed: None,
16058 duration_ms: 0,
16059 attempts: 0,
16060 }];
16061 let mut state = RunState::new(
16062 repo.clone(),
16063 "main".to_owned(),
16064 "deadbeef".to_owned(),
16065 "task".to_owned(),
16066 Config::default(),
16067 );
16068 state.reviews = vec![round];
16069 let mut runner = Runner {
16070 state,
16071 roles: ResolvedRoles {
16072 implementers: Vec::new(),
16073 judges: Vec::new(),
16074 reviewers: Vec::new(),
16075 fixer: None,
16076 conductor: conductor(),
16077 implementer_roster: Vec::new(),
16078 judge_roster: Vec::new(),
16079 reviewer_roster: Vec::new(),
16080 },
16081 sem: Arc::new(Semaphore::new(1)),
16082 pause: Pause::new(),
16083 interrupt: Pause::new(),
16084 };
16085 let shell = runner.state.config.shell();
16086 runner
16087 .stop_reviewing("round budget spent", &shell, &repo)
16088 .await
16089 .expect("stop_reviewing");
16090 assert_eq!(runner.state.status, RunStatus::Gating);
16091 assert_eq!(
16092 runner.state.contested_handoff.is_some(),
16093 expect,
16094 "{severity:?} + {vote:?}"
16095 );
16096 }
16097 }
16098
16099 #[tokio::test]
16112 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16113 crate::run::pin_test_home();
16114 let home = crate::run::home();
16115
16116 let tmp = tempfile::tempdir().expect("tempdir");
16117 let repo = tmp.path().join("repo");
16118 std::fs::create_dir_all(&repo).unwrap();
16119 init_repo(&repo);
16120 let head = crate::git::rev_parse(&repo, "HEAD")
16121 .await
16122 .expect("rev-parse");
16123 let cache_dir = tmp.path().join("target");
16126
16127 let mut config = Config::default();
16128 config.verify.e2e = vec![format!(
16129 "CARGO_TARGET_DIR='{}' test -f README.md",
16130 cache_dir.display()
16131 )];
16132 config.graph.review_rounds = 1;
16133 config.graph.timeout_verify = Some(2);
16136 config.graph.timeout_resource_wait = 2;
16137
16138 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16139 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16140 .expect("no io error acquiring directly")
16141 {
16142 crate::cache::AcquireOutcome::Acquired(g) => g,
16143 crate::cache::AcquireOutcome::Busy(b) => {
16144 panic!("expected the direct acquire to win the lease first: {b:?}")
16145 }
16146 };
16147
16148 let mut state = RunState::new(
16149 repo.clone(),
16150 "main".to_owned(),
16151 head.clone(),
16152 "task".to_owned(),
16153 config,
16154 );
16155 state.candidates = vec![Candidate {
16156 index: 0,
16157 label: 'A',
16158 agent: "alpha".to_owned(),
16159 branch: "does-not-exist".to_owned(),
16160 worktree: repo.clone(),
16161 summary: String::new(),
16162 stat: String::new(),
16163 files: 0,
16164 commits: 0,
16165 empty: false,
16166 failed: None,
16167 verified_noop: None,
16168 duration_ms: 0,
16169 folded: false,
16170 }];
16171 state.tally = Some(Tally {
16172 first_choice: BTreeMap::from([('A', 1)]),
16173 borda: BTreeMap::new(),
16174 winner: 'A',
16175 rankings: 1,
16176 unanimous_initial: true,
16177 deliberated: false,
16178 changed_votes: 0,
16179 unanimous_final: true,
16180 tie_break: None,
16181 judges: 0,
16182 present: 0,
16183 quorum: 0,
16184 met_quorum: true,
16185 uncontested: Some("only candidate A produced a change".to_owned()),
16186 });
16187 state.reviews = vec![ReviewRound {
16191 round: 1,
16192 head: head.clone(),
16193 verified_head: None,
16194 verified_at: None,
16195 reviews: Vec::new(),
16196 e2e: Vec::new(),
16197 fix: None,
16198 blocking: 1,
16199 answered: 1,
16200 expected: 1,
16201 clean: false,
16202 verify_retried: false,
16203 e2e_deferred: true,
16204 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16205 progressed: false,
16206 vote_split: false,
16207 reconsideration: Vec::new(),
16208 verdict: None,
16209 }];
16210
16211 let mut runner = Runner {
16212 state,
16213 roles: ResolvedRoles {
16214 implementers: Vec::new(),
16215 judges: Vec::new(),
16216 reviewers: Vec::new(),
16217 fixer: None,
16218 conductor: conductor(),
16219 implementer_roster: Vec::new(),
16220 judge_roster: Vec::new(),
16221 reviewer_roster: Vec::new(),
16222 },
16223 sem: Arc::new(Semaphore::new(1)),
16224 pause: Pause::new(),
16225 interrupt: Pause::new(),
16226 };
16227
16228 let shell = runner.state.config.shell();
16229 runner
16230 .stop_reviewing("round budget spent", &shell, &repo)
16231 .await
16232 .expect("stop_reviewing");
16233
16234 let last = runner.state.reviews.last().expect("round record");
16235 assert_eq!(
16236 last.e2e_status(),
16237 E2eStatus::ResourceBlocked,
16238 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16239 failed: {last:?}"
16240 );
16241 assert_eq!(
16242 last.verified_head.as_deref(),
16243 Some(head.as_str()),
16244 "which commit this attempt targeted is known even though nothing finished checking \
16245 it"
16246 );
16247 let first_attempt_at = last
16248 .verified_at
16249 .expect("when this attempt ran is known too");
16250 assert_ne!(
16251 runner.state.status,
16252 RunStatus::Blocked,
16253 "contention is evidence about the machine, not the patch — it must not settle the \
16254 run as blocked: {:?}",
16255 runner.state.status
16256 );
16257 assert!(
16258 !runner
16259 .state
16260 .events
16261 .iter()
16262 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16263 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16264 runner.state.events
16265 );
16266
16267 runner
16272 .stop_reviewing("round budget spent", &shell, &repo)
16273 .await
16274 .expect("stop_reviewing retry");
16275 assert_eq!(
16276 runner.state.reviews.len(),
16277 1,
16278 "no new round was started: {:?}",
16279 runner.state.reviews
16280 );
16281 let last = runner.state.reviews.last().expect("round record");
16282 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16283 assert!(
16284 last.verified_at.expect("still known") > first_attempt_at,
16285 "a second reentry must be a fresh attempt, not a stale copy of the first"
16286 );
16287 assert_ne!(runner.state.status, RunStatus::Blocked);
16288
16289 held.release();
16290 }
16291
16292 #[tokio::test]
16304 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16305 crate::run::pin_test_home();
16306 let home = crate::run::home();
16307
16308 let tmp = tempfile::tempdir().expect("tempdir");
16309 let repo = tmp.path().join("repo");
16310 std::fs::create_dir_all(&repo).unwrap();
16311 init_repo(&repo);
16312 let head = crate::git::rev_parse(&repo, "HEAD")
16313 .await
16314 .expect("rev-parse");
16315 let cache_dir = tmp.path().join("target");
16316
16317 let mut config = Config::default();
16318 config.verify.e2e = vec![format!(
16319 "CARGO_TARGET_DIR='{}' test -f README.md",
16320 cache_dir.display()
16321 )];
16322 config.graph.review_rounds = 1;
16323 config.graph.timeout_verify = Some(2);
16324 config.graph.timeout_resource_wait = 2;
16325
16326 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16327 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16328 .expect("no io error acquiring directly")
16329 {
16330 crate::cache::AcquireOutcome::Acquired(g) => g,
16331 crate::cache::AcquireOutcome::Busy(b) => {
16332 panic!("expected the direct acquire to win the lease first: {b:?}")
16333 }
16334 };
16335
16336 let mut state = RunState::new(
16337 repo.clone(),
16338 "main".to_owned(),
16339 head.clone(),
16340 "task".to_owned(),
16341 config,
16342 );
16343 state.candidates = vec![Candidate {
16344 index: 0,
16345 label: 'A',
16346 agent: "alpha".to_owned(),
16347 branch: "does-not-exist".to_owned(),
16348 worktree: repo.clone(),
16349 summary: String::new(),
16350 stat: String::new(),
16351 files: 0,
16352 commits: 0,
16353 empty: false,
16354 failed: None,
16355 verified_noop: None,
16356 duration_ms: 0,
16357 folded: false,
16358 }];
16359 state.tally = Some(Tally {
16360 first_choice: BTreeMap::from([('A', 1)]),
16361 borda: BTreeMap::new(),
16362 winner: 'A',
16363 rankings: 1,
16364 unanimous_initial: true,
16365 deliberated: false,
16366 changed_votes: 0,
16367 unanimous_final: true,
16368 tie_break: None,
16369 judges: 0,
16370 present: 0,
16371 quorum: 0,
16372 met_quorum: true,
16373 uncontested: Some("only candidate A produced a change".to_owned()),
16374 });
16375 state.reviews = vec![ReviewRound {
16379 round: 1,
16380 head: head.clone(),
16381 verified_head: Some(head.clone()),
16382 verified_at: Some(jiff::Timestamp::now()),
16383 reviews: Vec::new(),
16384 e2e: vec![CommandOutcome {
16385 command: format!(
16386 "CARGO_TARGET_DIR='{}' test -f README.md",
16387 cache_dir.display()
16388 ),
16389 code: None,
16390 output_tail: "waiting for the shared build cache".to_owned(),
16391 duration_ms: 0,
16392 resource_blocked: true,
16393 }],
16394 fix: None,
16395 blocking: 1,
16396 answered: 1,
16397 expected: 1,
16398 clean: false,
16399 verify_retried: false,
16400 e2e_deferred: false,
16401 e2e_defer_reason: None,
16402 progressed: false,
16403 vote_split: false,
16404 reconsideration: Vec::new(),
16405 verdict: None,
16406 }];
16407
16408 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16409 let mut runner = Runner {
16410 state,
16411 roles: ResolvedRoles {
16412 implementers: Vec::new(),
16413 judges: Vec::new(),
16414 reviewers: Vec::new(),
16415 fixer: None,
16416 conductor: conductor(),
16417 implementer_roster: Vec::new(),
16418 judge_roster: Vec::new(),
16419 reviewer_roster: Vec::new(),
16420 },
16421 sem: Arc::new(Semaphore::new(1)),
16422 pause: Pause::new(),
16423 interrupt: Pause::new(),
16424 };
16425
16426 runner.review_loop().await.expect("review_loop");
16431
16432 assert_eq!(
16433 runner.state.reviews.len(),
16434 1,
16435 "no new round was started on top of the unresolved one: {:?}",
16436 runner.state.reviews
16437 );
16438 let last = &runner.state.reviews[0];
16439 assert_eq!(
16440 last.e2e_status(),
16441 E2eStatus::ResourceBlocked,
16442 "still contended: {last:?}"
16443 );
16444 assert!(
16445 last.verified_at.expect("still known") > first_attempt_at,
16446 "review_loop must have actually retried the check, not left it exactly as found"
16447 );
16448 assert_ne!(
16449 runner.state.status,
16450 RunStatus::Blocked,
16451 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16452 runner.state.status
16453 );
16454
16455 held.release();
16456 }
16457
16458 #[tokio::test]
16459 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16460 crate::run::pin_test_home();
16461 for status in [RunStatus::Blocked, RunStatus::Gating] {
16462 let mut runner = runner_at(status);
16463 runner.state.config.graph.land = true;
16464 runner.state.gate_ran = true;
16465 runner.state.merge = Some(MergeOutcome {
16466 mode: MergeMode::Pr,
16467 ok: false,
16468 detail: "the fixer produced no commit".into(),
16469 empty: false,
16470 });
16471 let url = "https://example.invalid/o/r/pull/1";
16472 runner.state.pr = Some(crate::run::PrRecord {
16473 url: url.into(),
16474 number: 1,
16475 state: "open".into(),
16476 checks: "red".into(),
16477 round: 1,
16478 rounds: 1,
16479 red_at_merge: Vec::new(),
16480 });
16481 runner.state.save().unwrap();
16482 runner.execute_graph().await.unwrap();
16486 assert_eq!(runner.state.status, RunStatus::Blocked);
16487 assert!(runner.state.land_sync_pending);
16488 assert!(runner.state.events.iter().any(|event|
16489 event.node == "land" && event.message == format!("watching {url}")));
16490 assert_eq!(
16491 runner.state.merge.as_ref().unwrap().detail,
16492 "the fixer produced no commit"
16493 );
16494 }
16495 }
16496
16497 #[tokio::test]
16498 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16499 crate::run::pin_test_home();
16500 let tmp = tempfile::tempdir().expect("tempdir");
16501 let repo = tmp.path().join("repo");
16502 std::fs::create_dir_all(&repo).unwrap();
16503 init_repo(&repo);
16504
16505 let mut config = Config::default();
16506 config.merge.mode = MergeMode::Pr;
16507 config.graph.land = true;
16508 config.graph.land_approval = false;
16509
16510 let mut state = RunState::new(
16511 repo.clone(),
16512 "main".to_owned(),
16513 "deadbeef".to_owned(),
16514 "task".to_owned(),
16515 config,
16516 );
16517 state.candidates = vec![Candidate {
16518 index: 0,
16519 label: 'A',
16520 agent: "alpha".to_owned(),
16521 branch: "does-not-exist".to_owned(),
16522 worktree: repo.clone(),
16523 summary: String::new(),
16524 stat: String::new(),
16525 files: 0,
16526 commits: 0,
16527 empty: false,
16528 failed: None,
16529 verified_noop: None,
16530 duration_ms: 0,
16531 folded: false,
16532 }];
16533 state.tally = Some(Tally {
16534 first_choice: BTreeMap::from([('A', 1)]),
16535 borda: BTreeMap::new(),
16536 winner: 'A',
16537 rankings: 1,
16538 unanimous_initial: true,
16539 deliberated: false,
16540 changed_votes: 0,
16541 unanimous_final: true,
16542 tie_break: None,
16543 judges: 0,
16544 present: 0,
16545 quorum: 0,
16546 met_quorum: true,
16547 uncontested: Some("only candidate A produced a change".to_owned()),
16548 });
16549 state.reviews = vec![ReviewRound {
16550 round: 1,
16551 head: "deadbeef".to_owned(),
16552 verified_head: None,
16553 verified_at: None,
16554 reviews: Vec::new(),
16555 e2e: Vec::new(),
16556 fix: None,
16557 blocking: 0,
16558 answered: 0,
16559 expected: 0,
16560 clean: true,
16561 verify_retried: false,
16562 e2e_deferred: false,
16563 e2e_defer_reason: None,
16564 progressed: false,
16565 vote_split: false,
16566 reconsideration: Vec::new(),
16567 verdict: None,
16568 }];
16569 state.gate = vec![CommandOutcome {
16570 command: "test".to_owned(),
16571 code: Some(0),
16572 output_tail: String::new(),
16573 duration_ms: 0,
16574 resource_blocked: false,
16575 }];
16576 state.gate_ran = true;
16577 state.status = RunStatus::Landing;
16581 state.merge = Some(MergeOutcome {
16582 mode: MergeMode::Pr,
16583 ok: true,
16584 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16585 empty: false,
16586 });
16587
16588 ask_test_home();
16592 state.save().expect("save landing run");
16595 let store = ask::Questions::open();
16596 let q = ask_open_question(&store, &state.id);
16597
16598 let mut runner = Runner {
16599 state,
16600 roles: ResolvedRoles {
16601 implementers: Vec::new(),
16602 judges: Vec::new(),
16603 reviewers: Vec::new(),
16604 fixer: None,
16605 conductor: conductor(),
16606 implementer_roster: Vec::new(),
16607 judge_roster: Vec::new(),
16608 reviewer_roster: Vec::new(),
16609 },
16610 sem: Arc::new(Semaphore::new(1)),
16611 pause: Pause::new(),
16612 interrupt: Pause::new(),
16613 };
16614
16615 runner.execute().await.expect("execute");
16620
16621 assert_eq!(
16622 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16623 Some("https://example.invalid/x/y/pull/1"),
16624 "reentry must not push again or open a second pull request over the \
16625 one `land` is already watching"
16626 );
16627 assert_ne!(
16628 runner.state.status,
16629 RunStatus::Landing,
16630 "land could not actually reach the fake pull request, so it must \
16631 have given up rather than left the run silently parked forever"
16632 );
16633 assert_eq!(runner.state.status, RunStatus::Blocked);
16637 assert!(
16638 store.get(&q.id).unwrap().status.open(),
16639 "Blocked is still alive; settle_questions must have been a no-op here"
16640 );
16641 }
16642
16643 fn state_with_round(round: ReviewRound) -> RunState {
16644 let mut s = RunState::new(
16645 PathBuf::from("/repo"),
16646 "main".to_owned(),
16647 "abc1234".to_owned(),
16648 "add retries".to_owned(),
16649 Config::default(),
16650 );
16651 s.reviews = vec![round];
16652 s
16653 }
16654
16655 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16656 crate::verdict::Finding {
16657 id: id.to_owned(),
16658 severity,
16659 file: None,
16660 line: None,
16661 title: title.to_owned(),
16662 detail: String::new(),
16663 }
16664 }
16665
16666 #[test]
16667 fn pr_body_names_open_findings_and_declined_ones() {
16668 let round = ReviewRound {
16669 round: 2,
16670 head: "deadbee".to_owned(),
16671 verified_head: None,
16672 verified_at: None,
16673 reviews: vec![ReviewRecord {
16674 attempts: 0,
16675 reviewer: 1,
16676 agent: "alpha".to_owned(),
16677 summary: String::new(),
16678 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16679 vote: None,
16680 failed: None,
16681 duration_ms: 0,
16682 }],
16683 e2e: vec![CommandOutcome {
16684 command: "cargo test".to_owned(),
16685 code: Some(0),
16686 output_tail: String::new(),
16687 duration_ms: 0,
16688 resource_blocked: false,
16689 }],
16690 verify_retried: false,
16691 e2e_deferred: false,
16692 e2e_defer_reason: None,
16693 fix: Some(FixRecord {
16694 agent: "alpha".to_owned(),
16695 addressed: Vec::new(),
16696 rejected: vec![crate::verdict::Rejection {
16697 id: "R1-1-1".to_owned(),
16698 why: "not reachable from any caller".to_owned(),
16699 }],
16700 notes: String::new(),
16701 committed: true,
16702 failed: None,
16703 duration_ms: 0,
16704 continuation: None,
16705 }),
16706 blocking: 0,
16707 answered: 1,
16708 expected: 1,
16709 clean: false,
16710 progressed: true,
16711 vote_split: false,
16712 reconsideration: Vec::new(),
16713 verdict: None,
16714 };
16715 let state = state_with_round(round);
16716 let body = pr_message(&state, 'A').body;
16717
16718 assert!(body.contains("add retries"), "the task must still be there");
16719 assert!(body.contains("R2-1-1"), "{body}");
16720 assert!(!body.contains("unused import"), "{body}");
16721 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16722 assert!(
16723 !body.contains("not reachable from any caller"),
16724 "the reason it was declined: {body}"
16725 );
16726 }
16727
16728 #[test]
16729 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16730 let round = ReviewRound {
16731 round: 1,
16732 head: "deadbee".to_owned(),
16733 verified_head: None,
16734 verified_at: None,
16735 reviews: vec![ReviewRecord {
16736 attempts: 0,
16737 reviewer: 1,
16738 agent: "alpha".to_owned(),
16739 summary: String::new(),
16740 findings: Vec::new(),
16741 vote: None,
16742 failed: None,
16743 duration_ms: 0,
16744 }],
16745 e2e: Vec::new(),
16746 verify_retried: false,
16747 e2e_deferred: false,
16748 e2e_defer_reason: None,
16749 fix: None,
16750 blocking: 0,
16751 answered: 1,
16752 expected: 1,
16753 clean: true,
16754 progressed: false,
16755 vote_split: false,
16756 reconsideration: Vec::new(),
16757 verdict: None,
16758 };
16759 let state = state_with_round(round);
16760 let body = pr_message(&state, 'A').body;
16761 assert!(!body.contains("Open review findings"), "{body}");
16762 assert!(!body.contains("Declined"), "{body}");
16763 }
16764
16765 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16766 let mut state = RunState::new(
16767 PathBuf::from("/repo"),
16768 "main".to_owned(),
16769 "abc1234".to_owned(),
16770 instruction.to_owned(),
16771 Config::default(),
16772 );
16773 state.candidates.push(Candidate {
16774 index: 0,
16775 label: 'A',
16776 agent: "alpha".to_owned(),
16777 branch: "magi/x/A".to_owned(),
16778 worktree: PathBuf::from("/wt"),
16779 summary: summary.to_owned(),
16780 stat: String::new(),
16781 files: 1,
16782 commits: 1,
16783 empty: false,
16784 failed: None,
16785 verified_noop: None,
16786 folded: false,
16787 duration_ms: 0,
16788 });
16789 state
16790 }
16791
16792 fn review_state(subjects: &[&str]) -> RunState {
16793 let mut state = state_with_summary(
16794 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16795 "",
16796 );
16797 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16798 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16799 state
16800 }
16801
16802 #[tokio::test]
16805 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16806 ask_test_home();
16807 let tmp = tempfile::tempdir().unwrap();
16808 let repo = tmp.path().join("repo");
16809 std::fs::create_dir_all(&repo).unwrap();
16810 init_repo(&repo);
16811 let origin = tmp.path().join("origin.git");
16812 let g = |dir: &Path, args: &[&str]| {
16813 let out = std::process::Command::new("git")
16814 .args(args)
16815 .current_dir(dir)
16816 .quiet()
16817 .output()
16818 .expect("spawn git");
16819 assert!(
16820 out.status.success(),
16821 "git {args:?}: {}",
16822 String::from_utf8_lossy(&out.stderr)
16823 );
16824 };
16825 g(
16826 tmp.path(),
16827 &[
16828 "clone",
16829 "--bare",
16830 "-q",
16831 repo.to_str().unwrap(),
16832 origin.to_str().unwrap(),
16833 ],
16834 );
16835 g(
16836 &repo,
16837 &["remote", "add", "origin", origin.to_str().unwrap()],
16838 );
16839 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16840
16841 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16843 g(&repo, &["add", "-A"]);
16844 g(
16845 &repo,
16846 &["commit", "-q", "-m", "chore(deps): update a crate"],
16847 );
16848 g(&repo, &["push", "-q", "origin", "main"]);
16849 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16850 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16851 g(&repo, &["add", "-A"]);
16852 g(
16853 &repo,
16854 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16855 );
16856 g(&repo, &["checkout", "-q", "main"]);
16857
16858 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16859 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16860 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16862 assert_eq!(fallback, c1);
16863
16864 let mut state = review_state(&[
16865 "chore(deps): update a crate",
16866 "fix(daemon): apply a chosen action",
16867 ]);
16868 state.repo = repo.clone();
16869 state.base_branch = "main".to_owned();
16870 state.base_commit = c1;
16871 refresh_reviewed_commits(&mut state, "feat/own").await;
16872 assert_eq!(
16873 state.reviewed_commits,
16874 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16875 );
16876 assert_eq!(
16877 review_title(&state).as_deref(),
16878 Some("fix(daemon): apply a chosen action")
16879 );
16880 assert_eq!(
16881 leaked_subjects(&state, "feat/own").await,
16882 Some(vec!["chore(deps): update a crate".to_owned()])
16883 );
16884 state.config.merge.remote = "nowhere".to_owned();
16887 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16888 }
16889
16890 struct SummaryFixture {
16894 _tmp: tempfile::TempDir,
16895 repo: PathBuf,
16896 reply: PathBuf,
16897 runner: Runner,
16898 }
16899
16900 fn summary_fixture() -> SummaryFixture {
16901 ask_test_home();
16902 let tmp = tempfile::tempdir().unwrap();
16903 let repo = tmp.path().join("repo");
16904 std::fs::create_dir_all(&repo).unwrap();
16905 init_repo(&repo);
16906 let origin = tmp.path().join("origin.git");
16907 let g = |args: &[&str]| {
16908 let out = std::process::Command::new("git")
16909 .args(args)
16910 .current_dir(&repo)
16911 .quiet()
16912 .output()
16913 .expect("spawn git");
16914 assert!(
16915 out.status.success(),
16916 "git {args:?}: {}",
16917 String::from_utf8_lossy(&out.stderr)
16918 );
16919 };
16920 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16921 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16922 g(&["push", "-q", "origin", "main"]);
16923 g(&["checkout", "-q", "-b", "feat/own"]);
16924 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16925 g(&["add", "-A"]);
16926 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16927 let reply = tmp.path().join("reply.txt");
16928 std::fs::write(
16929 &reply,
16930 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16931 )
16932 .unwrap();
16933 let mut runner = runner_at(RunStatus::Gating);
16934 let mut state = review_state(&["feat(own): add the own file"]);
16935 state.repo = repo.clone();
16936 state.base_branch = "main".to_owned();
16937 state.candidates[0].branch = "feat/own".to_owned();
16938 state.candidates[0].worktree = repo.clone();
16939 let mut author = spec("alpha");
16940 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16941 author
16942 .env
16943 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16944 state.config.agents = vec![author];
16945 runner.state = state;
16946 SummaryFixture {
16947 _tmp: tmp,
16948 repo,
16949 reply,
16950 runner,
16951 }
16952 }
16953
16954 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16955 let winner = f.runner.state.candidates[0].clone();
16956 f.runner
16957 .guarded_pr_message(&winner, None, true)
16958 .await
16959 .unwrap()
16960 .expect("an English description never parks")
16961 }
16962
16963 fn clean_round_with_minor_findings() -> ReviewRound {
16964 let mut round = passing_round(2, None);
16965 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16966 round.reviews = vec![ReviewRecord {
16967 attempts: 0,
16968 reviewer: 1,
16969 agent: "alpha".into(),
16970 summary: "内部レビュー".into(),
16971 findings: vec![
16972 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16973 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16974 ],
16975 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16976 failed: None,
16977 duration_ms: 0,
16978 }];
16979 round
16980 }
16981
16982 #[tokio::test]
16983 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16984 for review_only in [false, true] {
16985 let mut f = summary_fixture();
16986 if !review_only {
16987 f.runner.state.candidates[0].agent = "alpha".into();
16988 }
16989 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16990 assert!(f.runner.state.reviews[0].clean);
16991 assert!(f.runner.state.reviews[0].fix.is_none());
16992 assert!(f.runner.state.open_findings().is_empty());
16993 assert_eq!(
16994 crate::pr_outcome::required_finding_ids(&f.runner.state),
16995 ["R2-1-1", "R2-1-2"]
16996 );
16997 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16998 let evidence = crate::pr_outcome::summary_evidence(
16999 &f.runner.state,
17000 crate::pr_outcome::FinalHead::At(&head),
17001 );
17002 assert!(
17003 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17004 );
17005 let omitted = posted(&mut f).await;
17007 assert!(omitted.body.contains("No description"));
17008 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17009 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17010 assert!(f.runner.state.change_summary.is_none());
17011 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17012 let described = posted(&mut f).await;
17013 assert!(described.body.contains("improve documentation for callers"));
17014 assert!(described.body.contains("remaining edge case"));
17015 assert!(described.body.contains("## Open review findings"));
17016 assert!(!described.body.contains("未修正"));
17017 }
17018 }
17019
17020 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17021 let mut f = summary_fixture();
17022 if !review_only {
17023 f.runner.state.candidates[0].agent = "alpha".into();
17024 }
17025 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17026 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17027 fixed
17028 .fix
17029 .as_mut()
17030 .unwrap()
17031 .rejected
17032 .push(crate::verdict::Rejection {
17033 id: "R1-1-1".into(),
17034 why: "呼び出されないコードのため辞退".into(),
17035 });
17036 let mut clean = clean_round_with_minor_findings();
17037 clean.verified_head = Some(head.clone());
17038 f.runner.state.reviews = vec![fixed, clean];
17039 ok_gate(&mut f.runner.state, Some(&head));
17040 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17041 f.runner.state.config.agents.push(scripted("beta", reply));
17042 f.runner.state.config.roles.language_judge =
17043 Some(crate::config::AgentChoice::One("beta".into()));
17044 f
17045 }
17046
17047 #[tokio::test]
17048 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17049 for review_only in [false, true] {
17050 let mut f = rewrite_fixture_with_findings(review_only,
17051 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17052 let m = posted(&mut f).await;
17053 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17054 let expected =
17055 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17056 assert!(m.body.contains(&expected));
17057 assert!(m.body.contains("## Review rounds"));
17058 assert!(
17059 m.body
17060 .contains("Gate (`verify.gate`): passed on the final commit")
17061 );
17062 assert!(
17063 m.body
17064 .contains("Tests (`verify.e2e`): passed on the final commit")
17065 );
17066 let c = f.runner.state.change_summary.as_ref().unwrap();
17067 assert!(
17068 m.body
17069 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17070 );
17071 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17072 assert!(crate::pr_outcome::covers_required_findings(
17073 &f.runner.state,
17074 &m.body
17075 ));
17076 assert!(m.body.contains("documentation remains incomplete"));
17077 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17078 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17079 assert!(f.runner.state.github_text.is_none());
17080 }
17081 }
17082
17083 #[tokio::test]
17084 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17085 for review_only in [false, true] {
17086 for reply in [
17087 "TITLE: fix: public description\nA short English description.",
17088 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17089 ] {
17090 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17091 let winner = f.runner.state.candidates[0].clone();
17092 let posted = f
17093 .runner
17094 .guarded_pr_message(&winner, None, true)
17095 .await
17096 .unwrap();
17097 assert!(posted.is_none());
17098 assert!(f.runner.state.parked);
17099 let withheld = f.runner.state.github_text.as_ref().unwrap();
17100 assert!(
17101 withheld.rewrite.is_none(),
17102 "an incomplete rewrite must not be selectable"
17103 );
17104 let q = ask::Questions::open()
17105 .get(withheld.question.as_ref().unwrap())
17106 .unwrap();
17107 assert!(
17108 !q.choices
17109 .contains(&crate::github_text::USE_REWRITE.to_owned())
17110 );
17111 assert!(
17112 q.detail.contains("R1-1-1")
17113 && q.detail.contains("R2-1-1")
17114 && q.detail.contains("R2-1-2")
17115 );
17116 }
17117 }
17118 }
17119
17120 #[tokio::test]
17121 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17122 for review_only in [false, true] {
17123 let mut f = summary_fixture();
17124 if !review_only {
17125 f.runner.state.candidates[0].agent = "alpha".into();
17126 }
17127 f.runner.state.instruction = "内部タスクの説明".into();
17128 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17129 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17130 f.runner.state.reviews = vec![
17131 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17132 round_with_notes(2, Some("二回目の内部修正報告です。")),
17133 passing_round(3, Some(&head)),
17134 ];
17135 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17136 ok_gate(&mut f.runner.state, Some(&head));
17137 let captured = f.repo.join("public-prompt.txt");
17138 let author = &mut f.runner.state.config.agents[0];
17139 author.command = vec![
17140 "sh".into(),
17141 "-c".into(),
17142 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17143 ];
17144 author.env.insert(
17145 "PROMPT_FILE".into(),
17146 captured.to_string_lossy().into_owned(),
17147 );
17148 let m = posted(&mut f).await;
17149 assert!(m.body.contains("Adds `own.txt`"));
17150 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17151 assert!(m.body.contains("R1-1-1"));
17152 assert!(
17153 m.body
17154 .contains("Gate (`verify.gate`): passed on the final commit")
17155 );
17156 assert!(
17157 m.body
17158 .contains("Tests (`verify.e2e`): passed on the final commit")
17159 );
17160 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17161 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17162 let prompt = std::fs::read_to_string(captured).unwrap();
17163 assert!(prompt.contains(&format!(
17164 "git diff {} {head}",
17165 f.runner.state.change_summary.as_ref().unwrap().base
17166 )));
17167 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17168 assert_eq!(
17169 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17170 "二回目の内部修正報告です。"
17171 );
17172 }
17173 }
17174
17175 #[tokio::test]
17176 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17177 for review_only in [false, true] {
17178 let mut f = summary_fixture();
17179 if !review_only {
17180 f.runner.state.candidates[0].agent = "alpha".into();
17181 }
17182 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17183 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17184 std::fs::write(&f.reply, "no usable description").unwrap();
17185 let failed = posted(&mut f).await;
17186 assert!(
17187 failed
17188 .body
17189 .contains("No description of what this change does could be generated")
17190 );
17191 assert!(failed.body.is_ascii());
17192 assert!(f.runner.state.github_text.is_none());
17193 assert!(f.runner.state.change_summary.is_none());
17194 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17195 let evidence = crate::pr_outcome::summary_evidence(
17196 &f.runner.state,
17197 crate::pr_outcome::FinalHead::At(&head),
17198 );
17199 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17200 assert!(
17201 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17202 .exists()
17203 );
17204 f.runner.state =
17205 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17206 std::fs::write(
17207 &f.reply,
17208 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17209 )
17210 .unwrap();
17211 let retried = posted(&mut f).await;
17212 assert!(retried.body.contains("Adds the committed own file."));
17213 assert_eq!(
17214 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17215 "日本語の修正報告"
17216 );
17217 }
17218 }
17219
17220 #[tokio::test]
17221 async fn same_head_verification_and_findings_invalidate_public_prose() {
17222 let mut f = summary_fixture();
17223 posted(&mut f).await;
17224 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17225 let original = f
17226 .runner
17227 .state
17228 .change_summary
17229 .as_ref()
17230 .unwrap()
17231 .evidence
17232 .clone();
17233 ok_gate(&mut f.runner.state, Some(&head));
17234 std::fs::write(
17235 &f.reply,
17236 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17237 )
17238 .unwrap();
17239 let verified = posted(&mut f).await;
17240 assert!(verified.body.contains("now verified"));
17241 assert_ne!(
17242 f.runner.state.change_summary.as_ref().unwrap().evidence,
17243 original
17244 );
17245 let mut round = round_with_notes(1, Some("内部報告"));
17246 round.reviews.push(ReviewRecord {
17247 attempts: 0,
17248 reviewer: 1,
17249 agent: "alpha".into(),
17250 summary: "内部レビュー".into(),
17251 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17252 vote: None,
17253 failed: None,
17254 duration_ms: 0,
17255 });
17256 round
17257 .fix
17258 .as_mut()
17259 .unwrap()
17260 .rejected
17261 .push(crate::verdict::Rejection {
17262 id: "R1-1-1".into(),
17263 why: "呼び出し元から到達できない".into(),
17264 });
17265 f.runner.state.reviews = vec![round];
17266 let omitted = posted(&mut f).await;
17268 assert!(omitted.body.contains("No description"));
17269 assert!(omitted.body.contains("R1-1-1"));
17270 assert!(!omitted.body.contains("到達"));
17271 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17272 let described = posted(&mut f).await;
17273 assert!(described.body.contains("unreachable from callers"));
17274 assert!(described.body.contains("R1-1-1"));
17275 assert!(!described.body.contains("到達"));
17276 let evidence = f
17277 .runner
17278 .state
17279 .change_summary
17280 .as_ref()
17281 .unwrap()
17282 .evidence
17283 .clone();
17284 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17285 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17286 assert!(posted(&mut f).await.body.contains("rationale updated"));
17287 assert_ne!(
17288 f.runner.state.change_summary.as_ref().unwrap().evidence,
17289 evidence
17290 );
17291 f.runner
17293 .state
17294 .change_summary
17295 .as_mut()
17296 .unwrap()
17297 .evidence
17298 .clear();
17299 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17300 assert!(posted(&mut f).await.body.contains("current specification"));
17301 }
17302
17303 #[tokio::test]
17304 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17305 let mut f = summary_fixture();
17306 let m = posted(&mut f).await;
17307 assert_eq!(m.title, "feat(own): add the own file");
17308 assert!(
17309 m.body
17310 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17311 "{}",
17312 m.body
17313 );
17314 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17315 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17316 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17317 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17318 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17319 let kept = f.runner.state.change_summary.as_ref().unwrap();
17320 assert_eq!(
17321 (kept.head.as_str(), kept.base.as_str()),
17322 (head.as_str(), base.as_str())
17323 );
17324 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17325 }
17326
17327 #[tokio::test]
17328 async fn review_only_summary_receives_the_configured_github_language() {
17329 let mut f = summary_fixture();
17330 f.runner.state.config.graph.github_language = "ja".into();
17331 let captured = f.repo.join("summary-prompt.txt");
17332 let author = &mut f.runner.state.config.agents[0];
17333 author.command = vec![
17334 "sh".into(),
17335 "-c".into(),
17336 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17337 ];
17338 author.env.insert(
17339 "PROMPT_FILE".into(),
17340 captured.to_string_lossy().into_owned(),
17341 );
17342 std::fs::write(
17343 &f.reply,
17344 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17345 )
17346 .unwrap();
17347 let m = posted(&mut f).await;
17348 let prompt = std::fs::read_to_string(captured).unwrap();
17349 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17350 assert!(!prompt.contains("in English"));
17351 assert_eq!(m.title, "feat(own): ファイルを追加");
17352 assert!(m.body.contains("own.txt を追加します。"));
17353 }
17354
17355 #[tokio::test]
17356 async fn review_fixes_come_after_the_whole_change_summary() {
17357 let mut f = summary_fixture();
17358 f.runner.state.reviews = vec![
17359 round_with_notes(1, Some("handled the PATH case")),
17360 round_with_notes(2, None),
17361 ];
17362 let m = posted(&mut f).await;
17363 let summary = m.body.find("## Summary").expect("summary");
17364 let fixes = m.body.find("## Review rounds").expect("review rounds");
17365 assert!(summary < fixes, "{}", m.body);
17366 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17367 assert!(m.body.contains("## Final verification"), "{}", m.body);
17368 }
17369
17370 #[tokio::test]
17371 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17372 let mut f = summary_fixture();
17373 let first = posted(&mut f).await;
17374 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17375 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17377 let again = posted(&mut f).await;
17378 assert_eq!((again.title, again.body), (first.title, first.body));
17379
17380 let g = |args: &[&str]| {
17381 let out = std::process::Command::new("git")
17382 .args(args)
17383 .current_dir(&f.repo)
17384 .quiet()
17385 .output()
17386 .expect("spawn git");
17387 assert!(out.status.success(), "git {args:?}");
17388 };
17389 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17390 g(&["add", "-A"]);
17391 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17392 let second = posted(&mut f).await;
17393 let kept = f.runner.state.change_summary.clone().unwrap();
17394 assert_ne!(kept.head, first_head);
17395 assert!(second.body.contains("Not read yet."), "{}", second.body);
17396 assert!(!second.body.contains("so the first summary is read"));
17397 assert_eq!(second.title, "feat(own): other");
17398 }
17399
17400 #[tokio::test]
17401 async fn an_unavailable_summary_is_stated_and_never_invented() {
17402 let mut f = summary_fixture();
17403 std::fs::write(&f.reply, "no title line here").unwrap();
17404 let m = posted(&mut f).await;
17405 assert!(
17406 m.body
17407 .contains("No description of what this change does could be generated"),
17408 "{}",
17409 m.body
17410 );
17411 assert!(m.body.contains("internal reports remain in local records"));
17412 assert!(f.runner.state.change_summary.is_none());
17413 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17414 let winner = f.runner.state.candidates[0].clone();
17416 let preview = f
17417 .runner
17418 .guarded_pr_message(&winner, None, false)
17419 .await
17420 .unwrap()
17421 .unwrap();
17422 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17423 }
17424
17425 #[test]
17426 fn pr_message_review_single_commit_uses_its_subject() {
17427 let state = review_state(&["feat(nats): per-role user"]);
17428 let m = pr_message(&state, 'A');
17429 assert_eq!(m.title, "feat(nats): per-role user");
17430 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17431 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17432 }
17433
17434 #[test]
17435 fn pr_message_review_multi_commit_takes_the_oldest() {
17436 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17437 let m = pr_message(&state, 'A');
17438 assert_eq!(m.title, "feat: the change");
17439 for s in ["feat: the change", "fix: typo", "fix: again"] {
17440 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17441 }
17442 }
17443
17444 #[test]
17445 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17446 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17447 let mut state = review_state(&[first, "fix: later fixup"]);
17448 state.candidates[0].branch = "機能/ログイン".to_owned();
17449 let m = pr_message(&state, 'A');
17450 assert!(
17451 m.title.starts_with("chore: land candidate A of run"),
17452 "{}",
17453 m.title
17454 );
17455 }
17456 }
17457
17458 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17459 BranchFacts {
17460 commits: commits
17461 .iter()
17462 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17463 .collect(),
17464 stat: stat.to_owned(),
17465 }
17466 }
17467
17468 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17469 let mut r = review_round(true, 0, 1, 1, true, true);
17470 r.round = round;
17471 r.fix = notes.map(|n| FixRecord {
17472 agent: "fixer".to_owned(),
17473 addressed: Vec::new(),
17474 rejected: Vec::new(),
17475 notes: n.to_owned(),
17476 committed: true,
17477 failed: None,
17478 duration_ms: 0,
17479 continuation: None,
17480 });
17481 r
17482 }
17483
17484 #[test]
17485 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17486 let state = review_state(&["ignored"]);
17487 let f = facts(
17488 &[
17489 (
17490 "fix(login): resolve PATH on macOS",
17491 "Login shells skip rc files.",
17492 ),
17493 ("fix: address review", ""),
17494 ],
17495 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17496 );
17497 let m = pr_message_with(&state, 'A', Some(&f));
17498 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17499 assert!(
17500 m.body
17501 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17502 "{}",
17503 m.body
17504 );
17505 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17506 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17507 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17508 for banned in [
17509 "Review the work already",
17510 "no task statement",
17511 "Original task",
17512 ] {
17513 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17514 }
17515 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17516 }
17517
17518 #[test]
17519 fn pr_message_review_truncates_a_huge_first_commit_body() {
17520 let state = review_state(&["ignored"]);
17521 let f = facts(
17522 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17523 "s",
17524 );
17525 let m = pr_message_with(&state, 'A', Some(&f));
17526 assert!(m.body.len() < 30_000, "{}", m.body.len());
17527 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17528 assert!(
17529 m.body.contains("- ... 1 more commit(s)"),
17530 "{}",
17531 m.body.len()
17532 );
17533 assert!(m.body.ends_with("magi:candidate-a\n"));
17534 }
17535
17536 #[test]
17537 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17538 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17539 assert_eq!(m.title, "feat: x");
17540 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17541 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17542 assert!(!m.body.contains("no task statement"), "{}", m.body);
17543 }
17544
17545 #[test]
17546 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17547 let mut state = review_state(&["日本語の件名"]);
17548 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17549 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17550 state.candidates[0].branch = "機能/ログイン".to_owned();
17551 assert!(
17552 pr_message(&state, 'A')
17553 .title
17554 .starts_with("chore: land candidate A")
17555 );
17556 }
17557
17558 #[test]
17559 fn pr_message_review_fixes_survive_a_clean_final_round() {
17560 let mut state = review_state(&["feat: x"]);
17561 state.reviews = vec![
17562 round_with_notes(1, Some("handled the PATH case")),
17563 round_with_notes(2, None),
17564 ];
17565 let body = pr_message(&state, 'A').body;
17566 assert!(
17567 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17568 "{body}"
17569 );
17570 assert!(body.contains("### Round 1"), "{body}");
17571
17572 state.reviews = vec![
17573 round_with_notes(1, Some("first fix")),
17574 round_with_notes(2, Some("")),
17575 round_with_notes(3, Some("second fix")),
17576 round_with_notes(4, None),
17577 ];
17578 let body = pr_message(&state, 'A').body;
17579 assert!(
17580 body.contains("### Round 1") && !body.contains("first fix"),
17581 "{body}"
17582 );
17583 assert!(
17584 body.contains("### Round 3") && !body.contains("second fix"),
17585 "{body}"
17586 );
17587 assert!(body.contains("### Round 2"), "{body}");
17588 }
17589
17590 const FINAL: &str = "abcdef1234567890";
17591
17592 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17593 let mut r = round_with_notes(round, None);
17594 r.verified_head = verified.map(str::to_owned);
17595 r
17596 }
17597
17598 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17599 state.gate = vec![CommandOutcome {
17600 command: "gate".to_owned(),
17601 code: Some(0),
17602 output_tail: String::new(),
17603 duration_ms: 0,
17604 resource_blocked: false,
17605 }];
17606 state.gate_ran = true;
17607 state.gate_head = head.map(str::to_owned);
17608 }
17609
17610 fn body_at(state: &RunState) -> String {
17611 pr_message_raw(
17612 state,
17613 'A',
17614 None,
17615 crate::pr_outcome::FinalHead::At(FINAL),
17616 ChangeNote::NotAsked,
17617 )
17618 .body
17619 }
17620
17621 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17622
17623 #[test]
17624 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17625 let mut state = review_state(&["feat: x"]);
17626 state.reviews = vec![
17627 round_with_notes(1, Some(SANDBOXED)),
17628 passing_round(2, Some(FINAL)),
17629 ];
17630 ok_gate(&mut state, Some(FINAL));
17631 let body = body_at(&state);
17632 assert!(!body.contains("Handled the PATH case."), "{body}");
17633 assert!(!body.contains("could not commit"), "{body}");
17634 assert!(!body.contains("could not run"), "{body}");
17635 assert!(
17636 body.contains("The fix is committed on the branch"),
17637 "{body}"
17638 );
17639 assert!(!body.contains("sandbox"), "{body}");
17640 assert!(
17641 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17642 "{body}"
17643 );
17644 assert!(
17645 body.contains("Gate (`verify.gate`): passed on the final commit"),
17646 "{body}"
17647 );
17648 assert!(
17650 state.reviews[0]
17651 .fix
17652 .as_ref()
17653 .unwrap()
17654 .notes
17655 .contains("could not commit")
17656 );
17657 }
17658
17659 #[test]
17660 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17661 let mut state = review_state(&["feat: x"]);
17662 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17663 r1.fix.as_mut().unwrap().committed = false;
17664 let mut r2 = passing_round(2, Some(FINAL));
17665 r2.e2e[0].code = Some(1);
17666 state.reviews = vec![r1, r2];
17667 let body = body_at(&state);
17668 assert!(!body.contains("could not commit"), "{body}");
17669 assert!(!body.contains("could not run"), "{body}");
17670 assert!(body.contains("This round produced no commit"), "{body}");
17671 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17672 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17673 assert!(!body.contains("passed"), "{body}");
17674 }
17675
17676 #[test]
17677 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17678 let mut state = review_state(&["feat: x"]);
17679 state.reviews = vec![
17680 round_with_notes(1, Some(SANDBOXED)),
17681 passing_round(2, Some("0123456789abcdef")),
17682 ];
17683 ok_gate(&mut state, Some("0123456789abcdef"));
17684 let body = body_at(&state);
17685 assert!(!body.contains("HTTP tests could not run"), "{body}");
17686 assert!(!body.contains("could not commit"), "{body}");
17688 assert!(
17689 body.contains("passed on an earlier commit (`0123456`)"),
17690 "{body}"
17691 );
17692 assert!(!body.contains("passed on the final commit"), "{body}");
17693
17694 ok_gate(&mut state, None);
17696 let body = body_at(&state);
17697 assert!(
17698 body.contains("passed on a commit that was not recorded"),
17699 "{body}"
17700 );
17701 }
17702
17703 #[test]
17704 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17705 let state = state_with_summary(
17706 "add retries to the client",
17707 "TITLE: feat: retries\n\nDid it.",
17708 );
17709 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17710 assert_eq!(m.title, "feat: retries");
17711 assert!(
17712 m.body.contains("<summary>Original task</summary>"),
17713 "{}",
17714 m.body
17715 );
17716 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17717 assert!(
17718 m.body
17719 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17720 "{}",
17721 m.body
17722 );
17723 }
17724
17725 #[test]
17726 fn pr_message_review_bounds_a_long_english_subject() {
17727 let long = format!("feat: {}", "word ".repeat(100));
17728 let m = pr_message(&review_state(&[&long]), 'A');
17729 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17730 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17731 }
17732
17733 #[test]
17734 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17735 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17736 let m = pr_message(&state, 'A');
17737 assert_eq!(m.title, "add retries");
17738 assert!(m.body.contains("<summary>Original task</summary>"));
17739 assert!(!m.body.contains("Commits under review"));
17740 assert_eq!(landing_subject_source(&state), state.instruction);
17741 }
17742
17743 #[test]
17744 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17745 let state = review_state(&["feat: the change", "fix: typo"]);
17746 let source = landing_subject_source(&state);
17747 assert_eq!(land::merge_subject("", &source), "feat: the change");
17748 assert_eq!(
17749 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17750 "feat: the change"
17751 );
17752 assert_eq!(
17754 land::merge_subject("feat: renamed by hand", &source),
17755 "feat: renamed by hand"
17756 );
17757 let blank = review_state(&["日本語"]);
17758 assert!(
17759 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17760 );
17761 }
17762
17763 #[test]
17764 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17765 let state = review_state(&["feat: the change"]);
17766 let source = landing_subject_source(&state);
17767 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17768 assert_eq!(
17769 land::merge_subject(landing_title(&state, old), &source),
17770 "feat: the change"
17771 );
17772 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17773 let task = state_with_summary("add retries", "");
17774 assert_eq!(landing_title(&task, old), old);
17775 }
17776
17777 #[test]
17778 fn pr_message_describes_the_change_not_the_task() {
17779 let state = state_with_summary(
17780 "今回やってほしいこと: results projector を直す",
17781 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17782 );
17783 let m = pr_message(&state, 'A');
17784 assert_eq!(m.title, "fix(web): batch the runs list reads");
17785 assert!(
17786 m.body.starts_with("## Summary\n\n- reads run.json once"),
17787 "{}",
17788 m.body
17789 );
17790 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17791 let task_at = m.body.find("今回やってほしいこと").unwrap();
17792 let details_at = m.body.find("<details>").unwrap();
17793 assert!(
17794 details_at < task_at,
17795 "the task lives inside <details>: {}",
17796 m.body
17797 );
17798 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17799 assert!(m.body.contains("magi:candidate-a"));
17800 }
17801
17802 #[test]
17803 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17804 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17805 let m = pr_message(&state, 'A');
17806 assert_eq!(m.title, "add retries");
17807 assert!(
17808 m.body.contains("## Summary\n\n- did some things"),
17809 "{}",
17810 m.body
17811 );
17812
17813 let none = RunState::new(
17814 PathBuf::from("/repo"),
17815 "main".to_owned(),
17816 "abc1234".to_owned(),
17817 "add retries".to_owned(),
17818 Config::default(),
17819 );
17820 let m = pr_message(&none, 'A');
17821 assert_eq!(m.title, "add retries");
17822 assert!(!m.body.contains("## Summary"), "{}", m.body);
17823 }
17824
17825 #[test]
17826 fn pr_message_refuses_the_candidate_commit_subject() {
17827 for bad in [
17828 "TITLE: magi: candidate A (uncommitted work)",
17829 "TITLE: chore: stuff (uncommitted work)",
17830 "TITLE: ",
17831 ] {
17832 let state = state_with_summary("add retries", bad);
17833 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17834 }
17835 }
17836
17837 #[test]
17838 fn pr_message_bounds_a_very_long_task_and_title() {
17839 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17840 let state = state_with_summary(&long, "- nothing");
17841 let m = pr_message(&state, 'A');
17842 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17843 assert!(!m.title.contains('\n'));
17844
17845 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17846 let m = pr_message(&state, 'A');
17847 assert!(m.title.starts_with("feat: "));
17848 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17849 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17850 }
17851
17852 fn long_title_of(instruction: &str) -> String {
17853 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17854 }
17855
17856 #[test]
17857 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17858 let first = "Make the landing path keep a readable title for long tasks";
17859 let line = format!(
17860 "{first}. {}",
17861 "And then keep going with more words ".repeat(20)
17862 );
17863 let t = long_title_of(&line);
17864 assert_eq!(t, first);
17865 assert!(!t.starts_with("chore: land"));
17866 }
17867
17868 #[test]
17869 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17870 let line = "word ".repeat(200);
17871 let t = long_title_of(&line);
17872 assert!(t.ends_with("word..."), "{t}");
17873 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17874 }
17875
17876 #[test]
17877 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17878 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17879 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17880 }
17881 }
17882
17883 #[test]
17884 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17885 let mut state = state_with_summary("再試行を追加する", "- nothing");
17886 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17887 state.config.graph.github_language = "ja".into();
17888 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17889 }
17890
17891 #[test]
17892 fn pr_message_title_limit_is_exact() {
17893 let at = "a".repeat(PR_TITLE_MAX);
17894 assert_eq!(long_title_of(&at), at);
17895 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17896 assert!(over.ends_with("..."), "{over}");
17897 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17898 }
17899
17900 #[test]
17901 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17902 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17903 let t = long_title_of(&line);
17904 assert!(t.ends_with("..."), "{t}");
17905 assert!(t.is_ascii(), "{t}");
17906 }
17907
17908 #[test]
17909 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17910 let line = format!(
17911 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17912 "plus more filler words ".repeat(20)
17913 );
17914 let t = long_title_of(&line);
17915 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17916 }
17917
17918 #[test]
17919 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17920 let line = format!("\n\n# {}", "title words ".repeat(40));
17921 let state = state_with_summary(&line, "- nothing");
17922 let m = pr_message(&state, 'A');
17923 assert!(m.title.starts_with("title words"), "{}", m.title);
17924 assert_eq!(
17925 land::merge_subject(&m.title, &landing_subject_source(&state)),
17926 m.title
17927 );
17928 assert_eq!(
17930 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17931 "feat: renamed by hand"
17932 );
17933 }
17934
17935 #[tokio::test]
17936 async fn github_text_rewrite_is_bounded_and_falls_back() {
17937 crate::run::pin_test_home();
17938 for (reply, accepted) in [
17939 (
17940 "TITLE: fix: retries\nAdd retries for failed requests.",
17941 true,
17942 ),
17943 (
17944 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17945 false,
17946 ),
17947 (
17948 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17949 true,
17950 ),
17951 (
17952 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17953 true,
17954 ),
17955 (
17956 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17957 true,
17958 ),
17959 (
17960 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17961 false,
17962 ),
17963 ("TITLE: fix: retries\n", false),
17964 ] {
17965 let dir = tempfile::tempdir().unwrap();
17966 let mut runner = runner_at(RunStatus::Gating);
17967 runner.state = state_with_summary(
17968 "add retries",
17969 "TITLE: fix: retries\n日本語の説明を書きます。",
17970 );
17971 runner.state.repo = dir.path().to_owned();
17972 runner.state.candidates[0].worktree = dir.path().to_owned();
17973 let mut author = spec("alpha");
17974 author.command = vec![
17975 "sh".into(),
17976 "-c".into(),
17977 "printf '%s' \"$REWRITE_REPLY\"".into(),
17978 ];
17979 author.env.insert("REWRITE_REPLY".into(), reply.into());
17980 runner.state.config.agents = vec![author];
17981 seed_public_summary(&mut runner);
17982 let winner = runner.state.candidates[0].clone();
17983 let first = runner
17984 .guarded_pr_message(&winner, None, true)
17985 .await
17986 .unwrap();
17987 let message = if accepted {
17988 first.expect("a passing rewrite never asks")
17989 } else {
17990 assert!(first.is_none());
17992 assert!(runner.state.parked);
17993 let store = ask::Questions::open();
17994 let asked: Vec<_> = store
17995 .list()
17996 .into_iter()
17997 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17998 .collect();
17999 assert_eq!(asked.len(), 1);
18000 assert!(!asked[0].detail.contains("secret"));
18001 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18002 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18003 assert_eq!(
18004 asked[0]
18005 .choices
18006 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18007 rewrite.is_some()
18008 );
18009 store
18012 .update(&asked[0].id, |q| {
18013 q.abandon("test");
18014 Ok(())
18015 })
18016 .unwrap();
18017 let again = runner
18018 .guarded_pr_message(&winner, None, true)
18019 .await
18020 .unwrap()
18021 .expect("an abandoned question falls back");
18022 let n = store
18023 .list()
18024 .into_iter()
18025 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18026 .count();
18027 assert_eq!(n, 1, "asked once per rejected text");
18028 again
18029 };
18030 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18031 assert!(!message.body.contains("secret"));
18032 assert!(!message.body.contains("prefix"));
18033 assert!(!message.title.contains("secret"));
18034 assert_eq!(
18035 message.body.contains("Add retries for failed requests."),
18036 accepted
18037 );
18038 assert_eq!(
18039 runner.state.seats["github-text-rewrite"].turns, 1,
18040 "only one rewrite invocation"
18041 );
18042 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18043 if accepted {
18044 assert!(runner.state.github_text.is_none());
18045 assert!(!runner.state.parked);
18046 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18047 assert!(message.body.contains("## Final verification"));
18048 assert!(message.body.contains("Evidence: final diff"));
18049 assert!(
18050 message
18051 .body
18052 .contains(&format!("magi:run/{}", runner.state.id))
18053 );
18054 }
18055 if !accepted {
18056 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18057 assert!(
18058 message
18059 .body
18060 .contains(&format!("magi:run/{}", runner.state.id))
18061 );
18062 }
18063 }
18064 }
18065
18066 fn scripted(id: &str, reply: &str) -> AgentSpec {
18067 let mut a = spec(id);
18068 a.command = vec![
18069 "sh".into(),
18070 "-c".into(),
18071 "printf '%s' \"$REWRITE_REPLY\"".into(),
18072 ];
18073 a.env.insert("REWRITE_REPLY".into(), reply.into());
18074 a
18075 }
18076
18077 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18078 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18079
18080 fn seed_public_summary(runner: &mut Runner) {
18082 let repo = &runner.state.repo;
18083 let g = |args: &[&str]| {
18084 let out = std::process::Command::new("git")
18085 .args(args)
18086 .current_dir(repo)
18087 .quiet()
18088 .output()
18089 .unwrap();
18090 assert!(out.status.success(), "git {args:?}");
18091 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18092 };
18093 g(&["init", "-q", "-b", "main"]);
18094 g(&[
18095 "-c",
18096 "user.name=Test",
18097 "-c",
18098 "user.email=test@example.invalid",
18099 "commit",
18100 "-q",
18101 "--allow-empty",
18102 "-m",
18103 "feat: fixture",
18104 ]);
18105 let head = g(&["rev-parse", "HEAD"]);
18106 g(&["update-ref", "refs/remotes/origin/main", &head]);
18107 let evidence = crate::pr_outcome::summary_evidence(
18108 &runner.state,
18109 crate::pr_outcome::FinalHead::At(&head),
18110 );
18111 let generated = &runner.state.candidates[0].summary;
18112 runner.state.change_summary = Some(crate::run::ChangeSummary {
18113 base: head.clone(),
18114 head,
18115 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18116 title: summary_title(generated).unwrap(),
18117 body: summary_without_title(generated),
18118 });
18119 }
18120
18121 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18122 let mut runner = runner_at(RunStatus::Gating);
18123 runner.state = state_with_summary("add retries", JA_SUMMARY);
18124 if review_only {
18125 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18126 }
18127 runner.state.repo = dir.to_owned();
18128 runner.state.candidates[0].worktree = dir.to_owned();
18129 runner.state.config.agents = agents;
18130 seed_public_summary(&mut runner);
18131 runner
18132 }
18133
18134 fn gate_questions(runner: &Runner) -> usize {
18135 ask::Questions::open()
18136 .list()
18137 .into_iter()
18138 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18139 .count()
18140 }
18141
18142 #[tokio::test]
18143 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18144 crate::run::pin_test_home();
18145 let dir = tempfile::tempdir().unwrap();
18146 let mut runner = rewrite_runner(
18147 dir.path(),
18148 true,
18149 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18150 );
18151 runner.state.config.roles.language_judge =
18152 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18153 let winner = runner.state.candidates[0].clone();
18154 let msg = runner
18155 .guarded_pr_message(&winner, None, true)
18156 .await
18157 .unwrap()
18158 .expect("an English rewrite posts without asking");
18159 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18160 assert!(msg.body.contains("<details>kept</details>"));
18161 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18162 assert_eq!(gate_questions(&runner), 0);
18163 assert!(runner.state.github_text.is_none());
18164 assert!(!runner.state.seats.contains_key("impl-A"));
18165 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18166 }
18167
18168 #[tokio::test]
18169 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18170 crate::run::pin_test_home();
18171 let dir = tempfile::tempdir().unwrap();
18172 let mut runner = rewrite_runner(
18173 dir.path(),
18174 false,
18175 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18176 );
18177 runner.state.config.roles.language_judge =
18178 Some(crate::config::AgentChoice::One("beta".into()));
18179 runner.state.seats.insert(
18180 "impl-A".into(),
18181 SeatState::new("impl-A", "alpha", runner.state.seed),
18182 );
18183 let winner = runner.state.candidates[0].clone();
18184 let msg = runner
18185 .guarded_pr_message(&winner, None, true)
18186 .await
18187 .unwrap()
18188 .expect("rewritten");
18189 assert!(msg.body.contains("<details>kept</details>"));
18190 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18191 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18192 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18193 }
18194
18195 #[tokio::test]
18196 async fn rewrite_reuses_a_resumable_implementer_session() {
18197 crate::run::pin_test_home();
18198 let dir = tempfile::tempdir().unwrap();
18199 let mut runner = rewrite_runner(
18200 dir.path(),
18201 false,
18202 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18203 );
18204 runner.state.config.roles.language_judge =
18205 Some(crate::config::AgentChoice::One("beta".into()));
18206 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18207 seat.turns = 1;
18208 runner.state.seats.insert("impl-A".into(), seat);
18209 let winner = runner.state.candidates[0].clone();
18210 runner
18211 .guarded_pr_message(&winner, None, true)
18212 .await
18213 .unwrap()
18214 .expect("rewritten");
18215 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18216 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18217 }
18218
18219 #[tokio::test]
18220 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18221 crate::run::pin_test_home();
18222 let dir = tempfile::tempdir().unwrap();
18223 let mut broken = spec("bad");
18224 broken.command = vec!["false".into()];
18225 let mut runner = rewrite_runner(
18226 dir.path(),
18227 true,
18228 vec![
18229 broken,
18230 scripted("noformat", "Add retries."),
18231 scripted("good", EN_REWRITE),
18232 ],
18233 );
18234 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18235 "bad".into(),
18236 "noformat".into(),
18237 "good".into(),
18238 ]));
18239 let winner = runner.state.candidates[0].clone();
18240 let msg = runner
18241 .guarded_pr_message(&winner, None, true)
18242 .await
18243 .unwrap()
18244 .expect("the third agent rewrites");
18245 assert!(msg.body.contains("`src/net.rs`"));
18246 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18247 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18248 }
18249
18250 #[tokio::test]
18251 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18252 crate::run::pin_test_home();
18253 for (agents, role, saved) in [
18255 (
18256 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18257 Some("beta"),
18258 true,
18259 ),
18260 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18261 (Vec::new(), None, false),
18262 ] {
18263 let dir = tempfile::tempdir().unwrap();
18264 let mut runner = rewrite_runner(dir.path(), true, agents);
18265 runner.state.config.roles.language_judge =
18266 role.map(|r| crate::config::AgentChoice::One(r.into()));
18267 let winner = runner.state.candidates[0].clone();
18268 let first = runner
18269 .guarded_pr_message(&winner, None, true)
18270 .await
18271 .unwrap();
18272 assert!(first.is_none(), "withheld text parks the run");
18273 assert!(runner.state.parked);
18274 assert_eq!(gate_questions(&runner), 1);
18275 let g = runner.state.github_text.as_ref().unwrap();
18276 assert_eq!(g.rewrite.is_some(), saved);
18277 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18278 }
18279 }
18280
18281 #[tokio::test]
18282 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18283 crate::run::pin_test_home();
18284 let dir = tempfile::tempdir().unwrap();
18285 let mut runner = runner_at(RunStatus::Gating);
18286 runner.state =
18287 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18288 runner.state.repo = dir.path().to_owned();
18289 runner.state.candidates[0].worktree = dir.path().to_owned();
18290 let mut author = spec("alpha");
18291 author.command = vec![
18292 "sh".into(),
18293 "-c".into(),
18294 "printf '%s' \"$REWRITE_REPLY\"".into(),
18295 ];
18296 author.env.insert(
18297 "REWRITE_REPLY".into(),
18298 format!(
18299 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18300 "English sentence. ".repeat(600)
18301 ),
18302 );
18303 runner.state.config.agents = vec![author];
18304 seed_public_summary(&mut runner);
18305 let winner = runner.state.candidates[0].clone();
18306 assert!(
18307 runner
18308 .guarded_pr_message(&winner, None, true)
18309 .await
18310 .unwrap()
18311 .is_none()
18312 );
18313 let record = runner.state.github_text.clone().unwrap();
18314 let rewrite = record.rewrite.clone().unwrap();
18315 assert_eq!(rewrite.failed, ["body-language"]);
18316 assert!(
18317 rewrite
18318 .body
18319 .ends_with(&format!("magi:run/{}", runner.state.id))
18320 );
18321 let store = ask::Questions::open();
18322 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18323 assert!(
18324 q.choices
18325 .contains(&crate::github_text::USE_REWRITE.to_owned())
18326 );
18327 for label in [
18328 "ORIGINAL",
18329 "FALLBACK",
18330 "REWRITE",
18331 "Rules still failed: body-language",
18332 "truncated",
18333 ] {
18334 assert!(q.detail.contains(label), "{label}");
18335 }
18336 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18337 &runner.state,
18338 &crate::github_text::artifact_name(&record.fingerprint),
18339 ))
18340 .unwrap();
18341 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18342 runner.state =
18344 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18345 store
18346 .update(&q.id, |q| {
18347 q.say("まだ日本語")?;
18348 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18349 })
18350 .unwrap();
18351 assert!(
18352 runner
18353 .guarded_pr_message(&winner, None, true)
18354 .await
18355 .unwrap()
18356 .is_none()
18357 );
18358 let retry = runner.state.github_text.as_ref().unwrap();
18359 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18360 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18361 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18362 assert!(
18363 q.posting_comparison
18364 .as_ref()
18365 .unwrap()
18366 .matches_detail(&q.detail)
18367 );
18368 store
18369 .update(&q.id, |q| {
18370 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18371 })
18372 .unwrap();
18373 let message = runner
18374 .guarded_pr_message(&winner, None, true)
18375 .await
18376 .unwrap()
18377 .unwrap();
18378 assert_eq!(message.title, rewrite.title);
18379 assert_eq!(message.body, rewrite.body);
18380 assert_eq!(
18381 runner.state.github_text.as_ref().unwrap().choice,
18382 Some(GithubTextChoice::Rewrite)
18383 );
18384 let again = runner
18385 .guarded_pr_message(&winner, None, true)
18386 .await
18387 .unwrap()
18388 .unwrap();
18389 assert_eq!(again.title, message.title);
18390 assert_eq!(again.body, message.body);
18391 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18392 assert!(
18393 !runner
18394 .state
18395 .events
18396 .iter()
18397 .any(|e| e.message.contains(&rewrite.title)
18398 || e.message.contains("English sentence."))
18399 );
18400 }
18401
18402 #[tokio::test]
18403 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18404 crate::run::pin_test_home();
18405 let dir = tempfile::tempdir().unwrap();
18406 let mut runner = runner_at(RunStatus::Gating);
18407 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18408 runner.state.repo = dir.path().to_owned();
18409 runner.state.candidates[0].worktree = dir.path().to_owned();
18410 seed_public_summary(&mut runner);
18411 let winner = runner.state.candidates[0].clone();
18412 assert!(
18413 runner
18414 .guarded_pr_message(&winner, None, true)
18415 .await
18416 .unwrap()
18417 .is_none()
18418 );
18419 let record = runner.state.github_text.as_ref().unwrap();
18420 assert!(record.rewrite.is_none());
18421 let mut old = serde_json::to_value(record).unwrap();
18423 old.as_object_mut().unwrap().remove("rewrite");
18424 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18425 assert!(old.rewrite.is_none());
18426 let id = record.question.clone().unwrap();
18427 ask::Questions::open()
18428 .update(&id, |q| {
18429 q.choices.push(crate::github_text::USE_REWRITE.into());
18430 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18431 })
18432 .unwrap();
18433 let message = runner
18434 .guarded_pr_message(&winner, None, true)
18435 .await
18436 .unwrap()
18437 .unwrap();
18438 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18439 assert_eq!(
18440 runner.state.github_text.as_ref().unwrap().choice,
18441 Some(GithubTextChoice::Fallback)
18442 );
18443 }
18444
18445 #[tokio::test]
18446 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18447 crate::run::pin_test_home();
18448 let dir = tempfile::tempdir().unwrap();
18449 let mut runner = runner_at(RunStatus::Gating);
18450 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18451 runner.state.repo = dir.path().to_owned();
18452 runner.state.candidates[0].worktree = dir.path().to_owned();
18453 seed_public_summary(&mut runner);
18454 let winner = runner.state.candidates[0].clone();
18455 assert!(
18456 runner
18457 .guarded_pr_message(&winner, None, true)
18458 .await
18459 .unwrap()
18460 .is_none()
18461 );
18462 let store = ask::Questions::open();
18463 let find = |run: &str| {
18464 store
18465 .list()
18466 .into_iter()
18467 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18468 .unwrap()
18469 };
18470 let run = runner.state.id.clone();
18471 let q = find(&run);
18472 let fp = runner
18473 .state
18474 .github_text
18475 .as_ref()
18476 .unwrap()
18477 .fingerprint
18478 .clone();
18479 let name = crate::github_text::artifact_name(&fp);
18480 let path = crate::run::artifact_path(&runner.state, &name);
18481 assert!(q.detail.contains(&path.display().to_string()));
18482 assert!(q.detail.contains("title-language: title"));
18483 let saved = std::fs::read_to_string(&path).expect("artifact written");
18484 assert!(saved.contains("日本語のタイトル"));
18485 assert!(q.cwd.is_none());
18486 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18487 let say = |id: &str, text: &str, choice: &str| {
18489 store
18490 .update(id, |q| {
18491 q.thread.push(ask::Turn {
18492 who: ask::Who::Operator,
18493 body: text.to_owned(),
18494 at: jiff::Timestamp::now(),
18495 note: None,
18496 });
18497 q.answer(ask::Answer::Choice(choice.to_owned()))
18498 })
18499 .unwrap();
18500 };
18501 say(&q.id, "まだ日本語", "use my text");
18502 assert!(
18503 runner
18504 .guarded_pr_message(&winner, None, true)
18505 .await
18506 .unwrap()
18507 .is_none()
18508 );
18509 let second = find(&run);
18510 assert_ne!(second.id, q.id);
18511 assert!(second.detail.contains("did not pass"));
18512 assert!(second.detail.contains("日本語のタイトル"));
18515 assert!(!second.detail.contains("まだ日本語"));
18516 assert!(second.detail.contains(&path.display().to_string()));
18517 say(&second.id, "fix: retry failed requests", "use my text");
18518 let message = runner
18519 .guarded_pr_message(&winner, None, true)
18520 .await
18521 .unwrap()
18522 .expect("a vetted replacement is posted");
18523 assert_eq!(message.title, "fix: retry failed requests");
18524 assert_eq!(
18525 runner
18526 .state
18527 .github_text
18528 .as_ref()
18529 .unwrap()
18530 .chosen_title
18531 .as_deref(),
18532 Some("fix: retry failed requests")
18533 );
18534 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18535 let again = runner
18537 .guarded_pr_message(&winner, None, true)
18538 .await
18539 .unwrap()
18540 .unwrap();
18541 assert_eq!(again.title, message.title);
18542 assert_eq!(
18543 store
18544 .list()
18545 .into_iter()
18546 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18547 .count(),
18548 2
18549 );
18550 }
18551
18552 #[test]
18553 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18554 let mut state = state_with_summary(
18558 "add retries",
18559 "TITLE: fix(web): batch reads\n- reads run.json once",
18560 );
18561 state.config.graph.language = "ja".to_owned();
18562 let m = pr_message(&state, 'A');
18563 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18564 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18565
18566 let task = "今回やってほしいこと: results projector を直す";
18569 let mut state = state_with_summary(task, "- no title line");
18570 state.config.graph.language = "ja".to_owned();
18571 let m = pr_message(&state, 'A');
18572 assert_eq!(
18573 m.title,
18574 format!("chore: land candidate A of run {}", state.id)
18575 );
18576 assert!(
18577 m.body.contains(&format!(
18578 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18579 )),
18580 "{}",
18581 m.body
18582 );
18583 }
18584
18585 #[test]
18586 fn pr_message_scrubs_home_paths_and_addresses() {
18587 let state = state_with_summary(
18588 "fix it in /Users/someone/src/x",
18589 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18590 );
18591 let m = pr_message(&state, 'A');
18592 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18593 assert!(!m.body.contains(leak), "{}", m.body);
18594 }
18595 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18596 }
18597
18598 #[test]
18599 fn pr_message_survives_a_task_that_closes_details() {
18600 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18601 let m = pr_message(&state, 'A');
18602 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18603 }
18604
18605 #[test]
18606 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18607 let cmd = manual_merge_command(
18608 MergeStyle::Squash,
18609 Path::new("/repo"),
18610 "b",
18611 "fix: \"quoted\" $(x) `y`\n\nbody",
18612 );
18613 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18614 }
18615
18616 #[test]
18617 fn manual_merge_command_matches_the_configured_style() {
18618 let repo = Path::new("/repo");
18619 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18620
18621 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18622 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18623
18624 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18625 assert_eq!(
18626 squash,
18627 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18628 \"Merge magi run 0832 (candidate A)\""
18629 );
18630
18631 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18632 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18633 }
18634
18635 #[test]
18636 fn a_nudge_gets_a_quarter_of_the_budget() {
18637 assert_eq!(retry_budget(secs(1200), true), secs(300));
18639 assert_eq!(retry_budget(secs(3600), true), secs(900));
18640 }
18641
18642 #[test]
18643 fn a_resent_prompt_keeps_the_whole_budget() {
18644 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18647 assert_eq!(retry_budget(secs(60), false), secs(60));
18648 }
18649
18650 #[test]
18651 fn the_floor_never_exceeds_the_original_budget() {
18652 assert_eq!(retry_budget(secs(60), true), secs(60));
18656 assert_eq!(retry_budget(secs(480), true), secs(120));
18657 assert_eq!(retry_budget(secs(0), true), secs(0));
18658 }
18659
18660 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18661 agent::CommandEvidence {
18662 id: "item1".to_owned(),
18663 description: "cargo test".to_owned(),
18664 exit_code,
18665 result_summary: String::new(),
18666 source: "codex".to_owned(),
18667 }
18668 }
18669
18670 #[test]
18671 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18672 assert!(!has_unconfirmed_command(&[]));
18676 }
18677
18678 #[test]
18679 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18680 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18684 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18685 assert!(!has_unconfirmed_command(&[
18686 evidence(Some(0)),
18687 evidence(Some(101))
18688 ]));
18689 }
18690
18691 #[test]
18692 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18693 assert!(has_unconfirmed_command(&[
18694 evidence(Some(0)),
18695 evidence(None)
18696 ]));
18697 }
18698
18699 #[test]
18700 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18701 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18702 assert_eq!(
18703 verified_noop_claim(true, &[], text).as_deref(),
18704 Some("already fixed by b32cfc4, on main.")
18705 );
18706 }
18707
18708 #[test]
18709 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18710 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18713 assert!(verified_noop_claim(false, &[], text).is_none());
18714 }
18715
18716 #[test]
18717 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18718 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18719 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18720 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18722 }
18723
18724 #[test]
18725 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18726 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18727 }
18728
18729 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18732 runner.state.candidates = shape
18733 .iter()
18734 .enumerate()
18735 .map(|(i, &(empty, verified))| Candidate {
18736 index: i,
18737 label: (b'A' + i as u8) as char,
18738 agent: "sonnet".to_owned(),
18739 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18740 worktree: PathBuf::from(format!("/wt/{i}")),
18741 summary: String::new(),
18742 stat: String::new(),
18743 files: 0,
18744 commits: 0,
18745 empty,
18746 failed: None,
18747 verified_noop: verified.map(str::to_owned),
18748 duration_ms: 0,
18749 folded: false,
18750 })
18751 .collect();
18752 }
18753
18754 #[test]
18755 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18756 ask_test_home();
18757 let mut runner = runner_at(RunStatus::Implementing);
18758 set_candidates(
18759 &mut runner,
18760 &[
18761 (true, Some("already on main at b32cfc4")),
18762 (true, Some("same fix, see the existing test")),
18763 ],
18764 );
18765
18766 runner
18767 .after_implement()
18768 .expect("a verified no-op is not an error");
18769
18770 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18771 }
18772
18773 #[test]
18774 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18775 ask_test_home();
18776 let mut runner = runner_at(RunStatus::Implementing);
18777 set_candidates(
18781 &mut runner,
18782 &[(true, Some("already on main at b32cfc4")), (true, None)],
18783 );
18784
18785 let err = runner
18786 .after_implement()
18787 .expect_err("an unverified empty candidate must still fail the run");
18788
18789 assert!(
18790 err.to_string().contains("no candidate produced a change"),
18791 "{err}"
18792 );
18793 assert_eq!(runner.state.status, RunStatus::Failed);
18794 }
18795
18796 #[test]
18797 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18798 ask_test_home();
18799 let mut runner = runner_at(RunStatus::Implementing);
18800 set_candidates(&mut runner, &[(true, None), (true, None)]);
18801
18802 let err = runner
18803 .after_implement()
18804 .expect_err("no candidate declared anything; this is an ordinary failure");
18805
18806 assert!(
18807 err.to_string().contains("no candidate produced a change"),
18808 "{err}"
18809 );
18810 assert_eq!(runner.state.status, RunStatus::Failed);
18811 }
18812}
18813
18814#[derive(Debug)]
18817struct GraftRestoreFailed(String);
18818
18819impl std::fmt::Display for GraftRestoreFailed {
18820 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18821 write!(
18822 f,
18823 "could not restore the winner to its pre-graft commit: {}",
18824 self.0
18825 )
18826 }
18827}
18828
18829impl std::error::Error for GraftRestoreFailed {}
18830
18831#[cfg(test)]
18835mod resource_wait_tests {
18836 use super::*;
18837 use std::collections::BTreeMap;
18838
18839 fn sh(id: &str, script: &str) -> AgentSpec {
18840 AgentSpec {
18841 id: id.to_owned(),
18842 kind: crate::config::AgentKind::Command,
18843 model: None,
18844 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18845 extra_args: Vec::new(),
18846 env: BTreeMap::new(),
18847 prompt_delivery: None,
18848 max_turn_secs: None,
18849 }
18850 }
18851
18852 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18853 SeatJob {
18854 seat: SeatState::new("impl-A", &spec.id, 1),
18855 spec,
18856 cwd: dir.to_path_buf(),
18857 prompt: "do it".to_owned(),
18858 timeout,
18859 allow_write: true,
18860 sessions: false,
18861 artifacts: dir.join("art"),
18862 stem: "impl-A".to_owned(),
18863 handover: None,
18864 }
18865 }
18866
18867 fn state(resource_wait_secs: u64) -> RunState {
18868 let mut config = Config::default();
18869 config.graph.timeout_resource_wait = resource_wait_secs;
18870 RunState::new(
18871 PathBuf::from("/repo"),
18872 "main".to_owned(),
18873 "abc1234".to_owned(),
18874 "task".to_owned(),
18875 config,
18876 )
18877 }
18878
18879 fn hold(cache: &Path) -> crate::cache::Guard {
18880 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18881 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18882 crate::cache::AcquireOutcome::Acquired(g) => g,
18883 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18884 }
18885 }
18886
18887 async fn run(
18888 jobs: Vec<SeatJob>,
18889 sem: Arc<Semaphore>,
18890 cache: Option<&Path>,
18891 st: &mut RunState,
18892 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18893 let prompts = Prompts::default();
18894 let ctx = WaveCtx {
18895 run: "r",
18896 node: "implement",
18897 prompts: &prompts,
18898 cache,
18899 round: None,
18900 carry_seats: false,
18901 pauses: None,
18902 };
18903 wave(jobs, sem, &ctx, st, 0).await
18904 }
18905
18906 #[tokio::test]
18907 async fn verification_cache_wait_preserves_the_command_budget() {
18908 crate::run::pin_test_home();
18909 let tmp = tempfile::tempdir().unwrap();
18910 let cache = tmp.path().join("target");
18911 let guard = hold(&cache);
18912 let release = tokio::spawn(async move {
18913 tokio::time::sleep(Duration::from_millis(100)).await;
18914 guard.release();
18915 });
18916 let mut st = state(10);
18917 let budget = Duration::from_millis(50);
18918 let wait = repository_wait(&st, None);
18919 let (outcomes, _) = with_cache_lease(
18920 &mut st,
18921 Some(&cache),
18922 "gate",
18923 "gate",
18924 tmp.path(),
18925 "abc1234",
18926 budget,
18927 "test gate",
18928 &wait,
18929 |_, available| async move {
18930 assert_eq!(
18931 available, budget,
18932 "contention must not shorten verification"
18933 );
18934 (Vec::new(), false, Vec::new())
18935 },
18936 )
18937 .await
18938 .unwrap();
18939 release.await.unwrap();
18940 assert!(outcomes.is_empty(), "{outcomes:?}");
18941 assert!(
18942 st.events
18943 .iter()
18944 .any(|event| event.message.contains("waiting for"))
18945 );
18946 assert!(st.resource_wait.is_none());
18947 hold(&cache).release();
18949 }
18950
18951 #[tokio::test]
18952 async fn verification_cache_wait_expires_without_starting_commands() {
18953 crate::run::pin_test_home();
18954 let tmp = tempfile::tempdir().unwrap();
18955 let cache = tmp.path().join("target");
18956 let _held = hold(&cache);
18957 let mut st = state(0);
18958 let wait = repository_wait(&st, None);
18959 let (outcomes, _) = with_cache_lease(
18960 &mut st,
18961 Some(&cache),
18962 "gate",
18963 "gate",
18964 tmp.path(),
18965 "abc1234",
18966 Duration::from_secs(60),
18967 "test gate",
18968 &wait,
18969 |_, _| async { panic!("commands must not start without the cache lease") },
18970 )
18971 .await
18972 .unwrap();
18973 assert_eq!(outcomes.len(), 1);
18974 assert!(outcomes[0].resource_blocked);
18975 assert!(st.resource_wait.is_none());
18976 }
18977
18978 #[tokio::test]
18979 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18980 crate::run::pin_test_home();
18981 let tmp = tempfile::tempdir().unwrap();
18982 let cache = tmp.path().join("target-a");
18983 let guard = hold(&cache);
18984 tokio::spawn(async move {
18986 tokio::time::sleep(Duration::from_millis(1500)).await;
18987 guard.release();
18988 });
18989 let mut st = state(60);
18990 let results = run(
18991 vec![job(
18992 sh("alpha", "echo ok"),
18993 tmp.path(),
18994 Duration::from_secs(1),
18995 )],
18996 Arc::new(Semaphore::new(1)),
18997 Some(&cache),
18998 &mut st,
18999 )
19000 .await;
19001 assert!(
19002 matches!(results[0].2, AgentOutcome::Ok(_)),
19003 "the wait must not be charged to the agent"
19004 );
19005 assert!(st.resource_block.is_none());
19006 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19007 assert!(st.resource_wait.is_none() && st.active.is_empty());
19008 }
19009
19010 #[tokio::test]
19011 #[cfg(unix)]
19012 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19013 use std::os::fd::AsRawFd as _;
19014 crate::run::pin_test_home();
19015 let tmp = tempfile::tempdir().unwrap();
19016 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19017 let common = tmp.path().join(".git");
19018 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19019 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19020 assert_eq!(
19021 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19022 0
19023 );
19024 let mut st = state(1);
19025 st.config.coordination = crate::config::test_coordination();
19026 let mut reader = job(
19027 sh("reader", "echo must-not-run"),
19028 tmp.path(),
19029 Duration::from_secs(30),
19030 );
19031 reader.allow_write = false;
19032 let results = run(
19033 vec![reader.clone()],
19034 Arc::new(Semaphore::new(1)),
19035 None,
19036 &mut st,
19037 )
19038 .await;
19039 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19040 assert_eq!(results[0].1.turns, 0);
19041 assert!(FailClass::of(&results[0].2).is_none());
19042 assert_eq!(
19043 st.resource_block.as_ref().unwrap().kind,
19044 "repository coordination"
19045 );
19046 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19047 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19048 drop(external);
19049 let editor = crate::repo_lock::SeatLease::for_checkout(
19050 tmp.path(),
19051 true,
19052 crate::config::test_coordination().as_ref(),
19053 )
19054 .await
19055 .unwrap();
19056 let pause = Pause::new();
19057 let interrupt = Pause::new();
19058 let prompts = Prompts::default();
19059 let ctx = WaveCtx {
19060 run: "r",
19061 node: "review",
19062 prompts: &prompts,
19063 cache: None,
19064 round: None,
19065 carry_seats: false,
19066 pauses: Some((&pause, &interrupt)),
19067 };
19068 let (results, ()) = tokio::join!(
19069 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19070 async {
19071 tokio::time::sleep(Duration::from_millis(50)).await;
19072 pause.park();
19073 }
19074 );
19075 assert!(
19076 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19077 );
19078 assert_eq!(results[0].1.turns, 0);
19079 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19080 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19081 drop(editor);
19082 }
19083
19084 #[tokio::test]
19085 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19086 crate::run::pin_test_home();
19087 let tmp = tempfile::tempdir().unwrap();
19088 let cache = tmp.path().join("target-b");
19089 let _held = hold(&cache);
19090 let mut st = state(1);
19091 let results = run(
19092 vec![job(
19093 sh("alpha", "echo ok"),
19094 tmp.path(),
19095 Duration::from_secs(1),
19096 )],
19097 Arc::new(Semaphore::new(1)),
19098 Some(&cache),
19099 &mut st,
19100 )
19101 .await;
19102 let (_, seat, out) = &results[0];
19103 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19104 assert!(wave_blocked(&results));
19105 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19106 assert_eq!(seat.turns, 0, "nothing was asked");
19107 assert!(
19108 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19109 "no agent was spawned"
19110 );
19111 assert!(FailClass::of(out).is_none(), "not a failure class");
19112 let block = st.resource_block.as_ref().expect("recorded for the resume");
19113 assert_eq!(block.kind, "shared build cache");
19114 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19115 assert!(st.active.is_empty());
19116 }
19117
19118 #[tokio::test]
19119 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19120 crate::run::pin_test_home();
19121 let tmp = tempfile::tempdir().unwrap();
19122 let mut st = state(1);
19123 let results = run(
19124 vec![job(
19125 sh("alpha", "echo ok"),
19126 tmp.path(),
19127 Duration::from_secs(1),
19128 )],
19129 Arc::new(Semaphore::new(0)),
19130 None,
19131 &mut st,
19132 )
19133 .await;
19134 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19135 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19136 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19137 }
19138
19139 #[tokio::test]
19140 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19141 crate::run::pin_test_home();
19142 let tmp = tempfile::tempdir().unwrap();
19143 let mut st = state(60);
19144 let results = run(
19145 vec![job(
19146 sh("alpha", "sleep 30"),
19147 tmp.path(),
19148 Duration::from_secs(1),
19149 )],
19150 Arc::new(Semaphore::new(1)),
19151 None,
19152 &mut st,
19153 )
19154 .await;
19155 let out = &results[0].2;
19156 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19157 let class = FailClass::of(out).expect("a failure");
19158 assert_eq!(class, FailClass::Timeout);
19159 assert!(should_hand_over(None, &class), "existing handover policy");
19160 assert!(st.resource_block.is_none());
19161 }
19162}