1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(
488 repo: &Path,
489 base_branch: &str,
490 remote: &str,
491 coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493 let _lease = crate::repo_lock::SeatLease::operation_with(
494 repo,
495 coordination,
496 &crate::repo_lock::WaitPolicy::current(),
497 )
498 .await?;
499 let tracking = format!("{remote}/{base_branch}");
500 let fetched = git::fetch(repo, remote, base_branch).await;
501 if let Ok(out) = &fetched
502 && out.ok()
503 && git::rev_exists(repo, &tracking).await
504 {
505 return Ok(crate::run::FetchedBase {
506 remote: remote.to_owned(),
507 branch: base_branch.to_owned(),
508 commit: git::rev_parse(repo, &tracking).await?,
509 fetched_at: jiff::Timestamp::now(),
510 });
511 }
512 let why = match &fetched {
513 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514 Ok(_) => format!("{remote} has no {base_branch}"),
515 Err(e) => e.to_string(),
516 };
517 bail!(
520 "cannot read {tracking} ({why}); refusing to branch off the local \
521 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522 base / remote in magi.toml. This is launcher preparation: check the \
523 launcher's Git administration access, network and repository coordination \
524 before retrying; an unattended seat cannot approve those permissions"
525 )
526}
527
528struct FixClaim {
545 path: PathBuf,
546}
547
548impl FixClaim {
549 fn acquire(dir: &Path) -> Result<Self> {
550 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551 let path = dir.join("fix.lock");
552 match Self::create(&path) {
553 Ok(claim) => Ok(claim),
554 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555 if Self::reclaim_if_dead(&path) {
556 Self::create(&path).with_context(|| format!("lock {}", path.display()))
557 } else {
558 bail!(
559 "another repair or `magi fix` is already running for this run ({} exists)",
560 path.display()
561 )
562 }
563 }
564 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565 }
566 }
567
568 fn create(path: &Path) -> std::io::Result<Self> {
569 let mut f = std::fs::OpenOptions::new()
570 .write(true)
571 .create_new(true)
572 .open(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
627 repo: &Path,
628 instruction: String,
629 also_scan: &str,
630 config: Config,
631 origin: Origin,
632 ) -> Result<Self> {
633 let repo = git::toplevel(repo).await?;
634 let missing = agent::missing_programs(&config.agents);
635 if !missing.is_empty() {
636 bail!(
637 "these agent programs are not on PATH: {}. Fix the roster in \
638 magi.toml or install them.",
639 missing.join(", ")
640 );
641 }
642 let base_branch =
643 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644 .await?;
645 let fetched_base = resolve_base(
646 &repo,
647 &base_branch,
648 &config.merge.remote,
649 config.coordination.as_ref(),
650 )
651 .await?;
652 let base_commit = fetched_base.commit.clone();
653 let roles = config.resolve_roles()?;
654 let max_parallel = config.graph.max_parallel.max(1);
655 let seeds = refs::resolve(
658 &repo,
659 &base_commit,
660 &config.merge.remote,
661 &format!("{also_scan}\n{instruction}"),
662 )
663 .await;
664 refs::plan(&repo, &seeds).await?;
665 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666 state.fetched_base = Some(fetched_base);
670 state.origin = Some(origin);
671 for seed in &seeds {
672 state.event(
673 "seed",
674 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675 );
676 }
677 state.seeds = seeds;
678 state.event("start", format!("run {} created", state.id));
679 state.save()?;
680 Ok(Self {
681 state,
682 roles,
683 sem: Arc::new(Semaphore::new(max_parallel)),
684 pause: Pause::new(),
685 interrupt: Pause::new(),
686 })
687 }
688
689 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703 Self::review_taking_over(repo, branch, config, None, origin).await
704 }
705
706 pub async fn review_taking_over(
713 repo: &Path,
714 branch: &str,
715 config: Config,
716 takeover: Option<crate::handover::Takeover>,
717 origin: Origin,
718 ) -> Result<Self> {
719 let repo = git::toplevel(repo).await?;
720 let missing = agent::missing_programs(&config.agents);
721 if !missing.is_empty() {
722 bail!(
723 "these agent programs are not on PATH: {}. Fix the roster in \
724 magi.toml or install them.",
725 missing.join(", ")
726 );
727 }
728 let base_branch =
729 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730 .await?;
731 if base_branch == branch {
732 bail!("`{branch}` is the base branch; there is nothing to review against");
733 }
734 let fetched_base = resolve_base(
735 &repo,
736 &base_branch,
737 &config.merge.remote,
738 config.coordination.as_ref(),
739 )
740 .await?;
741 let base_commit = fetched_base.commit.clone();
742
743 let roles = config.resolve_roles()?;
744 let max_parallel = config.graph.max_parallel.max(1);
745 let mut state = RunState::new(
746 repo.clone(),
747 base_branch,
748 base_commit.clone(),
749 String::new(),
750 config,
751 );
752 state.fetched_base = Some(fetched_base);
753 state.origin = Some(origin);
754
755 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762 earlier: Vec::new(),
763 home: crate::run::home(),
764 choice: None,
765 task_instruction: None,
766 });
767 let wait = repository_wait(&state, None);
768 let coordination_config = state.config.coordination.clone();
769 let coordination =
770 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771 .await?;
772 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773 if let Some(released) = &released {
774 state.event(
775 "release",
776 format!(
777 "took `{branch}` over from run {}: its worktree was released: {}",
778 crate::run::short_of(&released.old_id),
779 released.audit
780 ),
781 );
782 }
783 if let Some(choice) = takeover.choice.as_ref()
787 && let Err(e) =
788 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789 .await
790 {
791 if let Some(released) = &released {
792 released.restore(&repo, branch).await;
793 }
794 return Err(e.context("applying the owner's answer about the diverged branch"));
795 }
796 for id in takeover.earlier.iter().rev() {
802 if let Ok(old) = RunState::load_under(id, &takeover.home)
803 && let Some(head) = old.pr_head.clone()
804 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805 {
806 state.pr_head = Some(head);
807 break;
808 }
809 }
810 drop(coordination);
811 let opened = Self::open_review(
812 &repo,
813 branch,
814 state,
815 roles,
816 max_parallel,
817 base_commit,
818 &takeover,
819 )
820 .await;
821 if opened.is_err()
822 && let Some(released) = &released
823 {
824 let _coordination = crate::repo_lock::SeatLease::operation_with(
825 &repo,
826 coordination_config.as_ref(),
827 &wait,
828 )
829 .await?;
830 released.restore(&repo, branch).await;
831 }
832 opened
833 }
834
835 pub async fn recover_pr(
837 repo: &Path,
838 task: &str,
839 selection: crate::pr_recovery::Selection,
840 mut config: Config,
841 ) -> Result<Self> {
842 selection.configure(&mut config)?;
843 selection.check_task(task, &selection.instruction)?;
844 let base_branch =
845 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
846 .await?;
847 let fetched_base = resolve_base(
848 repo,
849 &base_branch,
850 &config.merge.remote,
851 config.coordination.as_ref(),
852 )
853 .await?;
854 let base_commit = fetched_base.commit.clone();
855 selection
856 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
857 .await?;
858 Box::pin(selection.verify_source(repo)).await?;
859 if let Some(proof) = &selection.source_proof {
860 anyhow::ensure!(
861 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
862 "source rebase base is outside the fetched landing base"
863 );
864 }
865 let roles = config.resolve_roles()?;
866 let max_parallel = config.graph.max_parallel.max(1);
867 let mut state = RunState::new(
868 repo.to_path_buf(),
869 base_branch,
870 base_commit.clone(),
871 selection.instruction.clone(),
872 config,
873 );
874 state.fetched_base = Some(fetched_base);
875 state.origin = Some(crate::run::Origin::queue(task));
876 state.pr_head = Some(selection.branch.clone());
877 state.pr_recovery = Some(selection.clone());
878 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
879 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
880 }
881
882 async fn open_selected(
883 repo: &Path,
884 state: RunState,
885 roles: ResolvedRoles,
886 max_parallel: usize,
887 base_commit: String,
888 selection: crate::pr_recovery::Selection,
889 ) -> Result<Self> {
890 let mut state = state;
891 let branch = state.branch_for('A');
892 let wait = repository_wait(&state, None);
893 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
894 git::git(repo, &["branch", &branch, selection.source()]).await?;
895 drop(coordination);
896 state.event(
897 "recovery",
898 format!(
899 "selected implementation {} for {} (public lease {})",
900 selection.source(),
901 selection.url,
902 selection.head
903 ),
904 );
905 let takeover = crate::handover::Takeover {
906 earlier: Vec::new(),
907 home: crate::run::home(),
908 choice: None,
909 task_instruction: Some(selection.instruction.clone()),
910 };
911 state.save()?;
912 let id = state.id.clone();
913 let opened = Self::open_review(
914 repo,
915 &branch,
916 state,
917 roles,
918 max_parallel,
919 base_commit,
920 &takeover,
921 )
922 .await;
923 match opened {
924 Ok(runner) => Ok(runner),
925 Err(e) => {
926 if let Ok(mut partial) = RunState::load(&id) {
927 partial.status = RunStatus::Blocked;
928 partial.driver_exited = true;
929 partial.event(
930 "recovery",
931 format!("startup failed; retained source and candidate {branch}: {e:#}"),
932 );
933 partial.save()?;
934 }
935 Err(crate::pr_recovery::StartupFailed {
936 run: id,
937 cause: format!("{e:#}"),
938 }
939 .into())
940 }
941 }
942 }
943
944 async fn inherit(
949 repo: &Path,
950 branch: &str,
951 base_commit: &str,
952 takeover: &crate::handover::Takeover,
953 ) -> Option<crate::recovery::RecoveryRecord> {
954 use crate::recovery::{Relation, assess};
955 let newest = takeover.earlier.last()?;
956 let mut chosen = None;
957 for id in takeover.earlier.iter().rev() {
958 if let Ok(old) = RunState::load_under(id, &takeover.home)
959 && !old.reviews.is_empty()
960 && (old.winner().is_some_and(|w| w.branch == branch)
961 || old.pr_head.as_deref() == Some(branch))
962 {
963 chosen = Some(old);
964 break;
965 }
966 }
967 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
968 .await
969 .unwrap_or_default();
970 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
971 Some(last) if last.head == tip => Relation::Same,
972 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
973 _ => Relation::Diverged,
974 };
975 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
976 Some(assess(from, chosen.as_ref(), base_commit, relation))
977 }
978
979 async fn open_review(
982 repo: &Path,
983 branch: &str,
984 mut state: RunState,
985 roles: ResolvedRoles,
986 max_parallel: usize,
987 base_commit: String,
988 takeover: &crate::handover::Takeover,
989 ) -> Result<Self> {
990 let wait = repository_wait(&state, None);
991 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
992 if state.pr_recovery.is_none() {
993 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
994 }
995 let start = review_base(
998 repo,
999 &state.config.merge.remote,
1000 &state.base_branch,
1001 &base_commit,
1002 branch,
1003 )
1004 .await;
1005 let log = git::log_oneline(repo, &start, branch)
1006 .await
1007 .unwrap_or_default();
1008 let log_text = if log.trim().is_empty() {
1009 "(no commit messages)"
1010 } else {
1011 log.trim()
1012 };
1013 let mut instruction = format!(
1014 "Review the work already on branch `{branch}`. There is no task \
1015 statement: what the change claims to do is whatever its commits \
1016 say.\n\n{log_text}"
1017 );
1018 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1022 if let Some(task) = takeover
1023 .task_instruction
1024 .as_deref()
1025 .filter(|t| !t.trim().is_empty())
1026 && rec.reviewed_head.is_some()
1027 {
1028 instruction = format!(
1029 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1030 task.trim()
1031 );
1032 rec.criteria_inherited = true;
1033 }
1034 state.event("recovery", rec.summary());
1035 state.recovery = Some(rec);
1036 }
1037 if let Some(selected) = &state.pr_recovery {
1038 instruction = selected.instruction.clone();
1039 }
1040 state.task_instruction = takeover.task_instruction.clone();
1041 state.instruction = instruction;
1042 state.reviewed_commits = Some(
1043 git::subjects(repo, &start, branch)
1044 .await
1045 .unwrap_or_default(),
1046 );
1047
1048 let worktree = state.worktree_root().join("under-review");
1051 if let Some(parent) = worktree.parent() {
1052 tokio::fs::create_dir_all(parent).await.ok();
1053 }
1054 let path = worktree.to_string_lossy().to_string();
1055 git::git(repo, &["worktree", "add", &path, branch])
1056 .await
1057 .with_context(|| {
1058 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1059 })?;
1060
1061 let setup = match ensure_setup_config(&mut state, repo).await {
1062 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1063 Err(e) => Err(e),
1064 };
1065 if let Err(e) = setup {
1066 git::worktree_remove_clean(repo, &worktree).await.ok();
1067 return Err(e);
1068 }
1069 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1070 .await
1071 .unwrap_or(0);
1072 if commits == 0 {
1073 git::worktree_remove_clean(repo, &worktree).await.ok();
1074 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1075 }
1076 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1077 .await
1078 .map(|f| f.len())
1079 .unwrap_or(0);
1080 if files == 0
1081 && let (Ok(head_tree), Ok(base_tree)) = (
1082 git::tree_of(&worktree, "HEAD").await,
1083 git::tree_of(&worktree, &base_commit).await,
1084 )
1085 && head_tree == base_tree
1086 {
1087 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1088 git::worktree_remove_clean(repo, &worktree).await.ok();
1089 bail!(
1090 "`{branch}` at {} has a tree identical to base {}; this usually means \
1091 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1092 against `{}/{branch}`) rather than an empty change",
1093 short(&head),
1094 short(&base_commit),
1095 state.config.merge.remote
1096 );
1097 }
1098 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1099 .await
1100 .unwrap_or_default();
1101
1102 state.candidates.push(Candidate {
1103 index: 0,
1104 label: 'A',
1105 agent: EXISTING_BRANCH.to_owned(),
1108 branch: branch.to_owned(),
1109 worktree,
1110 summary: String::new(),
1111 stat,
1112 files,
1113 commits,
1114 empty: false,
1115 failed: None,
1116 verified_noop: None,
1117 duration_ms: 0,
1118 folded: false,
1119 });
1120 state.tally = Some(Tally {
1121 first_choice: BTreeMap::from([('A', 0)]),
1122 borda: BTreeMap::new(),
1123 winner: 'A',
1124 rankings: 0,
1125 unanimous_initial: false,
1126 deliberated: false,
1127 changed_votes: 0,
1128 unanimous_final: false,
1129 tie_break: None,
1130 judges: 0,
1134 present: 0,
1135 quorum: 0,
1136 met_quorum: true,
1137 uncontested: Some("review-only run: nothing competed".to_owned()),
1138 });
1139 state.status = RunStatus::Reviewing;
1140 state.prep_complete = Some(true);
1141 state.event(
1142 "start",
1143 format!(
1144 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1145 state.id
1146 ),
1147 );
1148 state.save()?;
1149 Ok(Self {
1150 state,
1151 roles,
1152 sem: Arc::new(Semaphore::new(max_parallel)),
1153 pause: Pause::new(),
1154 interrupt: Pause::new(),
1155 })
1156 }
1157
1158 pub fn resume(id: &str) -> Result<Self> {
1160 let state = RunState::load(id)?;
1161 crate::pr_head::ensure_owner(&state)?;
1162 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1163 bail!(
1164 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1165 );
1166 }
1167 if let Some(to) = &state.released_to {
1168 bail!(
1169 "run {} cannot be resumed: its worktree was released to run {}",
1170 state.short(),
1171 crate::run::short_of(to)
1172 );
1173 }
1174 let roles = state.config.resolve_roles()?;
1175 if let Some(id) = &state.config.graph.selected_implementer
1176 && state.candidates.iter().any(|c| c.agent != *id)
1177 {
1178 bail!(
1179 "replacement implementer pin does not match the saved candidates; task must stay held"
1180 );
1181 }
1182 let max_parallel = state.config.graph.max_parallel.max(1);
1183 Ok(Self {
1184 state,
1185 roles,
1186 sem: Arc::new(Semaphore::new(max_parallel)),
1187 pause: Pause::new(),
1188 interrupt: Pause::new(),
1189 })
1190 }
1191
1192 pub async fn execute(&mut self) -> Result<()> {
1199 if let Some(selection) = &self.state.pr_recovery
1200 && (selection.source_proof.is_some() || selection.source_run.is_some())
1201 {
1202 Box::pin(selection.verify_source(&self.state.repo)).await?;
1203 }
1204 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1207 || self.state.land_approval.is_some()
1208 || self.state.merge_decision.is_some()
1209 || !self.state.merge_repairs.is_empty()
1210 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1211 {
1212 let claim = FixClaim::acquire(&self.state.dir())?;
1213 let disk = RunState::load(&self.state.id)?;
1214 let checkpoint = |s: &RunState| {
1215 (
1216 s.merge_repairs.len(),
1217 s.merge_repairs
1218 .last()
1219 .map(|r| (r.question.clone(), r.phase)),
1220 s.land_approval
1221 .as_ref()
1222 .map(|a| (a.question.clone(), a.head.clone())),
1223 )
1224 };
1225 if checkpoint(&disk) != checkpoint(&self.state) {
1226 bail!(
1227 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1228 );
1229 }
1230 Some(claim)
1231 } else {
1232 None
1233 };
1234 let mut execution = self.claim_driver()?;
1235 let runner = &mut *execution.runner;
1236 let wait = runner.repository_wait();
1237 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1240 let result = match result {
1241 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1242 record_repository_block(&mut runner.state, &e);
1243 runner.park_here()?;
1244 Ok(())
1245 }
1246 other => other,
1247 };
1248 execution.driver.finish(&mut runner.state)?;
1249 let ended = if result.is_err() {
1250 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1251 } else {
1252 crate::notices::run_ended(&runner.state)
1253 };
1254 if let Some(notice) = ended {
1255 crate::notices::raise(notice);
1256 }
1257 result
1258 }
1259
1260 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1261 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1262 }
1263
1264 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1265 let wait = self.repository_wait();
1266 coordinate_operation(&mut self.state, cwd, &wait).await
1267 }
1268
1269 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1271 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1272 let disk = RunState::load(&self.state.id)?;
1273 crate::pr_head::ensure_owner(&disk)?;
1274 if disk.liveness(false) == crate::run::Liveness::Live {
1275 bail!("run already has a live driver; wait for its execution to end");
1276 }
1277 if let Some(to) = &disk.released_to {
1278 bail!(
1279 "run {} cannot continue: its worktree was released to run {}",
1280 self.state.short(),
1281 crate::run::short_of(to)
1282 );
1283 }
1284 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1285 let mut execution = DriverExecution {
1286 runner: self,
1287 driver,
1288 };
1289 let _owner = owner;
1292 let state = &mut execution.runner.state;
1293 state.parked = false;
1298 state.resource_wait = None;
1299 state.resource_block = None;
1300 state.clear_active();
1307 let pid = std::process::id();
1322 state.driver_pid = Some(pid);
1323 state.driver_started_at = crate::proc::process_started_at(pid);
1324 state.driver_exited = false;
1325 state.begin_pass();
1326 execution.driver.bind(state);
1327 state.save()?;
1328 execution.driver.record(state)?;
1329 Ok(execution)
1330 }
1331
1332 async fn execute_graph(&mut self) -> Result<()> {
1333 self.state.configure_recovery_resume()?;
1334 self.persist_candidates().await?;
1337 if self
1338 .state
1339 .merge_decision
1340 .as_ref()
1341 .is_some_and(|decision| !decision.closed)
1342 && self.state.status != RunStatus::Landing
1343 {
1344 if crate::merge_decision::resumable_hold(&self.state) {
1345 self.state.merge = Some(MergeOutcome {
1346 mode: MergeMode::Pr,
1347 ok: false,
1348 detail: self
1349 .state
1350 .merge_decision
1351 .as_ref()
1352 .unwrap()
1353 .target
1354 .pr
1355 .clone(),
1356 empty: false,
1357 });
1358 self.run_land().await?;
1359 self.settle_questions();
1360 return Ok(());
1361 }
1362 bail!(
1363 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1364 );
1365 }
1366 if self.state.config.graph.selected_implementer.is_some()
1367 && self.state.status == RunStatus::Blocked
1368 && self.state.tally.is_none()
1369 && self.state.candidate_preservation.is_empty()
1370 {
1371 bail!(
1372 "selected implementation is held; explicitly release the owning task to retry it"
1373 );
1374 }
1375
1376 if self.state.merge_repairs.last().is_some_and(|r| {
1377 matches!(
1378 r.phase,
1379 crate::run::MergeRepairPhase::Requested
1380 | crate::run::MergeRepairPhase::Fixing
1381 | crate::run::MergeRepairPhase::Reviewing
1382 )
1383 }) {
1384 if self.repair_merge_review().await? {
1385 self.run_land().await?;
1386 }
1387 self.settle_questions();
1388 return Ok(());
1389 }
1390 if self.state.pr_publication_pending {
1393 self.merge().await?;
1394 if self.resource_halt()? {
1395 return Ok(());
1396 }
1397 self.state.save()?;
1398 self.settle_questions();
1399 return Ok(());
1400 }
1401 if self.state.status == RunStatus::Stalled {
1414 if self.recover_stall().await? {
1415 self.finish_after_tally().await?;
1416 } else {
1417 self.state.save()?;
1419 }
1420 return Ok(());
1421 }
1422 if self.state.status == RunStatus::Landing {
1432 self.run_land().await?;
1433 self.settle_questions();
1438 return Ok(());
1439 }
1440 if self.state.github_text.is_some()
1445 && self.state.merge.is_none()
1446 && self.state.status == RunStatus::Gating
1447 {
1448 self.merge().await?;
1449 if self.resource_halt()? {
1450 return Ok(());
1451 }
1452 self.state.save()?;
1453 return Ok(());
1454 }
1455 if let Err(e) = self.prep().await {
1456 if e.is::<crate::repo_lock::ResourceBlocked>() {
1457 return Err(e);
1458 }
1459 self.state.status = RunStatus::Blocked;
1460 self.state
1461 .prep_failure
1462 .get_or_insert_with(|| format!("{e:#}"));
1463 self.state
1464 .event("prep", format!("preparation stopped: {e:#}"));
1465 self.state.save()?;
1466 return Err(e);
1467 }
1468 if self.park_here()? {
1469 return Ok(());
1470 }
1471 self.advise().await?;
1472 if self.park_here()? {
1473 return Ok(());
1474 }
1475 self.implement().await?;
1476 if self.park_here()? {
1477 return Ok(());
1478 }
1479 if self.state.status == RunStatus::VerifiedNoop {
1483 return Ok(());
1484 }
1485 self.judge().await?;
1486 if self.park_here()? {
1487 return Ok(());
1488 }
1489 self.deliberate().await?;
1490 if self.park_here()? {
1491 return Ok(());
1492 }
1493 self.vote().await?;
1494 if self.park_here()? {
1495 return Ok(());
1496 }
1497 self.tally()?;
1498 if self.state.status == RunStatus::Stalled {
1503 self.state.save()?;
1507 return Ok(());
1508 }
1509 self.finish_after_tally().await?;
1510 Ok(())
1511 }
1512
1513 fn stop_for_resource(&mut self) -> Result<()> {
1518 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1520 {
1521 self.state.status = RunStatus::Blocked;
1522 }
1523 self.state.save()
1524 }
1525
1526 fn resource_halt(&mut self) -> Result<bool> {
1528 if self.state.resource_block.is_none() {
1529 return Ok(false);
1530 }
1531 self.stop_for_resource()?;
1532 Ok(true)
1533 }
1534
1535 fn park_here(&mut self) -> Result<bool> {
1542 let parking = self.pause.parked() || self.interrupt.parked();
1543 if !parking && self.state.resource_block.is_some() {
1544 self.stop_for_resource()?;
1545 return Ok(true);
1546 }
1547 if !self.pause.parked() && !self.interrupt.parked() {
1552 return Ok(false);
1553 }
1554 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1555 let why = match cause.as_ref().map(ParkCause::label) {
1556 Some(reason) => format!(
1557 "parked after `{}` ({reason}) — resume to carry on from here",
1558 self.state.status.as_str()
1559 ),
1560 None => format!(
1561 "parked after `{}` — resume to carry on from here",
1562 self.state.status.as_str()
1563 ),
1564 };
1565 self.state.event("park", why);
1566 self.state
1567 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1568 self.state.parked = true;
1569 self.state.save()?;
1570 Ok(true)
1571 }
1572
1573 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1575 if self.state.prep_failure.is_none() {
1576 return Ok(());
1577 }
1578 let failed_at = self
1579 .state
1580 .events
1581 .iter()
1582 .rev()
1583 .find(|event| event.node == "prep")
1584 .map_or(self.state.updated_at, |event| event.at);
1585 anyhow::ensure!(
1586 approval_at > failed_at,
1587 "preparation retry requires a new explicit release after the failure"
1588 );
1589 self.state.prep_failure = None;
1590 self.state.prep_complete = Some(false);
1591 self.state
1592 .event("prep-retry", "operator approved retry of preparation");
1593 Ok(())
1594 }
1595
1596 pub fn on_pause(&mut self, pause: Pause) {
1598 self.pause = pause;
1599 }
1600
1601 pub fn watch_interrupt(&mut self, pause: Pause) {
1607 self.interrupt = pause;
1608 }
1609
1610 fn settle_questions(&mut self) {
1630 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1631 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1632 }
1633 }
1634
1635 async fn finish_after_tally(&mut self) -> Result<()> {
1638 self.graft().await?;
1641 if self.resource_halt()? {
1642 return Ok(());
1643 }
1644 self.fold_losers().await?;
1645 self.sync_to_base().await?;
1650 if self.state.status == RunStatus::AlreadyInBase {
1651 return Ok(());
1652 }
1653 self.review_loop().await?;
1654 if self.resource_halt()? {
1655 return Ok(());
1656 }
1657 self.sync_to_base().await?;
1658 if self.state.status == RunStatus::AlreadyInBase {
1659 return Ok(());
1660 }
1661 self.gate().await?;
1662 self.merge().await?;
1663 if self.resource_halt()? {
1664 return Ok(());
1665 }
1666 self.state.save()?;
1667 Ok(())
1668 }
1669
1670 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1677 let result = match ensure_setup_config(&mut self.state, repo).await {
1678 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1679 Err(e) => Err(e),
1680 };
1681 if let Err(e) = result {
1682 git::worktree_remove_clean(repo, wt).await.ok();
1683 if in_prep {
1684 self.rollback_prep(repo).await;
1685 }
1686 self.state.status = RunStatus::Blocked;
1687 self.state.event(
1688 "setup",
1689 format!("worktree setup failed in {}: {e:#}", wt.display()),
1690 );
1691 self.state.save()?;
1692 return Err(e);
1693 }
1694 Ok(())
1695 }
1696
1697 async fn rollback_prep(&mut self, repo: &Path) {
1699 let root = self.state.worktree_root();
1700 for c in std::mem::take(&mut self.state.candidates) {
1701 if matches!(
1702 git::worktree_remove_clean(repo, &c.worktree).await,
1703 Ok(true)
1704 ) {
1705 git::branch_delete(repo, &c.branch).await.ok();
1706 } else {
1707 self.state.candidates.push(c);
1708 }
1709 }
1710 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1711 for j in 1..=seats {
1712 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1713 .await
1714 .ok();
1715 }
1716 for k in 1..=self.state.config.graph.advisors {
1717 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1718 .await
1719 .ok();
1720 }
1721 }
1722
1723 async fn prep(&mut self) -> Result<()> {
1724 if let Some(reason) = &self.state.prep_failure {
1725 bail!(
1726 "candidate preparation previously failed: {reason}; correct the source before retrying"
1727 );
1728 }
1729 if self
1730 .state
1731 .prep_complete
1732 .unwrap_or(!self.state.candidates.is_empty())
1733 {
1734 return Ok(());
1735 }
1736 let repo = self.state.repo.clone();
1737 let _coordination = self.coordinate(&repo).await?;
1740 self.rollback_prep(&repo).await;
1741 if !self.state.candidates.is_empty() {
1742 bail!(
1743 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1744 );
1745 }
1746 self.state.prep_complete = Some(false);
1747 self.state.status = RunStatus::Prep;
1748 self.state.save()?;
1749 let repo = self.state.repo.clone();
1750 let base = self.state.base_commit.clone();
1751 let plan = refs::plan(&repo, &self.state.seeds).await?;
1752 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1753 let root = self.state.worktree_root();
1754 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1755
1756 let hooks_dir = self.state.dir().join("hooks");
1759 if self.state.config.blind.commit_msg_hook {
1760 std::fs::create_dir_all(&hooks_dir)
1761 .with_context(|| format!("create {}", hooks_dir.display()))?;
1762 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1763 let path = hooks_dir.join("commit-msg");
1764 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1765 make_executable(&path)?;
1766 git::acquire_worktree_config(&repo).await?;
1774 self.state.enabled_worktree_config = true;
1775 }
1776
1777 ensure_setup_config(&mut self.state, &repo).await?;
1780
1781 for (index, (spec, label)) in self
1782 .roles
1783 .implementers
1784 .clone()
1785 .into_iter()
1786 .zip(labels)
1787 .enumerate()
1788 {
1789 let branch = self.state.branch_for(label);
1790 let worktree = root.join(format!("cand-{label}"));
1791 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1792 if self.state.config.blind.commit_msg_hook {
1793 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1794 }
1795 git::local_exclude(&worktree, "/.magi/").await?;
1796 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1797 git::worktree_remove_clean(&repo, &worktree).await.ok();
1800 git::branch_delete(&repo, &branch).await.ok();
1801 self.rollback_prep(&repo).await;
1802 self.state.status = RunStatus::Blocked;
1803 self.state
1804 .event("prep", format!("worktree setup failed: {e:#}"));
1805 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1806 self.state.save()?;
1807 return Err(e);
1808 }
1809 self.state.candidates.push(Candidate {
1810 index,
1811 label,
1812 agent: spec.id.clone(),
1813 branch,
1814 worktree,
1815 summary: String::new(),
1816 stat: String::new(),
1817 files: 0,
1818 commits: 0,
1819 empty: false,
1820 failed: None,
1821 verified_noop: None,
1822 duration_ms: 0,
1823 folded: false,
1824 });
1825 self.state.save()?;
1826 for pick in &plan.picks {
1827 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1828 {
1829 self.state.status = RunStatus::Blocked;
1830 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1831 self.state
1832 .event("prep", format!("cannot apply referenced commit: {e}"));
1833 self.state.save()?;
1834 return Err(e);
1835 }
1836 }
1837 }
1838
1839 for j in 1..=self.roles.judges.len() {
1840 let wt = root.join(format!("judge-{j}"));
1841 if !wt.exists() {
1842 git::worktree_add_detached(&repo, &wt, &base).await?;
1843 self.setup_seat_worktree(&repo, &wt, true).await?;
1844 }
1845 }
1846
1847 if self.state.config.graph.advise {
1855 for k in 1..=self.state.config.graph.advisors {
1856 let wt = root.join(format!("advisor-{k}"));
1857 if !wt.exists() {
1858 git::worktree_add_detached(&repo, &wt, &base).await?;
1859 self.setup_seat_worktree(&repo, &wt, true).await?;
1860 }
1861 }
1862 }
1863
1864 let authors: Vec<&str> = self
1869 .roles
1870 .implementers
1871 .iter()
1872 .map(|a| a.id.as_str())
1873 .collect();
1874 let overlap: Vec<String> = self
1875 .roles
1876 .judges
1877 .iter()
1878 .enumerate()
1879 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1880 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1881 .collect();
1882 if !overlap.is_empty() {
1883 let note = format!(
1884 "{} also authored a candidate; blind, but the panel is less \
1885 independent than {} distinct agents would be",
1886 overlap.join(", "),
1887 self.roles.judges.len()
1888 );
1889 self.state.event("prep", note);
1890 }
1891
1892 self.state.event(
1893 "prep",
1894 format!(
1895 "{} candidates, {} judges, base {} ({})",
1896 self.state.candidates.len(),
1897 self.roles.judges.len(),
1898 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1899 self.state.base_branch
1900 ),
1901 );
1902 self.state.prep_complete = Some(true);
1903 self.state.status = RunStatus::Implementing;
1904 self.state.save()?;
1905 Ok(())
1906 }
1907
1908 async fn advise(&mut self) -> Result<()> {
1941 let implement_untouched = self
1942 .state
1943 .candidates
1944 .iter()
1945 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1946 if !self.state.config.graph.advise || self.state.advise_attempted {
1947 return Ok(());
1948 }
1949 if !implement_untouched {
1950 self.state.event(
1951 "advise",
1952 "skipping the design-deliberation stage: at least one \
1953 candidate already shows implementation progress, so this \
1954 run is past the point the stage exists to run before"
1955 .to_owned(),
1956 );
1957 self.state.advise_attempted = true;
1958 self.state.save()?;
1959 return Ok(());
1960 }
1961 let run_id = self.state.id.clone();
1962 let prompts = self.state.config.prompts.clone();
1963 let instruction = self.state.instruction.clone();
1964 let language = self.state.config.graph.language.clone();
1965 let root = self.state.worktree_root();
1966 let n = self.state.config.graph.advisors;
1967 let where_recorded = self.state.dir().join("run.json");
1968
1969 let seats = match self.state.config.advisors() {
1970 Ok(seats) if !seats.is_empty() => seats,
1971 Ok(_) => {
1972 self.state.event(
1973 "advise",
1974 format!(
1975 "[graph] advisors is 0; skipping the design-deliberation \
1976 stage and continuing without a synthesis brief (see {})",
1977 where_recorded.display()
1978 ),
1979 );
1980 self.state.advise_attempted = true;
1981 self.state.save()?;
1982 return Ok(());
1983 }
1984 Err(e) => {
1985 self.state.event(
1986 "advise",
1987 format!(
1988 "could not resolve advisor seats ({e:#}); continuing \
1989 without a design-deliberation brief (see {})",
1990 where_recorded.display()
1991 ),
1992 );
1993 self.state.advise_attempted = true;
1994 self.state.save()?;
1995 return Ok(());
1996 }
1997 };
1998
1999 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2000 let artifacts = agent::artifacts_dir(&self.state.dir());
2001 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2002
2003 let mut jobs = Vec::new();
2004 for (i, spec) in seats.iter().cloned().enumerate() {
2005 let seat_key = format!("advisor-{}", i + 1);
2006 let seat = self.seat(&seat_key, &spec.id);
2007 jobs.push(SeatJob {
2008 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2009 spec,
2010 seat,
2011 cwd: worktrees[i % worktrees.len()].clone(),
2012 timeout,
2013 allow_write: false,
2014 sessions: false,
2015 artifacts: artifacts.clone(),
2016 stem: seat_key,
2017 handover: None,
2018 });
2019 }
2020
2021 self.state.event(
2022 "advise",
2023 format!(
2024 "{} advisor seat(s) sketching a design in parallel",
2025 jobs.len()
2026 ),
2027 );
2028 let mut quota_losses = Vec::new();
2029 let cache = self.state.config.cache_dir();
2030 let ctx = WaveCtx {
2031 carry_seats: false,
2032 pauses: Some((&self.pause, &self.interrupt)),
2033 run: &run_id,
2034 node: "advise",
2035 prompts: &prompts,
2036 cache: cache.as_deref(),
2037 round: None,
2038 };
2039 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2040 let results = ask_json_wave::<Proposal>(
2041 jobs,
2042 Arc::clone(&self.sem),
2043 self.state.config.graph.retries,
2044 &advisor_roster,
2045 &ctx,
2046 &mut quota_losses,
2047 &mut self.state,
2048 &|p: &Proposal| p.validate(),
2049 )
2050 .await;
2051 if self.state.resource_block.is_some() {
2052 return self.stop_for_resource();
2053 }
2054 self.state.quota.extend(quota_losses);
2055
2056 let mut records = Vec::with_capacity(results.len());
2057 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2058 let agent_id = seat.agent.clone();
2059 self.state.seats.insert(seat.key.clone(), seat);
2060 match res {
2061 Ok((proposal, out)) => {
2062 self.state
2063 .event("advise", format!("advisor-{} proposed a design", i + 1));
2064 records.push(advise::AdvisorRecord::proposed(
2065 i + 1,
2066 agent_id,
2067 proposal,
2068 out.duration_ms,
2069 ));
2070 }
2071 Err(e) => {
2072 self.state.event(
2073 "advise",
2074 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2075 );
2076 records.push(advise::AdvisorRecord::failed(
2077 i + 1,
2078 agent_id,
2079 e.to_string(),
2080 ));
2081 }
2082 }
2083 }
2084
2085 let mut advice = advise::Advice {
2086 records,
2087 synthesis: None,
2088 };
2089 if advice.proposals().is_empty() {
2090 self.state.event(
2091 "advise",
2092 "no advisor produced a usable proposal; continuing without a \
2093 synthesis brief"
2094 .to_owned(),
2095 );
2096 } else {
2097 match self
2098 .synthesize_brief(
2099 &advice,
2100 &instruction,
2101 &language,
2102 &worktrees[0],
2103 &artifacts,
2104 &run_id,
2105 &prompts,
2106 cache.as_deref(),
2107 )
2108 .await
2109 {
2110 Ok(Some(text)) => {
2111 self.state.event(
2112 "advise",
2113 "synthesized a design brief for the implementer".to_owned(),
2114 );
2115 advice.synthesis = Some(text);
2116 }
2117 Ok(None) => {
2118 self.state.event(
2119 "advise",
2120 "the synthesis seat produced nothing usable; continuing \
2121 without a design brief"
2122 .to_owned(),
2123 );
2124 }
2125 Err(e) => {
2126 if e.is::<crate::repo_lock::ResourceBlocked>() {
2127 return Err(e);
2128 }
2129 self.state.event(
2130 "advise",
2131 format!("could not synthesize a design brief: {e:#}"),
2132 );
2133 }
2134 }
2135 }
2136 advise::apply_reflection(&mut advice);
2137
2138 self.state.advice = Some(advice);
2139 self.state.advise_attempted = true;
2140 self.state.save()?;
2141 Ok(())
2142 }
2143
2144 #[allow(clippy::too_many_arguments)]
2156 async fn synthesize_brief(
2157 &mut self,
2158 advice: &advise::Advice,
2159 instruction: &str,
2160 language: &str,
2161 cwd: &Path,
2162 artifacts: &Path,
2163 run_id: &str,
2164 prompts: &Prompts,
2165 cache: Option<&Path>,
2166 ) -> Result<Option<String>> {
2167 let chain = agent::pick_chain(
2168 &self.state.config.agents,
2169 self.state.config.roles.synthesizer.as_ref(),
2170 &agent::installed,
2171 "synthesizer",
2172 )?;
2173 let proposals = advice.proposals();
2174 let mut prompt = prompt::with_overlay(
2175 prompt::synthesize_brief(instruction, &proposals, language),
2176 prompts.overlay("advise"),
2177 );
2178 if cache.is_some() {
2179 prompt.push('\n');
2184 prompt.push_str(&prompt::build_cache_note("advise", false));
2185 }
2186 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2187 let mut last = None;
2192 for (n, spec) in chain.iter().enumerate() {
2193 if n > 0 {
2194 self.state
2195 .event("advise", format!("synthesis falling back to {}", spec.id));
2196 }
2197 let mut seat = self.seat("advise-synthesis", &spec.id);
2198 let wait = self.repository_wait();
2199 let outcome = wait
2200 .scope(agent::invoke(
2201 spec,
2202 &mut seat,
2203 &Invocation {
2204 scope: crate::agent::InvocationScope::Repository,
2205 coordination: self.state.config.coordination.as_ref(),
2206 fetched_base: None,
2207 cwd,
2208 prompt: &prompt,
2209 timeout,
2210 allow_write: false,
2211 unsandboxed: false,
2212 sessions: false,
2213 artifacts,
2214 stem: &if n == 0 {
2215 "advise-synthesis".to_owned()
2216 } else {
2217 format!("advise-synthesis-{}", spec.id)
2218 },
2219 run: run_id,
2220 node: "advise",
2221 cache_dir: None,
2222 attachments: &[],
2223 writable: &[],
2224 },
2225 ))
2226 .await;
2227 if outcome.is_ok() {
2228 self.state.seats.insert(seat.key.clone(), seat);
2229 }
2230 let advance = agent::chain_advances(&outcome);
2231 last = Some(outcome);
2232 if !advance {
2233 break;
2234 }
2235 }
2236 let out = last.expect("a chain holds at least one agent")?;
2239 if !out.usable() {
2240 return Ok(None);
2241 }
2242 let text =
2243 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2244 Ok((!text.trim().is_empty()).then_some(text))
2245 }
2246
2247 async fn implement(&mut self) -> Result<()> {
2250 self.persist_candidates().await?;
2251 let run_id = self.state.id.clone();
2256 let prompts = self.state.config.prompts.clone();
2257 let todo: Vec<usize> = self
2258 .state
2259 .candidates
2260 .iter()
2261 .enumerate()
2262 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2263 .map(|(i, _)| i)
2264 .collect();
2265 if todo.is_empty() {
2266 return self.after_implement();
2267 }
2268 self.state.status = RunStatus::Implementing;
2269
2270 let language = self.state.config.graph.language.clone();
2271 let github_language = self.state.config.graph.github_language.clone();
2272 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2273 let sessions = self.state.config.graph.sessions;
2274 let artifacts = agent::artifacts_dir(&self.state.dir());
2275 let brief = self
2279 .state
2280 .advice
2281 .as_ref()
2282 .and_then(|a| a.synthesis.as_deref())
2283 .map(str::to_owned);
2284 let attachments = self.state.attachments.clone();
2285
2286 let mut jobs = Vec::new();
2287 for &i in &todo {
2288 let (index, label, worktree) = {
2289 let c = &self.state.candidates[i];
2290 (c.index, c.label, c.worktree.clone())
2291 };
2292 let spec = self.roles.implementers[index].clone();
2293 let seat_key = format!("impl-{label}");
2294 let seat = self.seat(&seat_key, &spec.id);
2295 let instruction = seeded_instruction(&self.state);
2296 jobs.push(SeatJob {
2297 spec,
2298 seat,
2299 prompt: prompt::implement_in(
2300 &instruction,
2301 &worktree.to_string_lossy(),
2302 &language,
2303 &github_language,
2304 brief.as_deref(),
2305 &attachments,
2306 ),
2307 cwd: worktree,
2308 timeout,
2309 allow_write: true,
2310 sessions,
2311 artifacts: artifacts.clone(),
2312 stem: format!("impl-{label}"),
2313 handover: None,
2314 });
2315 }
2316
2317 self.state.event(
2318 "implement",
2319 format!("{} candidates in parallel", jobs.len()),
2320 );
2321 let mut sent = jobs.clone();
2327 let cache = self.state.config.cache_dir();
2328 let ctx = WaveCtx {
2329 carry_seats: false,
2330 pauses: Some((&self.pause, &self.interrupt)),
2331 run: &run_id,
2332 node: "implement",
2333 prompts: &prompts,
2334 cache: cache.as_deref(),
2335 round: None,
2336 };
2337 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2338 if !wave_blocked(&results) {
2339 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2340 .await;
2341 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2342 .await?;
2343 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2344 .await;
2345 }
2346 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2347
2348 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2349 && results
2350 .iter()
2351 .any(|(_, _, out)| FailClass::of(out).is_some());
2352 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2353 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2354 continue;
2355 }
2356 let seat_key = seat.key.clone();
2357 let agent = seat.agent.clone();
2366 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2367 self.state.seats.insert(seat.key.clone(), seat);
2368 let label = self.state.candidates[i].label;
2369
2370 let (summary, duration, failed, verified_claim) = match out {
2371 AgentOutcome::Ok(o) => {
2372 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2373 let failed = (!o.usable()).then(|| {
2374 if o.timed_out {
2375 "agent timed out".to_owned()
2376 } else {
2377 format!(
2378 "agent exited with {}",
2379 crate::notices::exit_status(o.exit_code)
2380 )
2381 }
2382 });
2383 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2384 (text, o.duration_ms, failed, verified_claim)
2385 }
2386 AgentOutcome::Dropped(o) => {
2392 let why = o
2393 .dropped
2394 .as_ref()
2395 .map(|d| d.why.as_str())
2396 .unwrap_or("the CLI ended the stream without delivering its answer");
2397 (
2398 String::new(),
2399 o.duration_ms,
2400 Some(format!("the CLI dropped the stream ({why})")),
2401 None,
2402 )
2403 }
2404 AgentOutcome::Quota(o) => {
2405 self.state.quota.push(QuotaLoss {
2406 seat: seat_key,
2407 node: "implement".to_owned(),
2408 at: Timestamp::now(),
2409 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2410 });
2411 (
2412 String::new(),
2413 o.duration_ms,
2414 Some("rate limited (quota); produced no change".to_owned()),
2415 None,
2416 )
2417 }
2418 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2419 (String::new(), 0, Some(e), None)
2420 }
2421 };
2422
2423 let c = &mut self.state.candidates[i];
2426 if !exhausted_the_fallback_chain {
2427 c.agent = agent;
2428 }
2429 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2430 c.duration_ms = duration;
2431 c.failed = failed;
2432 c.verified_noop = verified_claim;
2433 self.state
2434 .candidate_preservation
2435 .insert(label, "awaiting persistence".to_owned());
2436 }
2437 self.state.save()?;
2438 self.persist_candidates().await?;
2439
2440 if resource_blocked {
2441 return self.stop_for_resource();
2442 }
2443 if selected_failed {
2444 self.state.status = RunStatus::Blocked;
2445 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2446 self.state.save()?;
2447 bail!(
2448 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2449 );
2450 }
2451 self.after_implement()
2452 }
2453
2454 async fn persist_candidates(&mut self) -> Result<()> {
2456 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2457 for label in labels {
2458 let i = self
2459 .state
2460 .candidates
2461 .iter()
2462 .position(|c| c.label == label)
2463 .context("pending candidate is missing")?;
2464 let c = self.state.candidates[i].clone();
2465 let result = async {
2466 let wait = crate::repo_lock::WaitPolicy::new(
2467 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2468 || false,
2469 );
2470 let _coordination =
2471 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2472 if let Err(e) = git::snapshot_uncommitted(
2475 &c.worktree,
2476 &self
2477 .state
2478 .dir()
2479 .join("artifacts")
2480 .join(format!("candidate-{label}-recovery")),
2481 )
2482 .await
2483 {
2484 self.state.event(
2485 "preserve",
2486 format!("candidate {label} snapshot incomplete: {e:#}"),
2487 );
2488 }
2489 let rescued = git::rescue_commit(
2490 &c.worktree,
2491 &format!("magi: candidate {label} (uncommitted work)"),
2492 )
2493 .await?;
2494 self.state.note_withheld("implement", &rescued.withheld);
2495 let base = &self.state.base_commit;
2496 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2497 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2498 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2499 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2500 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2501 if c.empty && commits > 0 && !patch.trim().is_empty()
2502 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2503 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2504 || self.state.tally.is_some())
2505 {
2506 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2510 }
2511 let c = &mut self.state.candidates[i];
2512 c.commits = commits;
2513 c.stat = stat;
2514 c.files = files;
2515 c.empty = commits == 0 || patch.trim().is_empty();
2516 if !c.empty {
2517 c.failed = None;
2518 c.verified_noop = None;
2519 }
2520 self.state.event(
2521 "preserve",
2522 format!("candidate {label}: {files} files, {commits} commits"),
2523 );
2524 Ok::<_, anyhow::Error>(())
2525 }
2526 .await;
2527 match result {
2528 Ok(()) => {
2529 self.state.candidate_preservation.remove(&label);
2530 }
2531 Err(e) => {
2532 let reason = format!("candidate {label} preservation failed: {e:#}");
2533 self.state.event("preserve", &reason);
2534 self.state.candidate_preservation.insert(label, reason);
2535 }
2536 }
2537 if self.state.candidate_preservation.is_empty()
2538 && self.state.status == RunStatus::Blocked
2539 && self.state.candidates.iter().all(|c| c.failed.is_none())
2540 {
2541 self.state.status = RunStatus::Implementing;
2545 self.state.event(
2546 "preserve",
2547 "candidate persistence recovered; implementation can continue",
2548 );
2549 }
2550 self.state.save()?;
2551 }
2552 if !self.state.candidate_preservation.is_empty() {
2553 self.state.status = RunStatus::Blocked;
2554 self.state.save()?;
2555 bail!(
2556 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2557 );
2558 }
2559 Ok(())
2560 }
2561
2562 async fn resume_undelivered(
2590 &mut self,
2591 results: &mut [(usize, SeatState, AgentOutcome)],
2592 sent: &[SeatJob],
2593 prompts: &Prompts,
2594 run_id: &str,
2595 ) {
2596 for (wi, seat, out) in results.iter_mut() {
2597 let Some(dropped) = (match &*out {
2598 AgentOutcome::Dropped(o) => o.dropped.clone(),
2599 _ => None,
2600 }) else {
2601 continue;
2602 };
2603 let Some(job) = sent.get(*wi) else { continue };
2604 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2606 self.state.event(
2607 "implement",
2608 format!(
2609 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2610 work is in the tree",
2611 seat.key, dropped.output_tokens, dropped.why
2612 ),
2613 );
2614 continue;
2615 }
2616 if !has_context(&job.spec, seat, job.sessions) {
2624 self.state.event(
2625 "implement",
2626 format!(
2627 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2628 is no session left to resume",
2629 seat.key, dropped.output_tokens, dropped.why
2630 ),
2631 );
2632 continue;
2633 }
2634 self.state.event(
2635 "implement",
2636 format!(
2637 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2638 conversation",
2639 seat.key, dropped.output_tokens, dropped.why
2640 ),
2641 );
2642 let mut retry = job.clone();
2643 retry.seat = seat.clone();
2644 retry.prompt = prompt::resume_after_drop(&dropped.why);
2645 retry.timeout = retry_budget(job.timeout, true);
2646 retry.stem = format!("{}-resume", job.stem);
2647 let cache = self.state.config.cache_dir();
2648 let ctx = WaveCtx {
2649 carry_seats: false,
2650 pauses: Some((&self.pause, &self.interrupt)),
2651 run: run_id,
2652 node: "implement",
2653 prompts,
2654 cache: cache.as_deref(),
2655 round: None,
2656 };
2657 let (resumed_seat, resumed) =
2658 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2659 *seat = resumed_seat;
2660 *out = resumed;
2661 }
2662 }
2663
2664 async fn resume_seat_handovers(
2729 &mut self,
2730 results: &mut [(usize, SeatState, AgentOutcome)],
2731 sent: &mut [SeatJob],
2732 prompts: &Prompts,
2733 run_id: &str,
2734 ) -> Result<()> {
2735 let instruction = seeded_instruction(&self.state);
2736 let language = self.state.config.graph.language.clone();
2737 let github_language = self.state.config.graph.github_language.clone();
2738 let brief = self
2739 .state
2740 .advice
2741 .as_ref()
2742 .and_then(|a| a.synthesis.as_deref())
2743 .map(str::to_owned);
2744 let attachments = self.state.attachments.clone();
2745 for (wi, seat, out) in results.iter_mut() {
2746 let others: BTreeSet<String> = sent
2749 .iter()
2750 .enumerate()
2751 .filter(|(j, _)| j != wi)
2752 .map(|(_, j)| j.spec.id.clone())
2753 .collect();
2754 let Some(job) = sent.get_mut(*wi) else {
2755 continue;
2756 };
2757 let start = self
2762 .roles
2763 .implementer_roster
2764 .iter()
2765 .position(|s| s.id == job.spec.id)
2766 .unwrap_or(0);
2767 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2768 let mut fallback_attempt = 0usize;
2769 let mut prev: Option<FailClass> = None;
2770 while let Some(cur) = FailClass::of(&*out) {
2771 if !should_hand_over(prev.as_ref(), &cur) {
2772 break;
2773 }
2774 let Some(next) =
2775 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2776 .cloned()
2777 else {
2778 break;
2779 };
2780 tried.insert(next.id.clone());
2781 fallback_attempt += 1;
2782
2783 let coordination = self.coordinate(&job.cwd).await?;
2784 if let Ok(r) = git::rescue_commit(
2785 &job.cwd,
2786 &format!(
2787 "magi: candidate {} (uncommitted work before {} fallback)",
2788 seat.key,
2789 if cur == FailClass::Quota {
2790 "quota"
2791 } else {
2792 "handover"
2793 }
2794 ),
2795 )
2796 .await
2797 {
2798 self.state.note_withheld("implement", &r.withheld);
2799 }
2800 drop(coordination);
2801
2802 record_handover(
2803 &mut self.state,
2804 "implement",
2805 &seat.key,
2806 &seat.agent,
2807 &next.id,
2808 &cur,
2809 &fail_reason(&*out),
2810 );
2811 prev = Some(cur.clone());
2812
2813 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2814 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2815 job.spec = next.clone();
2823 let mut retry = job.clone();
2824 retry.seat = new_seat;
2825 retry.prompt = prompt::implement_in(
2826 &instruction,
2827 &job.cwd.to_string_lossy(),
2828 &language,
2829 &github_language,
2830 brief.as_deref(),
2831 &attachments,
2832 );
2833 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2834 let cache = self.state.config.cache_dir();
2835 let ctx = WaveCtx {
2836 carry_seats: false,
2837 pauses: Some((&self.pause, &self.interrupt)),
2838 run: run_id,
2839 node: "implement",
2840 prompts,
2841 cache: cache.as_deref(),
2842 round: None,
2843 };
2844 let (fallback_seat, fallback_out) = run_one(
2845 retry,
2846 Arc::clone(&self.sem),
2847 &ctx,
2848 &mut self.state,
2849 fallback_attempt,
2850 )
2851 .await;
2852 *seat = fallback_seat;
2853 *out = fallback_out;
2854 }
2855 }
2856 Ok(())
2857 }
2858
2859 async fn resume_unconfirmed_commands(
2883 &mut self,
2884 results: &mut [(usize, SeatState, AgentOutcome)],
2885 sent: &[SeatJob],
2886 prompts: &Prompts,
2887 run_id: &str,
2888 ) {
2889 for (wi, seat, out) in results.iter_mut() {
2890 let AgentOutcome::Ok(o) = &*out else {
2891 continue;
2892 };
2893 if !has_unconfirmed_command(&o.commands) {
2894 continue;
2895 }
2896 let Some(job) = sent.get(*wi) else { continue };
2897 if !has_context(&job.spec, seat, job.sessions) {
2898 self.state.event(
2899 "implement",
2900 format!(
2901 "{}: the reply named a command whose own CLI never confirmed the exit \
2902 status of, but there is no session left to resume",
2903 seat.key
2904 ),
2905 );
2906 continue;
2907 }
2908 self.state.event(
2909 "implement",
2910 format!(
2911 "{}: the reply named a command whose own CLI never confirmed the exit \
2912 status of; resuming the conversation",
2913 seat.key
2914 ),
2915 );
2916 let mut retry = job.clone();
2917 retry.seat = seat.clone();
2918 retry.prompt = prompt::resume_incomplete(
2919 "a command in your last reply had no confirmed exit status",
2920 );
2921 retry.timeout = retry_budget(job.timeout, true);
2922 retry.stem = format!("{}-confirm", job.stem);
2923 let cache = self.state.config.cache_dir();
2924 let ctx = WaveCtx {
2925 carry_seats: false,
2926 pauses: Some((&self.pause, &self.interrupt)),
2927 run: run_id,
2928 node: "implement",
2929 prompts,
2930 cache: cache.as_deref(),
2931 round: None,
2932 };
2933 let (resumed_seat, resumed) =
2934 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2935 *seat = resumed_seat;
2936 *out = resumed;
2937 }
2938 }
2939
2940 async fn continue_fix_report(
2961 &mut self,
2962 mut seat: SeatState,
2963 parse_err: String,
2964 job: &SeatJob,
2965 prompts: &Prompts,
2966 run_id: &str,
2967 round: usize,
2968 ) -> (
2969 SeatState,
2970 Option<FixReport>,
2971 Option<String>,
2972 ContinuationRecord,
2973 ) {
2974 let mut last_err = parse_err;
2975 let mut cumulative_wait_ms = 0u64;
2976 let mut attempts = 0usize;
2977 loop {
2978 if !has_context(&job.spec, &seat, job.sessions) {
2979 self.state.event(
2980 "fix",
2981 format!(
2982 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2983 session left to resume into"
2984 ),
2985 );
2986 let outcome = if attempts == 0 {
2987 ContinuationOutcome::NoSession
2988 } else {
2989 ContinuationOutcome::Exhausted
2990 };
2991 return (
2992 seat,
2993 None,
2994 Some(format!("unparsable fix report: {last_err}")),
2995 ContinuationRecord {
2996 attempts,
2997 cumulative_wait_ms,
2998 outcome,
2999 },
3000 );
3001 }
3002 if attempts >= MAX_FIX_CONTINUATIONS {
3003 self.state.event(
3004 "fix",
3005 format!(
3006 "round {round}: fixer's reply still had no adoption report after \
3007 {attempts} continuation(s) ({last_err}); giving up"
3008 ),
3009 );
3010 return (
3011 seat,
3012 None,
3013 Some(format!(
3014 "unparsable fix report after {attempts} continuation(s): {last_err}"
3015 )),
3016 ContinuationRecord {
3017 attempts,
3018 cumulative_wait_ms,
3019 outcome: ContinuationOutcome::Exhausted,
3020 },
3021 );
3022 }
3023 attempts += 1;
3024 self.state.event(
3025 "fix",
3026 format!(
3027 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3028 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3029 ),
3030 );
3031 let mut retry = job.clone();
3032 retry.seat = seat.clone();
3033 retry.prompt = prompt::resume_incomplete(&last_err);
3034 retry.timeout = retry_budget(job.timeout, true);
3035 retry.stem = format!("{}-continue{attempts}", job.stem);
3036 let cache = self.state.config.cache_dir();
3037 let ctx = WaveCtx {
3038 carry_seats: false,
3039 pauses: Some((&self.pause, &self.interrupt)),
3040 run: run_id,
3041 node: "fix",
3042 prompts,
3043 cache: cache.as_deref(),
3044 round: Some(round),
3045 };
3046 let (resumed_seat, resumed_out) = run_one(
3047 retry,
3048 Arc::clone(&self.sem),
3049 &ctx,
3050 &mut self.state,
3051 attempts,
3052 )
3053 .await;
3054 seat = resumed_seat;
3055 match resumed_out {
3056 AgentOutcome::Ok(o) => {
3057 cumulative_wait_ms += o.duration_ms;
3058 match verdict::extract_json::<FixReport>(&o.text) {
3059 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3060 self.state.event(
3061 "fix",
3062 format!(
3063 "round {round}: fixer's adoption report recovered after \
3064 {attempts} continuation(s)"
3065 ),
3066 );
3067 return (
3068 seat,
3069 Some(report),
3070 None,
3071 ContinuationRecord {
3072 attempts,
3073 cumulative_wait_ms,
3074 outcome: ContinuationOutcome::Resumed,
3075 },
3076 );
3077 }
3078 Ok(_) => {
3086 last_err = "the reply parsed, but it reported a command whose own CLI \
3087 never confirmed an exit status"
3088 .to_owned();
3089 }
3090 Err(e) => last_err = e.to_string(),
3091 }
3092 }
3093 AgentOutcome::Quota(o) => {
3094 cumulative_wait_ms += o.duration_ms;
3095 self.state.quota.push(QuotaLoss {
3096 seat: seat.key.clone(),
3097 node: "fix".to_owned(),
3098 at: Timestamp::now(),
3099 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3100 });
3101 self.state.event(
3102 "fix",
3103 format!(
3104 "round {round}: continuation rate limited (quota); not retrying now"
3105 ),
3106 );
3107 return (
3108 seat,
3109 None,
3110 Some("rate limited (quota) while recovering the fix report".to_owned()),
3111 ContinuationRecord {
3112 attempts,
3113 cumulative_wait_ms,
3114 outcome: ContinuationOutcome::QuotaLost,
3115 },
3116 );
3117 }
3118 AgentOutcome::Dropped(o) => {
3119 cumulative_wait_ms += o.duration_ms;
3120 let why = o
3121 .dropped
3122 .as_ref()
3123 .map(|d| d.why.as_str())
3124 .unwrap_or("the CLI ended the stream without delivering its answer");
3125 last_err = format!("the CLI dropped the stream ({why})");
3126 }
3127 AgentOutcome::Failed(e) => last_err = e,
3128 AgentOutcome::ResourceBlocked(e) => {
3131 return (
3132 seat,
3133 None,
3134 Some(format!("the continuation was not started: {e}")),
3135 ContinuationRecord {
3136 attempts: attempts.saturating_sub(1),
3137 cumulative_wait_ms,
3138 outcome: ContinuationOutcome::NoSession,
3139 },
3140 );
3141 }
3142 }
3143 }
3144 }
3145
3146 fn after_implement(&mut self) -> Result<()> {
3147 if !self.state.candidate_preservation.is_empty() {
3148 bail!("candidate preservation incomplete; judging is unsafe");
3149 }
3150 if self.state.leaks.is_empty() {
3152 let cfg = self.state.config.blind.clone();
3153 let mut leaks = Vec::new();
3154 for c in &self.state.candidates {
3155 let Some(patch) =
3156 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3157 else {
3158 continue;
3159 };
3160 leaks.extend(blind::scan(
3161 &format!("candidate {} patch", c.label),
3162 &patch,
3163 &cfg.vendor_tokens,
3164 ));
3165 }
3166 if !leaks.is_empty() {
3167 let summary = leaks
3168 .iter()
3169 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3170 .collect::<Vec<_>>()
3171 .join(", ");
3172 match cfg.on_leak {
3173 LeakPolicy::Fail => {
3174 self.state.status = RunStatus::Failed;
3175 self.state
3176 .event("blind", format!("vendor text in a patch: {summary}"));
3177 self.state.leaks = leaks;
3178 self.state.save()?;
3179 self.settle_questions();
3180 bail!(
3181 "blind.on_leak = \"fail\" and vendor text reached a \
3182 judged patch: {summary}"
3183 );
3184 }
3185 LeakPolicy::Redact => self.state.event(
3186 "blind",
3187 format!("redacting vendor text for judging: {summary}"),
3188 ),
3189 LeakPolicy::Warn => self.state.event(
3190 "blind",
3191 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3192 ),
3193 }
3194 self.state.leaks = leaks;
3195 }
3196 }
3197
3198 if self.state.viable().is_empty() {
3199 if self.state.all_candidates_verified_noop() {
3200 self.state.status = RunStatus::VerifiedNoop;
3211 self.state.save()?;
3212 self.settle_questions();
3213 return Ok(());
3214 }
3215 self.state.status = RunStatus::Failed;
3216 self.state.save()?;
3217 self.settle_questions();
3218 bail!("no candidate produced a change; nothing to judge");
3219 }
3220 self.state.status = RunStatus::Judging;
3221 self.state.save()?;
3222 Ok(())
3223 }
3224
3225 async fn judge(&mut self) -> Result<()> {
3228 let run_id = self.state.id.clone();
3233 let prompts = self.state.config.prompts.clone();
3234 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3235 return Ok(());
3236 }
3237 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3238 if viable.len() == 1 {
3239 self.state.judge_skipped = true;
3246 self.state.event(
3247 "judge",
3248 format!(
3249 "only candidate {} produced a change; judging skipped",
3250 viable[0].label
3251 ),
3252 );
3253 self.state.save()?;
3254 return Ok(());
3255 }
3256 self.state.status = RunStatus::Judging;
3257
3258 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3259 let language = self.state.config.graph.language.clone();
3260 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3261 let sessions = self.state.config.graph.sessions;
3262 let artifacts = agent::artifacts_dir(&self.state.dir());
3263 let root = self.state.worktree_root();
3264 let base_short = short(&self.state.base_commit);
3265
3266 let mut jobs = Vec::new();
3267 let mut orders = Vec::new();
3268 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3269 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3270 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3271 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3272 let seat_key = format!("judge-{}", j + 1);
3273 let seat = self.seat(&seat_key, &spec.id);
3274 jobs.push(SeatJob {
3275 prompt: prompt::judge(
3276 &self.state.instruction,
3277 &views,
3278 self.roles.judges.len(),
3279 &base_short,
3280 &language,
3281 ),
3282 spec,
3283 seat,
3284 cwd: root.join(format!("judge-{}", j + 1)),
3285 timeout,
3286 allow_write: false,
3287 sessions,
3288 artifacts: artifacts.clone(),
3289 stem: format!("judge-{}", j + 1),
3290 handover: None,
3291 });
3292 }
3293
3294 self.state.event(
3295 "judge",
3296 format!(
3297 "{} judges ranking {} candidates blind",
3298 jobs.len(),
3299 viable.len()
3300 ),
3301 );
3302 let labels_for_check = labels.clone();
3303 let mut quota_losses = Vec::new();
3304 let cache = self.state.config.cache_dir();
3305 let ctx = WaveCtx {
3306 carry_seats: false,
3307 pauses: Some((&self.pause, &self.interrupt)),
3308 run: &run_id,
3309 node: "judge",
3310 prompts: &prompts,
3311 cache: cache.as_deref(),
3312 round: None,
3313 };
3314 let results = ask_json_wave::<Ranking>(
3315 jobs,
3316 Arc::clone(&self.sem),
3317 self.state.config.graph.retries,
3318 &self.roles.judge_roster,
3319 &ctx,
3320 &mut quota_losses,
3321 &mut self.state,
3322 &move |r: &Ranking| r.validate(&labels_for_check),
3323 )
3324 .await;
3325 if self.state.resource_block.is_some() {
3326 return self.stop_for_resource();
3327 }
3328 self.state.quota.extend(quota_losses);
3329
3330 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3331 let agent_id = seat.agent.clone();
3332 self.state.seats.insert(seat.key.clone(), seat);
3333 let mut record = Judgement {
3334 judge: j + 1,
3335 seat: format!("judge-{}", j + 1),
3336 agent: agent_id,
3337 ranking: Vec::new(),
3338 reasons: BTreeMap::new(),
3339 confidence: None,
3340 strengths: BTreeMap::new(),
3341 order: orders[j].clone(),
3342 failed: None,
3343 duration_ms: 0,
3344 };
3345 match res {
3346 Ok((ranking, out)) => {
3347 record.ranking = ranking.normalized();
3348 record.reasons = ranking.reasons;
3349 record.strengths = ranking.strengths;
3350 record.confidence = ranking.confidence;
3351 record.duration_ms = out.duration_ms;
3352 self.state.event(
3353 "judge",
3354 format!(
3355 "judge {} ranked {}",
3356 j + 1,
3357 record.ranking.iter().collect::<String>()
3358 ),
3359 );
3360 }
3361 Err(e) => {
3362 record.failed = Some(e.to_string());
3363 self.state
3364 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3365 }
3366 }
3367 self.state.judgements.push(record);
3368 self.state.save()?;
3369 }
3370 Ok(())
3371 }
3372
3373 async fn deliberate(&mut self) -> Result<()> {
3376 let run_id = self.state.id.clone();
3381 let prompts = self.state.config.prompts.clone();
3382 if !self.state.deliberation.is_empty() {
3383 return Ok(());
3384 }
3385 let tops: Vec<char> = self
3386 .state
3387 .judgements
3388 .iter()
3389 .filter_map(|j| j.ranking.first().copied())
3390 .collect();
3391 let rounds = self.state.config.graph.deliberate_rounds;
3392 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3393 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3394 self.state.event(
3395 "deliberate",
3396 format!("judges agreed on {} outright; no deliberation", tops[0]),
3397 );
3398 }
3399 self.state.status = RunStatus::Voting;
3400 self.state.save()?;
3401 return Ok(());
3402 }
3403
3404 self.state.status = RunStatus::Deliberating;
3405 self.state.event(
3406 "deliberate",
3407 format!(
3408 "split: first choices were {} — opening {rounds} round(s)",
3409 tops.iter().collect::<String>()
3410 ),
3411 );
3412
3413 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3414 let language = self.state.config.graph.language.clone();
3415 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3416 let sessions = self.state.config.graph.sessions;
3417 let artifacts = agent::artifacts_dir(&self.state.dir());
3418 let root = self.state.worktree_root();
3419 let base_short = short(&self.state.base_commit);
3420
3421 for round in 1..=rounds {
3425 let mut turns: Vec<DeliberationTurn> = Vec::new();
3426 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3427 if self.state.judgements[j].failed.is_some() {
3428 continue;
3429 }
3430 let seat_key = format!("judge-{}", j + 1);
3431 let spec = self.occupant(&seat_key, spec);
3432 let mut seat = self.seat(&seat_key, &spec.id);
3433 let transcript = self.transcript(&turns, j);
3434 let build = |context: Option<&str>| {
3435 prompt::deliberate(
3436 &self.state.instruction,
3437 context,
3438 &transcript,
3439 round,
3440 rounds,
3441 &language,
3442 )
3443 };
3444 let block = self.candidate_block(&viable, &base_short);
3445 let full = build(Some(&block));
3446 let text = if has_context(&spec, &seat, sessions) {
3447 build(None)
3448 } else {
3449 full.clone()
3450 };
3451 let job = SeatJob {
3452 spec,
3453 seat: seat.clone(),
3454 prompt: text,
3455 cwd: root.join(format!("judge-{}", j + 1)),
3456 timeout,
3457 allow_write: false,
3458 sessions,
3459 artifacts: artifacts.clone(),
3460 stem: format!("delib-{round}-judge-{}", j + 1),
3461 handover: Some(full),
3462 };
3463 let cache = self.state.config.cache_dir();
3464 let ctx = WaveCtx {
3465 carry_seats: false,
3466 pauses: Some((&self.pause, &self.interrupt)),
3467 run: &run_id,
3468 node: "deliberate",
3469 prompts: &prompts,
3470 cache: cache.as_deref(),
3471 round: None,
3472 };
3473 let mut losses = Vec::new();
3477 let mut results = ask_wave_with::<String>(
3478 vec![job],
3479 Arc::clone(&self.sem),
3480 0,
3481 &self.roles.judge_roster,
3482 &ctx,
3483 &mut losses,
3484 &mut self.state,
3485 &|text: &str| {
3486 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3487 },
3488 )
3489 .await;
3490 self.state.quota.extend(losses);
3491 let (updated, res, _) = results.pop().expect("one job in, one result out");
3492 seat = updated;
3493 let agent_id = seat.agent.clone();
3494 self.state.seats.insert(seat.key.clone(), seat);
3495 let body = match res {
3496 Ok((body, _)) => body,
3497 Err(e) => {
3500 self.state
3501 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3502 continue;
3503 }
3504 };
3505 let tentative = verdict::extract_json::<Position>(&body)
3506 .ok()
3507 .and_then(|p| p.tentative)
3508 .and_then(|s| s.trim().chars().next())
3509 .map(|c| c.to_ascii_uppercase());
3510 self.state.event(
3511 "deliberate",
3512 format!(
3513 "round {round}: judge {} now favours {}",
3514 j + 1,
3515 tentative.map_or("—".to_owned(), |c| c.to_string())
3516 ),
3517 );
3518 turns.push(DeliberationTurn {
3519 judge: j + 1,
3520 agent: agent_id,
3521 body: blind::sanitize_prose(&body, &self.state.config.blind),
3522 tentative,
3523 });
3524 }
3525 self.state
3526 .deliberation
3527 .push(DeliberationRound { round, turns });
3528 self.state.save()?;
3529 }
3530
3531 self.state.status = RunStatus::Voting;
3532 self.state.save()?;
3533 Ok(())
3534 }
3535
3536 async fn vote(&mut self) -> Result<()> {
3539 let run_id = self.state.id.clone();
3544 let prompts = self.state.config.prompts.clone();
3545 if !self.state.votes.is_empty() {
3546 return Ok(());
3547 }
3548 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3549 if viable.len() == 1 {
3550 return Ok(());
3551 }
3552 self.state.status = RunStatus::Voting;
3553
3554 let language = self.state.config.graph.language.clone();
3555 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3556 let sessions = self.state.config.graph.sessions;
3557 let artifacts = agent::artifacts_dir(&self.state.dir());
3558 let root = self.state.worktree_root();
3559 let base_short = short(&self.state.base_commit);
3560 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3561
3562 let mut jobs = Vec::new();
3563 let mut seats_at = Vec::new();
3564 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3565 if self
3566 .state
3567 .judgements
3568 .get(j)
3569 .is_some_and(|r| r.failed.is_some())
3570 {
3571 continue;
3572 }
3573 let seat_key = format!("judge-{}", j + 1);
3574 let spec = self.occupant(&seat_key, spec);
3575 let seat = self.seat(&seat_key, &spec.id);
3576 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3577 let text = if has_context(&spec, &seat, sessions) {
3578 prompt::final_vote(&viable, &language)
3579 } else {
3580 full.clone()
3581 };
3582 jobs.push(SeatJob {
3583 spec,
3584 seat,
3585 prompt: text,
3586 cwd: root.join(format!("judge-{}", j + 1)),
3587 timeout,
3588 allow_write: false,
3589 sessions,
3590 artifacts: artifacts.clone(),
3591 stem: format!("vote-judge-{}", j + 1),
3592 handover: Some(full),
3593 });
3594 seats_at.push(j);
3595 }
3596
3597 self.state.event(
3598 "vote",
3599 format!(
3600 "collecting {} final votes one by one, privately",
3601 jobs.len()
3602 ),
3603 );
3604 let allowed = viable.clone();
3605 let mut quota_losses = Vec::new();
3606 let cache = self.state.config.cache_dir();
3607 let ctx = WaveCtx {
3608 carry_seats: false,
3609 pauses: Some((&self.pause, &self.interrupt)),
3610 run: &run_id,
3611 node: "vote",
3612 prompts: &prompts,
3613 cache: cache.as_deref(),
3614 round: None,
3615 };
3616 let results = ask_json_wave::<FinalVote>(
3617 jobs,
3618 Arc::clone(&self.sem),
3619 self.state.config.graph.retries,
3620 &self.roles.judge_roster,
3621 &ctx,
3622 &mut quota_losses,
3623 &mut self.state,
3624 &move |v: &FinalVote| match v.label() {
3625 Some(c) if allowed.contains(&c) => Ok(()),
3626 other => bail!("vote {other:?} is not one of {allowed:?}"),
3627 },
3628 )
3629 .await;
3630 if self.state.resource_block.is_some() {
3631 return self.stop_for_resource();
3632 }
3633 self.state.quota.extend(quota_losses);
3634
3635 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3636 let agent_id = seat.agent.clone();
3637 self.state.seats.insert(seat.key.clone(), seat);
3638 let initial = self
3639 .state
3640 .judgements
3641 .get(j)
3642 .and_then(|r| r.ranking.first().copied());
3643 let mut record = VoteRecord {
3644 judge: j + 1,
3645 agent: agent_id,
3646 vote: None,
3647 reason: String::new(),
3648 changed: false,
3649 };
3650 match res {
3651 Ok((v, _)) => {
3652 record.vote = v.label();
3653 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3654 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3655 self.state.event(
3656 "vote",
3657 format!(
3658 "judge {} voted {}{}",
3659 j + 1,
3660 record.vote.unwrap_or('?'),
3661 if record.changed { " (changed)" } else { "" }
3662 ),
3663 );
3664 }
3665 Err(e) => {
3666 self.state
3667 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3668 }
3669 }
3670 self.state.votes.push(record);
3671 self.state.save()?;
3672 }
3673 Ok(())
3674 }
3675
3676 fn tally(&mut self) -> Result<()> {
3679 if self.state.tally.is_some() {
3680 return Ok(());
3681 }
3682 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3683 let tops: Vec<char> = self
3684 .state
3685 .judgements
3686 .iter()
3687 .filter_map(|j| j.ranking.first().copied())
3688 .collect();
3689 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3690
3691 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3694 let mut cast: Vec<char> = Vec::new();
3695 for (i, j) in self.state.judgements.iter().enumerate() {
3696 let vote = self
3697 .state
3698 .votes
3699 .iter()
3700 .find(|v| v.judge == i + 1)
3701 .and_then(|v| v.vote)
3702 .or_else(|| j.ranking.first().copied());
3703 if let Some(v) = vote {
3704 *first_choice.entry(v).or_insert(0) += 1;
3705 cast.push(v);
3706 }
3707 }
3708
3709 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3710 for j in &self.state.judgements {
3711 let n = j.ranking.len();
3712 for (pos, label) in j.ranking.iter().enumerate() {
3713 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3714 }
3715 }
3716
3717 let best = first_choice.values().copied().max().unwrap_or(0);
3718 let mut leaders: Vec<char> = first_choice
3719 .iter()
3720 .filter(|(_, v)| **v == best)
3721 .map(|(k, _)| *k)
3722 .collect();
3723 let mut tie_break = None;
3724 if leaders.len() > 1 {
3725 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3726 let borda_leaders: Vec<char> = leaders
3727 .iter()
3728 .copied()
3729 .filter(|l| borda[l] == top_borda)
3730 .collect();
3731 tie_break = Some(if borda_leaders.len() == 1 {
3732 format!(
3733 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3734 leaders.len()
3735 )
3736 } else {
3737 format!(
3738 "{} way tie on both first-choice votes and Borda points, broken by label order",
3739 leaders.len()
3740 )
3741 });
3742 leaders = borda_leaders;
3743 leaders.sort_unstable();
3744 }
3745 let winner = *leaders
3746 .first()
3747 .or(viable.first())
3748 .context("no candidate to declare a winner from")?;
3749
3750 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3751 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3752 let deliberated = !self.state.deliberation.is_empty();
3753
3754 let quota_seats: std::collections::BTreeSet<&str> =
3758 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3759 let mut present = 0usize;
3760 for (i, j) in self.state.judgements.iter().enumerate() {
3761 if quota_seats.contains(j.seat.as_str()) {
3762 continue;
3763 }
3764 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3765 let voted = self
3766 .state
3767 .votes
3768 .iter()
3769 .any(|v| v.judge == i + 1 && v.vote.is_some());
3770 if ranked || voted {
3771 present += 1;
3772 }
3773 }
3774 let needs_quorum = viable.len() > 1;
3780 let judges_total = if needs_quorum {
3781 self.roles.judges.len()
3782 } else {
3783 0
3784 };
3785 let quorum = if needs_quorum {
3786 judges_total / 2 + 1
3787 } else {
3788 0
3789 };
3790 let met_quorum = !needs_quorum || present >= quorum;
3791 let uncontested = (!needs_quorum).then(|| {
3792 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3793 });
3794
3795 self.state.event(
3796 "tally",
3797 match &uncontested {
3798 Some(reason) => format!("winner {winner} — {reason}"),
3799 None => format!(
3800 "winner {winner} — votes {} | initial {} | {} changed | \
3801 {present}/{judges_total} judges{}",
3802 first_choice
3803 .iter()
3804 .map(|(k, v)| format!("{k}:{v}"))
3805 .collect::<Vec<_>>()
3806 .join(" "),
3807 if unanimous_initial {
3808 "unanimous"
3809 } else {
3810 "split"
3811 },
3812 changed_votes,
3813 if met_quorum {
3814 String::new()
3815 } else {
3816 format!(" — below quorum ({quorum} required)")
3817 },
3818 ),
3819 },
3820 );
3821 if !met_quorum {
3822 self.state.event(
3823 "stall",
3824 format!(
3825 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3826 the run stops here, resumable"
3827 ),
3828 );
3829 }
3830 self.state.tally = Some(Tally {
3831 first_choice,
3832 borda,
3833 winner,
3834 rankings: tops.len(),
3835 unanimous_initial,
3836 deliberated,
3837 changed_votes,
3838 unanimous_final,
3839 tie_break,
3840 judges: judges_total,
3841 present,
3842 quorum,
3843 met_quorum,
3844 uncontested,
3845 });
3846 self.state.status = if met_quorum {
3847 RunStatus::Reviewing
3848 } else {
3849 RunStatus::Stalled
3850 };
3851 self.state.save()?;
3852 Ok(())
3853 }
3854
3855 #[allow(clippy::too_many_lines)]
3876 async fn recover_stall(&mut self) -> Result<bool> {
3877 let run_id = self.state.id.clone();
3882 let prompts = self.state.config.prompts.clone();
3883 let quota_seats: BTreeSet<&str> =
3888 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3889 let absent: Vec<String> = self
3890 .state
3891 .judgements
3892 .iter()
3893 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3894 .map(|j| j.seat.clone())
3895 .collect();
3896 if absent.is_empty() {
3897 return Ok(false);
3898 }
3899 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3900 if viable.len() <= 1 {
3901 return Ok(false);
3902 }
3903 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3904 let language = self.state.config.graph.language.clone();
3905 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3906 let sessions = self.state.config.graph.sessions;
3907 let artifacts = agent::artifacts_dir(&self.state.dir());
3908 let root = self.state.worktree_root();
3909 let base_short = short(&self.state.base_commit);
3910 let candidates: Vec<Candidate> = viable.clone();
3911
3912 let mut positions: Vec<usize> = absent
3914 .iter()
3915 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3916 .collect();
3917 if positions.is_empty() {
3918 return Ok(false);
3919 }
3920 positions.sort_unstable();
3921 positions.dedup();
3922
3923 let mut judge_jobs = Vec::new();
3925 for &j in &positions {
3926 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3927 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3928 let seat_key = format!("judge-{}", j + 1);
3929 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3930 let seat = self.seat(&seat_key, &spec.id);
3931 judge_jobs.push(SeatJob {
3932 spec,
3933 seat,
3934 prompt: prompt::judge(
3935 &self.state.instruction,
3936 &views,
3937 self.roles.judges.len(),
3938 &base_short,
3939 &language,
3940 ),
3941 cwd: root.join(seat_key),
3942 timeout,
3943 allow_write: false,
3944 sessions,
3945 artifacts: artifacts.clone(),
3946 stem: format!("judge-{}-recover", j + 1),
3947 handover: None,
3948 });
3949 }
3950
3951 let labels_for_check = labels.clone();
3952 let mut judge_losses = Vec::new();
3953 let retries = self.state.config.graph.retries;
3954 let cache = self.state.config.cache_dir();
3955 let ctx = WaveCtx {
3956 carry_seats: false,
3957 pauses: Some((&self.pause, &self.interrupt)),
3958 run: &run_id,
3959 node: "judge",
3960 prompts: &prompts,
3961 cache: cache.as_deref(),
3962 round: None,
3963 };
3964 let results = ask_json_wave::<Ranking>(
3965 judge_jobs,
3966 Arc::clone(&self.sem),
3967 retries,
3968 &self.roles.judge_roster,
3969 &ctx,
3970 &mut judge_losses,
3971 &mut self.state,
3972 &move |r: &Ranking| r.validate(&labels_for_check),
3973 )
3974 .await;
3975 if self.state.resource_block.is_some() {
3976 self.stop_for_resource()?;
3977 return Ok(false);
3978 }
3979
3980 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3982 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3983 let agent_id = seat.agent.clone();
3984 self.state.seats.insert(seat.key.clone(), seat);
3985 let record = &mut self.state.judgements[j];
3986 match res {
3987 Ok((ranking, out)) => {
3988 record.agent = agent_id;
3989 record.ranking = ranking.normalized();
3990 record.reasons = ranking.reasons;
3991 record.strengths = ranking.strengths;
3992 record.confidence = ranking.confidence;
3993 record.failed = None;
3994 record.duration_ms = out.duration_ms;
3995 recovered.insert(j);
3996 self.state.event(
3997 "recover",
3998 format!("judge {} ranked again after the limit", j + 1),
3999 );
4000 }
4001 Err(e) => {
4002 self.state
4003 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4004 }
4005 }
4006 }
4007
4008 let mut vote_jobs = Vec::new();
4010 let mut vote_pos: Vec<usize> = Vec::new();
4011 for &j in &recovered {
4012 let seat_key = format!("judge-{}", j + 1);
4013 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4014 let seat = self.seat(&seat_key, &spec.id);
4015 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4016 let text = if has_context(&spec, &seat, sessions) {
4017 prompt::final_vote(&labels, &language)
4018 } else {
4019 full.clone()
4020 };
4021 vote_jobs.push(SeatJob {
4022 spec,
4023 seat,
4024 prompt: text,
4025 cwd: root.join(seat_key),
4026 timeout,
4027 allow_write: false,
4028 sessions,
4029 artifacts: artifacts.clone(),
4030 stem: format!("vote-judge-{}-recover", j + 1),
4031 handover: Some(full),
4032 });
4033 vote_pos.push(j);
4034 }
4035 let allowed = labels.clone();
4036 let mut vote_losses = Vec::new();
4037 let vote_retries = self.state.config.graph.retries;
4038 let vote_cache = self.state.config.cache_dir();
4039 let ctx = WaveCtx {
4040 carry_seats: false,
4041 pauses: Some((&self.pause, &self.interrupt)),
4042 run: &run_id,
4043 node: "vote",
4044 prompts: &prompts,
4045 cache: vote_cache.as_deref(),
4046 round: None,
4047 };
4048 let votes = ask_json_wave::<FinalVote>(
4049 vote_jobs,
4050 Arc::clone(&self.sem),
4051 vote_retries,
4052 &self.roles.judge_roster,
4053 &ctx,
4054 &mut vote_losses,
4055 &mut self.state,
4056 &move |v: &FinalVote| match v.label() {
4057 Some(c) if allowed.contains(&c) => Ok(()),
4058 other => bail!("vote {other:?} is not one of {allowed:?}"),
4059 },
4060 )
4061 .await;
4062 if self.state.resource_block.is_some() {
4063 self.stop_for_resource()?;
4064 return Ok(false);
4065 }
4066 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4067 let agent_id = seat.agent.clone();
4068 self.state.seats.insert(seat.key.clone(), seat);
4069 match res {
4070 Ok((v, _)) => {
4071 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4072 rec.vote = v.label();
4073 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4074 } else {
4075 self.state.votes.push(VoteRecord {
4076 judge: j + 1,
4077 agent: agent_id,
4078 vote: v.label(),
4079 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4080 changed: false,
4081 });
4082 }
4083 self.state.event(
4084 "recover",
4085 format!("judge {} voted again after the limit", j + 1),
4086 );
4087 }
4088 Err(e) => {
4089 self.state
4090 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4091 }
4092 }
4093 }
4094
4095 let recovered_keys: BTreeSet<String> = recovered
4099 .iter()
4100 .map(|&j| format!("judge-{}", j + 1))
4101 .collect();
4102 self.state
4103 .quota
4104 .retain(|q| !recovered_keys.contains(&q.seat));
4105 for loss in judge_losses.into_iter().chain(vote_losses) {
4109 if recovered_keys.contains(&loss.seat) {
4110 continue;
4111 }
4112 self.state.quota.retain(|q| q.seat != loss.seat);
4113 self.state.quota.push(loss);
4114 }
4115
4116 self.state.tally = None;
4118 self.tally()?;
4119 Ok(self
4120 .state
4121 .tally
4122 .as_ref()
4123 .map(|t| t.met_quorum)
4124 .unwrap_or(false))
4125 }
4126
4127 async fn graft(&mut self) -> Result<()> {
4143 let Err(e) = self.graft_inner().await else {
4144 return Ok(());
4145 };
4146 if e.is::<crate::repo_lock::ResourceBlocked>() {
4147 return Err(e);
4148 }
4149 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4150 self.state.status = RunStatus::Blocked;
4151 self.state.event("graft", format!("blocked: {e:#}"));
4152 self.state.save()?;
4153 return Err(e);
4154 }
4155 if let Some(rec) = self.state.graft.clone()
4160 && !rec.outcome.settled()
4161 {
4162 if let Some(winner) = self.state.winner().cloned() {
4163 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4164 if re.is::<crate::repo_lock::ResourceBlocked>() {
4165 return Err(re);
4166 }
4167 self.state.status = RunStatus::Blocked;
4168 self.state.event("graft", format!("blocked: {re:#}"));
4169 self.state.save()?;
4170 return Err(re);
4171 }
4172 }
4173 if let Some(g) = self.state.graft.as_mut() {
4174 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4175 }
4176 }
4177 self.state.event("graft", format!("skipped: {e:#}"));
4178 if let Err(e) = self.state.save() {
4179 tracing::warn!("save after graft: {e:#}");
4180 }
4181 Ok(())
4182 }
4183
4184 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4187 let _coordination = self.coordinate(wt).await?;
4188 crate::recovery::restore_repair(wt, pre)
4189 .await
4190 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4191 }
4192
4193 fn graft_skipped(&mut self, why: &str) {
4195 let message = format!("skipped: {why}");
4196 if !self
4197 .state
4198 .events
4199 .iter()
4200 .any(|e| e.node == "graft" && e.message == message)
4201 {
4202 self.state.event("graft", message);
4203 }
4204 }
4205
4206 async fn graft_inner(&mut self) -> Result<()> {
4207 if let Some(rec) = self.state.graft.clone()
4210 && !rec.outcome.settled()
4211 {
4212 if self.state.reviews.is_empty()
4216 && let Some(winner) = self.state.winner().cloned()
4217 {
4218 self.graft_restore(&winner.worktree, &rec.pre).await?;
4219 }
4220 if rec.outcome == GraftOutcome::WaitingForRestore {
4221 self.state.graft = None;
4222 self.state.save()?;
4223 } else {
4224 if let Some(g) = self.state.graft.as_mut() {
4225 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4226 }
4227 self.state.event(
4228 "graft",
4229 "rolled back: the previous round was interrupted; the winner is untouched",
4230 );
4231 self.state.save()?;
4232 return Ok(());
4233 }
4234 }
4235 let plan = match graft::plan(&self.state) {
4236 Ok(plan) => plan,
4237 Err(skip) => {
4238 if skip != graft::Skip::AlreadyDone {
4239 self.graft_skipped(skip.reason());
4240 self.state.save()?;
4241 }
4242 return Ok(());
4243 }
4244 };
4245 let Some(winner) = self.state.winner().cloned() else {
4246 return Ok(());
4247 };
4248 let wt = winner.worktree.clone();
4249 if !git::is_clean(&wt).await? {
4250 self.graft_skipped("the winner's worktree has uncommitted changes");
4251 self.state.save()?;
4252 return Ok(());
4253 }
4254 let pre = git::rev_parse(&wt, "HEAD").await?;
4255 self.state.graft = Some(GraftRecord {
4258 pre: pre.clone(),
4259 post: None,
4260 lines: None,
4261 limit: None,
4262 agent: None,
4263 outcome: GraftOutcome::Running,
4264 requested: graft::requested(&plan),
4265 adoption: None,
4266 adoption_error: None,
4267 });
4268 self.state.event(
4269 "graft",
4270 format!(
4271 "porting strengths of candidate(s) {} onto {}",
4272 plan.strengths
4273 .iter()
4274 .map(|(l, _, _)| l.to_string())
4275 .collect::<Vec<_>>()
4276 .join(", "),
4277 plan.winner
4278 ),
4279 );
4280 self.state.save()?;
4281
4282 let fix_prompt = prompt::graft(
4283 &self.state.instruction,
4284 plan.winner,
4285 &plan.strengths,
4286 &self.state.config.graph.language,
4287 );
4288 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4289 let sessions = self.state.config.graph.sessions;
4290 let artifacts = agent::artifacts_dir(&self.state.dir());
4291 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4292 let (_, seat, out) = self
4293 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4294 SeatJob {
4295 prompt: fix_prompt.clone(),
4296 spec,
4297 seat,
4298 cwd: wt.clone(),
4299 timeout,
4300 allow_write: true,
4301 sessions,
4302 artifacts: artifacts.clone(),
4303 stem: "graft".to_owned(),
4304 handover: None,
4305 }
4306 })
4307 .await;
4308 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4309 self.state.graft.as_mut().expect("graft record").outcome =
4312 GraftOutcome::WaitingForRestore;
4313 return self.stop_for_resource();
4314 }
4315 let mut why: Option<String> = None;
4316 match &out {
4317 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4318 why = Some("the fixer produced nothing usable".to_owned());
4319 }
4320 AgentOutcome::Ok(_) => {}
4321 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4322 AgentOutcome::Quota(o) => {
4323 self.state.quota.push(QuotaLoss {
4324 seat: seat.key.clone(),
4325 node: "graft".to_owned(),
4326 at: Timestamp::now(),
4327 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4328 });
4329 why = Some("rate limited (quota)".to_owned());
4330 }
4331 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4332 why = Some(format!(
4333 "the fixer failed: {}",
4334 e.lines()
4335 .next()
4336 .unwrap_or("")
4337 .chars()
4338 .take(160)
4339 .collect::<String>()
4340 ));
4341 }
4342 }
4343 let agent_id = seat.agent.clone();
4344 self.state.seats.insert(seat.key.clone(), seat);
4345
4346 let mut post = None;
4347 let mut lines = None;
4348 let _coordination = self.coordinate(&wt).await?;
4349 if why.is_none() {
4350 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4351 if let Ok(r) = &rescued {
4352 self.state.note_withheld("graft", &r.withheld);
4353 }
4354 let head = git::rev_parse(&wt, "HEAD").await?;
4355 let dirty = match &rescued {
4359 Ok(r) => git::status_porcelain(&wt)
4360 .await
4361 .map(|st| {
4362 st.lines().any(|l| {
4363 !r.withheld
4364 .iter()
4365 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4366 })
4367 })
4368 .unwrap_or(true),
4369 Err(_) => true,
4370 };
4371 if dirty {
4372 why = Some("the fixer left changes that could not be committed".to_owned());
4373 } else if head == pre {
4374 why = Some("the fixer changed nothing".to_owned());
4375 } else {
4376 lines = git::changed_lines(&wt, &pre, &head).await;
4377 post = Some(head);
4378 }
4379 }
4380 let (adoption, adoption_error) = match &out {
4381 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4382 _ => (None, None),
4383 };
4384 let requested = graft::requested(&plan);
4385 let outcome = match why {
4386 None => {
4387 self.state.event(
4388 "graft",
4389 format!(
4390 "applied: {}",
4391 lines.map_or_else(
4392 || "changed lines unavailable".to_owned(),
4393 |n| format!("{n} changed lines")
4394 )
4395 ),
4396 );
4397 GraftOutcome::Applied
4398 }
4399 Some(why) => {
4400 crate::recovery::restore_repair(&wt, &pre)
4403 .await
4404 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4405 self.state.event(
4406 "graft",
4407 format!("rolled back: {why}; the winner is untouched"),
4408 );
4409 GraftOutcome::RolledBack(why)
4410 }
4411 };
4412 self.state.graft = Some(GraftRecord {
4413 pre,
4414 post,
4415 lines,
4416 limit: None,
4417 agent: Some(agent_id),
4418 outcome,
4419 requested,
4420 adoption,
4421 adoption_error,
4422 });
4423 self.state.save()?;
4424 Ok(())
4425 }
4426
4427 async fn fold_losers(&mut self) -> Result<()> {
4430 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4431 return Ok(());
4432 };
4433 if !self.state.candidate_preservation.is_empty() {
4434 bail!("candidate preservation incomplete; refusing cleanup");
4435 }
4436 let repo = self.state.repo.clone();
4437 let _coordination = self.coordinate(&repo).await?;
4438 let mut folded = Vec::new();
4439 for i in 0..self.state.candidates.len() {
4440 let c = &self.state.candidates[i];
4441 if c.label == winner || c.folded {
4442 continue;
4443 }
4444 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4445 remove_preserved_worktree(&repo, &wt).await?;
4446 if git::branch_exists(&repo, &branch).await?
4447 && !git::branch_delete(&repo, &branch).await?
4448 {
4449 bail!("candidate branch removal refused");
4450 }
4451 self.state.candidates[i].folded = true;
4452 self.state.save()?;
4453 folded.push(label.to_string());
4454 }
4455 let root = self.state.worktree_root();
4457 for j in 1..=self.roles.judges.len() {
4458 let wt = root.join(format!("judge-{j}"));
4459 if wt.exists() {
4460 remove_preserved_worktree(&repo, &wt).await?;
4461 }
4462 }
4463 if self.state.config.graph.advise {
4466 for k in 1..=self.state.config.graph.advisors {
4467 let wt = root.join(format!("advisor-{k}"));
4468 if wt.exists() {
4469 remove_preserved_worktree(&repo, &wt).await?;
4470 }
4471 }
4472 }
4473 if !folded.is_empty() {
4474 self.state
4475 .event("fold", format!("folded candidates {}", folded.join(", ")));
4476 self.state.save()?;
4477 }
4478 Ok(())
4479 }
4480
4481 async fn sync_to_base(&mut self) -> Result<()> {
4521 if self.state.status == RunStatus::AlreadyInBase {
4522 return Ok(());
4523 }
4524 let conflicted = self
4525 .state
4526 .base_sync
4527 .as_ref()
4528 .is_some_and(|s| s.conflict.is_some());
4529 let Some(winner) = self.state.winner().cloned() else {
4530 return Ok(());
4531 };
4532
4533 let repo = self.state.repo.clone();
4534 let remote = self.state.config.merge.remote.clone();
4535 let base_branch = self.state.base_branch.clone();
4536 let tracking = format!("{remote}/{base_branch}");
4537
4538 git::fetch(&repo, &remote, &base_branch).await.ok();
4539 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4544 return Ok(());
4545 };
4546
4547 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4548 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4549 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4550
4551 if (behind > 0 || conflicted || head == tip)
4560 && self
4561 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4562 .await?
4563 {
4564 return Ok(());
4565 }
4566 if conflicted {
4567 return Ok(());
4568 }
4569
4570 if behind == 0 {
4571 if let Some(from) = self
4580 .state
4581 .rebase_fixes
4582 .iter()
4583 .rev()
4584 .find_map(|r| r.from.clone())
4585 && from != head
4586 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4587 .await
4588 .is_ok_and(|o| o.ok())
4589 {
4590 git::sync_to_head(&winner.worktree).await?;
4591 }
4592 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4598 if let Some(why) = &conflict {
4599 self.state.status = RunStatus::Blocked;
4600 self.state.event("land", why.clone());
4601 }
4602 self.state.base_sync = Some(BaseSync {
4603 tip,
4604 behind: 0,
4605 attempts,
4606 conflict,
4607 already_in: None,
4608 });
4609 self.state.save()?;
4610 return Ok(());
4611 }
4612
4613 if attempts >= BASE_SYNC_ROUNDS {
4614 let why = format!(
4615 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4616 rebase(s); rebasing again would only race it",
4617 winner.branch
4618 );
4619 self.state.status = RunStatus::Blocked;
4620 self.state.base_sync = Some(BaseSync {
4621 tip,
4622 behind,
4623 attempts,
4624 conflict: Some(why.clone()),
4625 already_in: None,
4626 });
4627 self.state.event("land", why);
4628 self.state.save()?;
4629 return Ok(());
4630 }
4631
4632 self.state.event(
4633 "land",
4634 format!(
4635 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4636 winner.branch
4637 ),
4638 );
4639 self.state.save()?;
4640
4641 let branch_tracking = format!("{remote}/{}", winner.branch);
4647 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4648 let remote_tip = if self.state.task_pr_branch().is_none()
4649 && matches!(&fetched_branch, Ok(o) if o.ok())
4650 {
4651 git::rev_parse(&repo, &branch_tracking).await.ok()
4652 } else {
4653 None
4654 };
4655 if let Some(theirs) = &remote_tip
4659 && !git::is_ancestor(&repo, theirs, &head).await
4660 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4661 .await
4662 .is_ok_and(|missing| missing.is_empty())
4663 {
4664 let why = format!(
4665 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4666 them",
4667 short(theirs),
4668 winner.branch
4669 );
4670 self.state.status = RunStatus::Blocked;
4671 self.state.base_sync = Some(BaseSync {
4672 tip,
4673 behind,
4674 attempts,
4675 conflict: Some(why.clone()),
4676 already_in: None,
4677 });
4678 self.state.event("land", why);
4679 self.state.save()?;
4680 return Ok(());
4681 }
4682
4683 let scratch = self.state.dir().join("base-sync");
4684 let rebased = match crate::rebase::rebase_with_fixer(
4685 &mut self.state,
4686 &scratch,
4687 &winner.branch,
4688 &tracking,
4689 )
4690 .await
4691 {
4692 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4693 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4694 Err(e) => Err(e),
4695 };
4696 let attempts = attempts + 1;
4697 match rebased {
4698 Ok(None) => {
4699 git::sync_to_head(&winner.worktree).await?;
4703 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4704 let mut conflict = None;
4705 if let Some(pinned) = &remote_tip {
4706 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4707 match pushed {
4708 Ok(o) if o.ok() => self.state.event(
4709 "land",
4710 format!("pushed rebased {} to {remote}", winner.branch),
4711 ),
4712 Ok(o) => {
4713 conflict = Some(format!(
4714 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4715 winner.branch,
4716 short(pinned),
4717 o.stderr.chars().take(600).collect::<String>()
4718 ));
4719 }
4720 Err(e) => {
4721 conflict = Some(format!(
4722 "rebased {} locally but could not push it: {e:#}",
4723 winner.branch
4724 ));
4725 }
4726 }
4727 }
4728 if let Some(why) = &conflict {
4729 self.state.status = RunStatus::Blocked;
4730 self.state.event("land", why.clone());
4731 }
4732 self.state.base_sync = Some(BaseSync {
4733 tip: tip.clone(),
4734 behind: 0,
4735 attempts,
4736 conflict,
4737 already_in: None,
4738 });
4739 self.state
4740 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4741 }
4742 Ok(Some(conflict)) => {
4743 let why = format!(
4744 "{} conflicts with {tracking} and did not rebase: {}",
4745 winner.branch,
4746 conflict.chars().take(600).collect::<String>()
4747 );
4748 self.state.status = RunStatus::Blocked;
4749 self.state.base_sync = Some(BaseSync {
4750 tip,
4751 behind,
4752 attempts,
4753 conflict: Some(why.clone()),
4754 already_in: None,
4755 });
4756 self.state.event("land", why);
4757 }
4758 Err(e) => {
4759 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4760 self.state.status = RunStatus::Blocked;
4761 self.state.base_sync = Some(BaseSync {
4762 tip,
4763 behind,
4764 attempts,
4765 conflict: Some(why.clone()),
4766 already_in: None,
4767 });
4768 self.state.event("land", why);
4769 }
4770 }
4771 self.state.save()?;
4772 Ok(())
4773 }
4774
4775 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4781 if self.state.task_pr_branch().is_some() {
4784 return None;
4785 }
4786 let repo = self.state.repo.clone();
4787 let remote = self.state.config.merge.remote.clone();
4788 let fetched = git::fetch(&repo, &remote, branch).await;
4789 if !matches!(&fetched, Ok(o) if o.ok()) {
4790 return None;
4791 }
4792 let branch_tracking = format!("{remote}/{branch}");
4793 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4794 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4795 return None;
4796 }
4797 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4798 .await
4799 .is_ok_and(|missing| missing.is_empty())
4800 {
4801 return Some(format!(
4802 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4803 them",
4804 short(&theirs)
4805 ));
4806 }
4807 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4808 Ok(o) if o.ok() => {
4809 self.state
4810 .event("land", format!("pushed rebased {branch} to {remote}"));
4811 None
4812 }
4813 Ok(o) => Some(format!(
4814 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4815 someone may have pushed): {}",
4816 short(&theirs),
4817 o.stderr.chars().take(600).collect::<String>()
4818 )),
4819 Err(e) => Some(format!(
4820 "{branch} is rebased locally but could not be pushed: {e:#}"
4821 )),
4822 }
4823 }
4824
4825 async fn settle_already_in(
4842 &mut self,
4843 branch: &str,
4844 tip: &str,
4845 head: &str,
4846 attempts: usize,
4847 behind: usize,
4848 ) -> Result<bool> {
4849 let repo = self.state.repo.clone();
4850 let remote = self.state.config.merge.remote.clone();
4851 let start = self.state.base_commit.clone();
4852 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4853 Ok(Some(e)) => e,
4854 Ok(None) => return Ok(false),
4855 Err(e) => {
4856 tracing::warn!("already-in-base check for {branch}: {e:#}");
4857 return Ok(false);
4858 }
4859 };
4860 let mut verified = vec![head.to_owned()];
4861 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4862 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4863 if matches!(&fetched, Ok(o) if o.ok())
4864 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4865 && theirs != head
4866 {
4867 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4868 Ok(Some(_)) => verified.push(theirs),
4869 _ => return Ok(false),
4870 }
4871 }
4872 let base_branch = self.state.base_branch.clone();
4873 let message = format!(
4874 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4875 land",
4876 evidence.names(),
4877 evidence.proof.as_str()
4878 );
4879 let closed =
4880 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4881 .await;
4882 match closed {
4883 Ok(Ok(url)) => self
4884 .state
4885 .event("land", format!("closed {url}: superseded on {base_branch}")),
4886 Ok(Err(why)) => self
4887 .state
4888 .event("land", format!("did not close a pull request: {why}")),
4889 Err(e) => {
4890 let why = format!(
4891 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4892 not be closed ({e:#}); resume to retry"
4893 );
4894 self.state.status = RunStatus::Blocked;
4895 self.state.base_sync = Some(BaseSync {
4896 tip: tip.to_owned(),
4897 behind,
4898 attempts,
4899 conflict: Some(why.clone()),
4900 already_in: None,
4901 });
4902 self.state.event("land", why);
4903 self.state.save()?;
4904 return Ok(true);
4905 }
4906 }
4907 self.state.status = RunStatus::AlreadyInBase;
4908 self.state.base_sync = Some(BaseSync {
4909 tip: tip.to_owned(),
4910 behind,
4911 attempts,
4912 conflict: None,
4913 already_in: Some(evidence),
4914 });
4915 self.state.event("land", message);
4916 self.state.save()?;
4917 self.settle_questions();
4918 Ok(true)
4919 }
4920
4921 fn landing_base(&self) -> String {
4931 self.state
4932 .base_sync
4933 .as_ref()
4934 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4935 }
4936
4937 pub async fn fix_selected(
4970 &mut self,
4971 ids: &[String],
4972 reason: &str,
4973 allow_stale: bool,
4974 ) -> Result<()> {
4975 let reason = reason.trim();
4976 if reason.is_empty() {
4977 bail!("a fix request needs a reason — that is the operator's own record of why");
4978 }
4979 if ids.is_empty() {
4980 bail!("no finding id given");
4981 }
4982 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4983 bail!(
4984 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4985 has already concluded — can be given a targeted fix. A run still \
4986 in progress should simply be resumed; a `merged` run's branch has \
4987 already landed, so its answer is a fresh `magi review <branch>`, \
4988 not reopening this run's own record",
4989 self.state.id,
4990 self.state.status.as_str()
4991 );
4992 }
4993 let Some(winner) = self.state.winner().cloned() else {
4994 bail!("run {} has no winning candidate to fix", self.state.id);
4995 };
4996 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4997 bail!(
4998 "branch `{}` no longer exists; this run cannot be extended",
4999 winner.branch
5000 );
5001 }
5002 let home = crate::run::home();
5003 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5004 bail!(
5005 "run {} is currently being worked on by another magi process",
5006 self.state.id
5007 );
5008 }
5009 let _claim = FixClaim::acquire(&self.state.dir())?;
5015
5016 let mut seen = BTreeSet::new();
5020 let mut findings = Vec::new();
5021 let mut missing = Vec::new();
5022 for id in ids {
5023 if !seen.insert(id.clone()) {
5024 continue;
5025 }
5026 match self.state.finding(id) {
5027 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5028 id: f.id.clone(),
5029 severity: f.severity,
5030 reviewer_vote: rec.vote,
5031 round: round.round,
5032 round_head: round.head.clone(),
5033 reviewer: rec.reviewer,
5034 agent: rec.agent.clone(),
5035 file: f.file.clone(),
5036 line: f.line,
5037 title: f.title.clone(),
5038 detail: f.detail.clone(),
5039 outcome: OperatorFixOutcome::Pending,
5040 }),
5041 None => missing.push(id.clone()),
5042 }
5043 }
5044 if !missing.is_empty() {
5045 bail!(
5046 "unknown finding id(s): {}; nothing was changed",
5047 missing.join(", ")
5048 );
5049 }
5050
5051 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5052 let stale_details: Vec<(String, String)> = findings
5053 .iter()
5054 .filter(|f| f.round_head != head_at_request)
5055 .map(|f| (f.id.clone(), f.round_head.clone()))
5056 .collect();
5057 let stale = !stale_details.is_empty();
5058 if stale && !allow_stale {
5059 bail!(
5060 "the branch has moved since some finding(s) were raised — {} — now \
5061 at {}; pass --allow-stale to fix anyway, or re-run review first",
5062 stale_details
5063 .iter()
5064 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5065 .collect::<Vec<_>>()
5066 .join(", "),
5067 short(&head_at_request)
5068 );
5069 }
5070
5071 let request = OperatorFixRequest {
5072 requested_at: Timestamp::now(),
5073 reason: reason.to_owned(),
5074 findings,
5075 head_at_request: head_at_request.clone(),
5076 allow_stale,
5077 stale,
5078 fix: None,
5079 result_head: None,
5080 follow_up_review_run: None,
5081 };
5082 self.state.event(
5083 "fix",
5084 format!(
5085 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5086 request.findings.len(),
5087 request
5088 .findings
5089 .iter()
5090 .map(|f| f.id.as_str())
5091 .collect::<Vec<_>>()
5092 .join(", "),
5093 ),
5094 );
5095 self.state.operator_fixes.push(request);
5102 self.state.save()?;
5103 let request_index = self.state.operator_fixes.len() - 1;
5104
5105 let repo = self.state.repo.clone();
5114 let coordination = self.coordinate(&repo).await?;
5115 if winner.worktree.exists() {
5116 let dirty = git::git(
5119 &winner.worktree,
5120 &["status", "--porcelain", "--untracked-files=all"],
5121 )
5122 .await?;
5123 let only_withheld = dirty.lines().all(|l| {
5124 l.strip_prefix("?? ")
5125 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5126 });
5127 if !only_withheld {
5128 bail!(
5129 "`{}` has uncommitted changes; refusing to touch it — commit or \
5130 discard them first",
5131 winner.worktree.display()
5132 );
5133 }
5134 git::worktree_remove(&self.state.repo, &winner.worktree)
5135 .await
5136 .ok();
5137 }
5138 let fix_worktree = self.state.worktree_root().join("operator-fix");
5139 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5140 git::git(
5141 &self.state.repo,
5142 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5143 )
5144 .await
5145 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5146 if !git::is_clean(&fix_worktree).await? {
5147 git::worktree_remove(&self.state.repo, &fix_worktree)
5148 .await
5149 .ok();
5150 bail!(
5151 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5152 winner.branch
5153 );
5154 }
5155 let repo_for_setup = self.state.repo.clone();
5156 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5157 Ok(()) => {
5158 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5159 }
5160 Err(e) => Err(e),
5161 };
5162 if let Err(e) = setup {
5163 git::worktree_remove(&self.state.repo, &fix_worktree)
5164 .await
5165 .ok();
5166 return Err(e);
5167 }
5168
5169 let run_id = self.state.id.clone();
5170 let prompts = self.state.config.prompts.clone();
5171 let language = self.state.config.graph.language.clone();
5172 let sessions = self.state.config.graph.sessions;
5173 let artifacts = agent::artifacts_dir(&self.state.dir());
5174 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5175 .findings
5176 .iter()
5177 .map(|f| Finding {
5178 id: f.id.clone(),
5179 severity: f.severity,
5180 file: f.file.clone(),
5181 line: f.line,
5182 title: f.title.clone(),
5183 detail: f.detail.clone(),
5184 })
5185 .collect();
5186 let fix_prompt = prompt::operator_fix_in(
5187 &self.state.instruction,
5188 &finding_list,
5189 reason,
5190 &stale_details,
5191 &head_at_request,
5192 &language,
5193 &self.state.config.graph.github_language,
5194 );
5195 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5196 drop(coordination);
5197 let (job, seat, out) = self
5198 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5199 prompt: fix_prompt.clone(),
5200 spec,
5201 seat,
5202 cwd: fix_worktree.clone(),
5203 timeout,
5204 allow_write: true,
5205 sessions,
5206 artifacts: artifacts.clone(),
5207 stem: "operator-fix".to_owned(),
5208 handover: None,
5209 })
5210 .await;
5211 let agent_id = seat.agent.clone();
5212
5213 let mut fix = FixRecord {
5214 agent: agent_id,
5215 addressed: Vec::new(),
5216 rejected: Vec::new(),
5217 notes: String::new(),
5218 committed: false,
5219 failed: None,
5220 duration_ms: 0,
5221 continuation: None,
5222 };
5223 let mut final_seat = seat.clone();
5224 match out {
5225 AgentOutcome::Ok(o) => {
5226 fix.duration_ms = o.duration_ms;
5227 let parsed = verdict::extract_json::<FixReport>(&o.text);
5228 let incomplete_reason = match &parsed {
5229 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5230 "the reply parsed, but it reported a command whose own CLI \
5231 never confirmed an exit status"
5232 .to_owned(),
5233 ),
5234 Ok(_) => None,
5235 Err(e) => Some(e.to_string()),
5236 };
5237 match incomplete_reason {
5238 None => {
5239 let report = parsed.expect("checked Ok above");
5240 fix.addressed = report.addressed;
5241 fix.rejected = report.rejected;
5242 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5243 }
5244 Some(reason) => {
5245 let (resumed_seat, resolved, failure, cont) = self
5246 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5247 .await;
5248 fix.duration_ms += cont.cumulative_wait_ms;
5249 fix.continuation = Some(cont);
5250 final_seat = resumed_seat;
5251 match resolved {
5252 Some(report) => {
5253 fix.addressed = report.addressed;
5254 fix.rejected = report.rejected;
5255 fix.notes =
5256 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5257 }
5258 None => fix.failed = failure,
5259 }
5260 }
5261 }
5262 }
5263 AgentOutcome::Dropped(o) => {
5264 fix.duration_ms = o.duration_ms;
5265 let why = o
5266 .dropped
5267 .as_ref()
5268 .map(|d| d.why.as_str())
5269 .unwrap_or("the CLI ended the stream without delivering its answer");
5270 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5271 }
5272 AgentOutcome::Quota(o) => {
5273 self.state.quota.push(QuotaLoss {
5274 seat: final_seat.key.clone(),
5275 node: "fix".to_owned(),
5276 at: Timestamp::now(),
5277 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5278 });
5279 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5280 }
5281 AgentOutcome::Failed(e) => fix.failed = Some(e),
5282 AgentOutcome::ResourceBlocked(e) => {
5283 fix.failed = Some(format!("the fixer was not started: {e}"));
5284 }
5285 }
5286 if fix.continuation.is_none() {
5287 fix.continuation = Some(ContinuationRecord::not_needed());
5288 }
5289 self.state.seats.insert(final_seat.key.clone(), final_seat);
5290
5291 let rescue_message = format!(
5292 "magi: operator-selected fix ({}) (uncommitted work)",
5293 self.state.operator_fixes[request_index]
5294 .findings
5295 .iter()
5296 .map(|f| f.id.as_str())
5297 .collect::<Vec<_>>()
5298 .join(", ")
5299 );
5300 let repo = self.state.repo.clone();
5301 let coordination = self.coordinate(&repo).await?;
5302 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5303 self.state.note_withheld("fix", &r.withheld);
5304 }
5305 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5306 fix.committed = after != head_at_request;
5307 git::worktree_remove(&self.state.repo, &fix_worktree)
5308 .await
5309 .ok();
5310 drop(coordination);
5311
5312 self.state.event(
5313 "fix",
5314 match &fix.failed {
5315 Some(reason) => format!(
5316 "operator fix: adoption report was lost ({reason}); {}",
5317 if fix.committed {
5318 "committed"
5319 } else {
5320 "NO new commit"
5321 }
5322 ),
5323 None => format!(
5324 "operator fix: {} addressed, {} rejected, {}",
5325 fix.addressed.len(),
5326 fix.rejected.len(),
5327 if fix.committed {
5328 "committed"
5329 } else {
5330 "NO new commit"
5331 }
5332 ),
5333 },
5334 );
5335
5336 for f in &mut self.state.operator_fixes[request_index].findings {
5343 f.outcome = if fix.failed.is_some() {
5344 OperatorFixOutcome::Unreported
5345 } else if fix.addressed.contains(&f.id) {
5346 OperatorFixOutcome::Addressed
5347 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5348 OperatorFixOutcome::Rejected { why: r.why.clone() }
5349 } else {
5350 OperatorFixOutcome::Unreported
5351 };
5352 }
5353
5354 let committed = fix.committed;
5355 if committed {
5356 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5357 }
5358 self.state.operator_fixes[request_index].fix = Some(fix);
5359 self.state.save()?;
5362
5363 if committed {
5364 self.state.event(
5365 "fix",
5366 format!(
5367 "operator fix committed {}; opening a follow-up review-only run",
5368 short(&after)
5369 ),
5370 );
5371 let origin =
5374 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5375 match Self::review(
5376 &self.state.repo,
5377 &winner.branch,
5378 self.state.config.clone(),
5379 origin,
5380 )
5381 .await
5382 {
5383 Ok(mut follow_up) => {
5384 follow_up.state.event(
5385 "start",
5386 format!(
5387 "requested by an operator fix on run {} for finding(s) {}",
5388 self.state.id,
5389 self.state.operator_fixes[request_index]
5390 .findings
5391 .iter()
5392 .map(|f| f.id.as_str())
5393 .collect::<Vec<_>>()
5394 .join(", "),
5395 ),
5396 );
5397 follow_up.state.save()?;
5398 let follow_up_id = follow_up.state.id.clone();
5399 let adopted = match crate::direct::adopt(
5402 &follow_up.state,
5403 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5404 ) {
5405 Ok(a) => a,
5406 Err(e) => {
5407 self.state.event(
5410 "fix",
5411 format!(
5412 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5413 ),
5414 );
5415 self.state.save()?;
5416 return Ok(());
5417 }
5418 };
5419 let executed = follow_up.execute().await;
5420 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5421 if let Some(a) = adopted {
5422 a.finish(&follow_up.state, executed);
5423 }
5424 if let Some(e) = failure {
5425 self.state.event(
5426 "fix",
5427 format!(
5428 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5429 ),
5430 );
5431 }
5432 self.state.operator_fixes[request_index].follow_up_review_run =
5433 Some(follow_up_id);
5434 }
5435 Err(e) => {
5436 self.state.event(
5437 "fix",
5438 format!("committed the fix but could not open a follow-up review: {e:#}"),
5439 );
5440 }
5441 }
5442 self.state.save()?;
5443 }
5444
5445 Ok(())
5446 }
5447
5448 async fn ask_fixer(
5468 &mut self,
5469 winner: &Candidate,
5470 node: &'static str,
5471 round: Option<usize>,
5472 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5473 ) -> (SeatJob, SeatState, AgentOutcome) {
5474 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5475 self.ask_fixer_among(attempts, node, round, None, build)
5476 .await
5477 }
5478
5479 async fn ask_fixer_among(
5484 &mut self,
5485 attempts: Vec<(AgentSpec, String)>,
5486 node: &'static str,
5487 round: Option<usize>,
5488 restore: Option<(&Path, &str)>,
5489 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5490 ) -> (SeatJob, SeatState, AgentOutcome) {
5491 let run_id = self.state.id.clone();
5492 let prompts = self.state.config.prompts.clone();
5493 let cache = self.state.config.cache_dir();
5494 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5495 let mut last = None;
5496 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5497 let seat = self.seat(&key, &spec.id);
5498 let mut job = build(spec.clone(), seat);
5499 if i > 0 {
5500 job.stem = format!("{}-{}", job.stem, spec.id);
5501 }
5502 let ctx = WaveCtx {
5503 carry_seats: false,
5504 pauses: Some((&self.pause, &self.interrupt)),
5505 run: &run_id,
5506 node,
5507 prompts: &prompts,
5508 cache: cache.as_deref(),
5509 round,
5510 };
5511 let (seat, out) =
5512 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5513 let advances = match &out {
5516 AgentOutcome::Ok(o) => agent::output_advances(o),
5517 AgentOutcome::ResourceBlocked(_) => false,
5518 _ => true,
5519 };
5520 if let Some(next) = ids.get(i + 1)
5521 && advances
5522 {
5523 self.state.seats.insert(seat.key.clone(), seat.clone());
5524 let class =
5525 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5526 let reason = match &out {
5527 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5528 other => fail_reason(other),
5529 };
5530 if let Some((wt, rev)) = restore
5534 && let Err(e) = self.graft_restore(wt, rev).await
5535 {
5536 let why =
5537 format!("could not restore the worktree before the next fixer: {e:#}");
5538 tracing::warn!("{why}");
5539 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5540 AgentOutcome::ResourceBlocked(why)
5541 } else {
5542 AgentOutcome::Failed(why)
5543 };
5544 last = Some((job, seat, out));
5545 break;
5546 }
5547 record_handover(
5548 &mut self.state,
5549 node,
5550 &seat.key,
5551 &spec.id,
5552 next,
5553 &class,
5554 &reason,
5555 );
5556 continue;
5557 }
5558 last = Some((job, seat, out));
5559 break;
5560 }
5561 last.expect("the fixer chain always has an entry")
5562 }
5563
5564 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5575 use crate::recovery::{Mode, Repair};
5576 if !self.state.reviews.is_empty() {
5577 return Ok(());
5578 }
5579 let Some(mut rec) = self.state.recovery.clone() else {
5580 return Ok(());
5581 };
5582 if rec.mode != Mode::VerifyRepair {
5583 return Ok(());
5584 }
5585 let wt = winner.worktree.clone();
5586 let mut head = git::rev_parse(&wt, "HEAD").await?;
5587 match rec.repair.clone() {
5588 Repair::Done { .. } => return Ok(()),
5589 Repair::WaitingForRestore { before } => {
5590 self.graft_restore(&wt, &before).await?;
5591 head = before;
5592 rec.repair = Repair::NotStarted;
5593 self.state.recovery = Some(rec.clone());
5594 self.state.save()?;
5595 }
5596 Repair::Started { before } => {
5597 if let Err(e) = self.graft_restore(&wt, &before).await {
5601 if e.is::<crate::repo_lock::ResourceBlocked>() {
5602 return Err(e);
5603 }
5604 return self
5605 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5606 .await;
5607 }
5608 rec.repair = Repair::Done {
5609 before: before.clone(),
5610 after: before,
5611 committed: false,
5612 failed: Some("interrupted; rolled back".to_owned()),
5613 };
5614 self.state
5615 .event("recovery", "the repair was interrupted and rolled back");
5616 self.state.recovery = Some(rec);
5617 self.state.save()?;
5618 return Ok(());
5619 }
5620 Repair::NotStarted => {}
5621 }
5622 if crate::recovery::recheck(&mut rec, &head) {
5623 self.state.event("recovery", rec.summary());
5624 self.state.recovery = Some(rec);
5625 self.state.save()?;
5626 return Ok(());
5627 }
5628 rec.repair = Repair::Started {
5629 before: head.clone(),
5630 };
5631 self.state.recovery = Some(rec.clone());
5632 self.state.save()?;
5633
5634 let language = self.state.config.graph.language.clone();
5635 let sessions = self.state.config.graph.sessions;
5636 let artifacts = agent::artifacts_dir(&self.state.dir());
5637 let fix_prompt = prompt::recovery_fix(
5638 &self.state.instruction,
5639 &rec,
5640 self.state.config.graph.review_rounds,
5641 &language,
5642 );
5643 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5644 self.state.event(
5645 "recovery",
5646 format!(
5647 "repair: the fixer starts from the failed verification on {}",
5648 short(&head)
5649 ),
5650 );
5651 let (_job, seat, out) = self
5652 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5653 prompt: fix_prompt.clone(),
5654 spec,
5655 seat,
5656 cwd: wt.clone(),
5657 timeout,
5658 allow_write: true,
5659 sessions,
5660 artifacts: artifacts.clone(),
5661 stem: "fix-recovery".to_owned(),
5662 handover: None,
5663 })
5664 .await;
5665 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5666 rec.repair = Repair::WaitingForRestore { before: head };
5667 self.state.recovery = Some(rec);
5668 return self.stop_for_resource();
5669 }
5670 let failed = match &out {
5671 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5672 AgentOutcome::Quota(o) => {
5673 self.state.quota.push(QuotaLoss {
5674 seat: seat.key.clone(),
5675 node: "fix".to_owned(),
5676 at: Timestamp::now(),
5677 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5678 });
5679 Some("rate limited (quota); the fixer could not run".to_owned())
5680 }
5681 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5682 AgentOutcome::Failed(e) => Some(e.clone()),
5683 };
5684 self.state.seats.insert(seat.key.clone(), seat);
5685 let coordination = self.coordinate(&wt).await?;
5686 let failed = match failed {
5687 Some(why) => {
5689 if let Err(e) = git::restore_to(&wt, &head).await {
5690 return self
5691 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5692 .await;
5693 }
5694 Some(why)
5695 }
5696 None => {
5697 if let Ok(r) = git::rescue_commit(
5698 &wt,
5699 "magi: recovery repair of the failed verification (uncommitted work)",
5700 )
5701 .await
5702 {
5703 self.state.note_withheld("fix", &r.withheld);
5704 }
5705 None
5706 }
5707 };
5708 let after = git::rev_parse(&wt, "HEAD").await?;
5709 drop(coordination);
5710 self.state.event(
5711 "recovery",
5712 match &failed {
5713 Some(why) => format!("repair: nothing kept ({why})"),
5714 None if after != head => {
5715 format!("repair: committed {}", short(&after))
5716 }
5717 None => "repair: the fixer made no commit".to_owned(),
5718 },
5719 );
5720 rec.repair = Repair::Done {
5721 before: head.clone(),
5722 committed: after != head,
5723 after,
5724 failed,
5725 };
5726 self.state.recovery = Some(rec);
5727 self.state.save()?;
5728 Ok(())
5729 }
5730
5731 async fn stop_recovery(
5734 &mut self,
5735 mut rec: crate::recovery::RecoveryRecord,
5736 head: &str,
5737 why: String,
5738 ) -> Result<()> {
5739 rec.repair = crate::recovery::Repair::Started {
5741 before: head.to_owned(),
5742 };
5743 self.state.recovery = Some(rec);
5744 self.state
5745 .event("recovery", format!("repair could not be undone: {why}"));
5746 self.state.status = RunStatus::Blocked;
5747 self.state.save()?;
5748 bail!("recovery repair could not be rolled back: {why}")
5749 }
5750
5751 async fn review_loop(&mut self) -> Result<()> {
5752 if self
5757 .state
5758 .base_sync
5759 .as_ref()
5760 .is_some_and(|s| s.conflict.is_some())
5761 {
5762 return Ok(());
5763 }
5764 let run_id = self.state.id.clone();
5769 let prompts = self.state.config.prompts.clone();
5770 let Some(winner) = self.state.winner().cloned() else {
5771 return Ok(());
5772 };
5773 let max_rounds = self.state.config.graph.review_rounds;
5774 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5784 if status == RunStatus::Gating {
5787 self.record_contested_handoff();
5788 }
5789 self.state.status = status;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793 self.state.status = RunStatus::Reviewing;
5794 if self
5804 .state
5805 .reviews
5806 .last()
5807 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5808 {
5809 let shell = self.state.config.shell();
5810 return self
5811 .stop_reviewing(
5812 "the last round's own verification never resolved",
5813 &shell,
5814 &winner.worktree,
5815 )
5816 .await;
5817 }
5818
5819 self.recovery_repair(&winner).await?;
5820 if self.resource_halt()? {
5821 return Ok(());
5822 }
5823
5824 let repo = self.state.repo.clone();
5825 let root = self.state.worktree_root();
5826 let language = self.state.config.graph.language.clone();
5827 let sessions = self.state.config.graph.sessions;
5828 let artifacts = self.merge_repair_artifacts();
5829 let base = self.landing_base();
5830 let base_short = short(&base);
5831 let reviewers = self.roles.reviewers.clone();
5832 let shell = self.state.config.shell();
5833
5834 for round in (self.state.reviews.len() + 1)..=max_rounds {
5835 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5836 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5837 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5838 let prev_verification = self
5847 .state
5848 .reviews
5849 .last()
5850 .and_then(|r| r.verification_summary(&head));
5851 let prior_section = match (&self.state.recovery, round) {
5854 (Some(rec), 1) if self.state.reviews.is_empty() => {
5855 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5856 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5857 Some(crate::recovery::head_of(
5858 &git::diff(&winner.worktree, reviewed, "HEAD")
5859 .await
5860 .unwrap_or_default(),
5861 crate::recovery::SINCE_DIFF_MAX,
5862 ))
5863 }
5864 _ => None,
5865 };
5866 Some(prompt::recovery_review(rec, since.as_deref()))
5867 }
5868 _ => None,
5869 };
5870
5871 let mut jobs = Vec::new();
5875 let coordination = self.coordinate(&repo).await?;
5876 for (r, spec) in reviewers.iter().cloned().enumerate() {
5877 let wt = root.join(format!("review-{}", r + 1));
5878 if wt.exists() {
5881 git::reset_detached(&wt, &head).await?;
5882 } else {
5883 git::worktree_add_detached(&repo, &wt, &head).await?;
5884 }
5885 self.setup_seat_worktree(&repo, &wt, false).await?;
5886 let seat_key = format!("review-{}", r + 1);
5887 let spec = pick_start_spec(
5890 &self.roles.reviewer_roster,
5891 spec,
5892 self.state.seat_history.get(&seat_key),
5893 );
5894 let seat = self.seat(&seat_key, &spec.id);
5895 jobs.push(SeatJob {
5896 prompt: prompt::review(&prompt::ReviewCtx {
5897 instruction: &self.state.instruction,
5898 branch: &winner.branch,
5899 base_short: &base_short,
5900 stat: &stat,
5901 patch: &patch,
5902 verification: prev_verification.as_ref(),
5903 prior: prior_section.as_deref(),
5904 reviewers: reviewers.len(),
5905 round,
5906 rounds: max_rounds,
5907 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5910 lens: Lens::for_seat(r),
5911 language: &language,
5912 github_language: &self.state.config.graph.github_language,
5913 }),
5914 spec,
5915 seat,
5916 cwd: wt,
5917 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5918 allow_write: false,
5919 sessions,
5920 artifacts: artifacts.clone(),
5921 stem: format!("review-{round}-{}", r + 1),
5922 handover: None,
5923 });
5924 }
5925
5926 drop(coordination);
5927 self.state.event(
5928 "review",
5929 format!(
5930 "round {round}: {} reviewers on {}",
5931 jobs.len(),
5932 short(&head)
5933 ),
5934 );
5935 let mut quota_losses = Vec::new();
5936 let review_retries = self.state.config.graph.retries;
5937 let review_cache = self.state.config.cache_dir();
5938 let ctx = WaveCtx {
5939 carry_seats: true,
5940 pauses: Some((&self.pause, &self.interrupt)),
5941 run: &run_id,
5942 node: "review",
5943 prompts: &prompts,
5944 cache: review_cache.as_deref(),
5945 round: Some(round),
5946 };
5947 let results = ask_json_wave::<Review>(
5948 jobs,
5949 Arc::clone(&self.sem),
5950 review_retries,
5951 &self.roles.reviewer_roster,
5952 &ctx,
5953 &mut quota_losses,
5954 &mut self.state,
5955 &|_: &Review| Ok(()),
5956 )
5957 .await;
5958 if self.state.resource_block.is_some() {
5959 return self.stop_for_resource();
5960 }
5961 let round_quota_missing = quota_losses.len();
5965 self.state.quota.extend(quota_losses);
5966
5967 let mut records = Vec::new();
5968 let mut all_findings = Vec::new();
5969 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5970 let agent_id = seat.agent.clone();
5971 self.state.seats.insert(seat.key.clone(), seat);
5972 let mut record = ReviewRecord {
5973 reviewer: r + 1,
5974 agent: agent_id,
5975 summary: String::new(),
5976 findings: Vec::new(),
5977 vote: None,
5978 failed: None,
5979 duration_ms: 0,
5980 attempts,
5986 };
5987 match res {
5988 Ok((review, out)) => {
5989 record.summary =
5997 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5998 record.vote = Some(review.vote);
5999 record.duration_ms = out.duration_ms;
6000 for (n, mut f) in review.findings.into_iter().enumerate() {
6001 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6004 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6005 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6006 f.file = f
6012 .file
6013 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6014 all_findings.push(f.clone());
6015 record.findings.push(f);
6016 }
6017 self.state.event(
6018 "review",
6019 format!(
6020 "round {round}: reviewer {} voted {} with {} finding(s)",
6021 r + 1,
6022 review.vote.label(),
6023 record.findings.len()
6024 ),
6025 );
6026 }
6027 Err(e) => {
6028 record.failed = Some(e.to_string());
6029 self.state.event(
6030 "review",
6031 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6032 );
6033 }
6034 }
6035 records.push(record);
6036 }
6037
6038 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6045 let vote_split =
6046 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6047 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6048 if vote_split {
6049 self.state.event(
6050 "review",
6051 format!(
6052 "round {round}: votes split ({}) — one round of reconsideration",
6053 initial_votes
6054 .iter()
6055 .map(|v| v.label())
6056 .collect::<Vec<_>>()
6057 .join(", ")
6058 ),
6059 );
6060 let panel: Vec<ReviewSeatReport<'_>> = records
6063 .iter()
6064 .filter_map(|r| {
6065 r.vote.map(|vote| ReviewSeatReport {
6066 reviewer: r.reviewer,
6067 vote,
6068 summary: &r.summary,
6069 findings: &r.findings,
6070 })
6071 })
6072 .collect();
6073
6074 let mut jobs = Vec::new();
6075 let mut seats_at = Vec::new();
6076 for (r, spec) in reviewers.iter().cloned().enumerate() {
6077 if records[r].vote.is_none() {
6081 continue;
6082 }
6083 let wt = root.join(format!("review-{}", r + 1));
6084 let seat_key = format!("review-{}", r + 1);
6085 let spec = self.occupant(&seat_key, spec);
6086 let seat = self.seat(&seat_key, &spec.id);
6087 let build = |with_patch: bool| {
6095 prompt::review_reconsider(&ReviewReconsiderCtx {
6096 instruction: &self.state.instruction,
6097 reviewer: r + 1,
6098 lens: Lens::for_seat(r),
6099 panel: &panel,
6100 patch: with_patch.then_some(ReviewPatch {
6101 branch: &winner.branch,
6102 base_short: &base_short,
6103 stat: &stat,
6104 patch: &patch,
6105 }),
6106 round,
6107 rounds: max_rounds,
6108 language: &language,
6109 })
6110 };
6111 let full = build(true);
6112 let prompt = if has_context(&spec, &seat, sessions) {
6113 build(false)
6114 } else {
6115 full.clone()
6116 };
6117 jobs.push(SeatJob {
6118 prompt,
6119 spec,
6120 seat,
6121 cwd: wt,
6122 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6123 allow_write: false,
6124 sessions,
6125 artifacts: artifacts.clone(),
6126 stem: format!("review-{round}-reconsider-{}", r + 1),
6127 handover: Some(full),
6128 });
6129 seats_at.push(r);
6130 }
6131
6132 let mut recon_quota_losses = Vec::new();
6133 let recon_cache = self.state.config.cache_dir();
6134 let recon_ctx = WaveCtx {
6135 carry_seats: true,
6136 pauses: Some((&self.pause, &self.interrupt)),
6137 run: &run_id,
6138 node: "review",
6139 prompts: &prompts,
6140 cache: recon_cache.as_deref(),
6141 round: Some(round),
6142 };
6143 let recon_results = ask_json_wave::<ReviewRevote>(
6144 jobs,
6145 Arc::clone(&self.sem),
6146 review_retries,
6147 &self.roles.reviewer_roster,
6148 &recon_ctx,
6149 &mut recon_quota_losses,
6150 &mut self.state,
6151 &|_: &ReviewRevote| Ok(()),
6152 )
6153 .await;
6154 if self.state.resource_block.is_some() {
6155 return self.stop_for_resource();
6156 }
6157 self.state.quota.extend(recon_quota_losses);
6158
6159 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6160 let agent_id = seat.agent.clone();
6161 self.state.seats.insert(seat.key.clone(), seat);
6162 let mut rec = ReviewRevoteRecord {
6163 reviewer: r + 1,
6164 agent: agent_id,
6165 vote: None,
6166 reason: String::new(),
6167 failed: None,
6168 };
6169 match res {
6170 Ok((rv, _)) => {
6171 rec.vote = Some(rv.vote);
6172 rec.reason =
6173 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6174 self.state.event(
6175 "review",
6176 format!(
6177 "round {round}: reviewer {} revoted {}",
6178 r + 1,
6179 rv.vote.label()
6180 ),
6181 );
6182 }
6183 Err(e) => {
6184 rec.failed = Some(e.to_string());
6185 self.state.event(
6186 "review",
6187 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6188 );
6189 }
6190 }
6191 reconsideration.push(rec);
6192 }
6193 } else if initial_votes.len() > 1 {
6194 self.state.event(
6195 "review",
6196 format!(
6197 "round {round}: votes agreed ({}) — no reconsideration",
6198 initial_votes[0].label()
6199 ),
6200 );
6201 }
6202
6203 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6204 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6205 let defer_e2e =
6216 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6217 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6218 let reason =
6219 format!("{blocking} blocking finding(s) already required a fix this round");
6220 self.state.event(
6221 "verify",
6222 format!(
6223 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6224 {}); it will run once a round has none left",
6225 short(&head)
6226 ),
6227 );
6228 (Vec::new(), false, true, Some(reason))
6229 } else {
6230 let e2e_commands = self.state.config.verify.e2e.clone();
6231 let cache_dir = self.state.config.cache_dir();
6232 let context = format!("round {round}");
6233 let wait = self.repository_wait();
6234 let (e2e, verify_retried) = with_cache_lease(
6235 &mut self.state,
6236 cache_dir.as_deref(),
6237 "e2e",
6238 "e2e",
6239 &winner.worktree,
6240 &head,
6241 verify_timeout,
6242 &context,
6243 &wait,
6244 |state, budget| {
6245 let shell = shell.clone();
6246 let e2e_commands = e2e_commands.clone();
6247 let worktree = winner.worktree.clone();
6248 let context = context.clone();
6249 async move {
6250 run_e2e_with_retry(
6251 state,
6252 &shell,
6253 &e2e_commands,
6254 &worktree,
6255 budget,
6256 &context,
6257 )
6258 .await
6259 }
6260 },
6261 )
6262 .await?;
6263 (e2e, verify_retried, false, None)
6264 };
6265
6266 let expected = records.len();
6267 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6268 let incomplete = answered < expected;
6269 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6270 let policy = self.state.config.graph.incomplete_review;
6271 let clean = round_is_clean(
6272 blocking,
6273 e2e_ok,
6274 answered,
6275 expected,
6276 round_quota_missing,
6277 policy,
6278 );
6279
6280 let mut round_record = ReviewRound {
6281 round,
6282 head: head.clone(),
6283 verified_head: None,
6284 verified_at: None,
6285 reviews: records,
6286 e2e,
6287 verify_retried,
6288 e2e_deferred,
6289 e2e_defer_reason,
6290 fix: None,
6291 blocking,
6292 answered,
6293 expected,
6294 clean,
6295 progressed: false,
6296 vote_split,
6297 reconsideration,
6298 verdict: None,
6299 };
6300 round_record.verdict = ReviewVote::worst(
6304 round_record
6305 .final_votes()
6306 .into_iter()
6307 .map(|(_, _, vote)| vote),
6308 );
6309 if !matches!(
6320 round_record.e2e_status(),
6321 E2eStatus::Deferred | E2eStatus::NotConfigured
6322 ) {
6323 round_record.verified_head = Some(head.clone());
6324 round_record.verified_at = Some(Timestamp::now());
6325 }
6326 let this_round_verification = round_record.verification_summary(&head);
6327
6328 if incomplete {
6329 let missing: Vec<String> = round_record
6330 .reviews
6331 .iter()
6332 .filter(|r| r.failed.is_some())
6333 .map(|r| format!("review-{}", r.reviewer))
6334 .collect();
6335 self.state.event(
6336 "review",
6337 format!(
6338 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6339 missing.join(", ")
6340 ),
6341 );
6342 }
6343
6344 if clean {
6345 self.state.event(
6346 "review",
6347 if incomplete && policy == IncompleteReviewPolicy::Warn {
6348 format!(
6349 "round {round}: clean (warn policy, incomplete panel) — no \
6350 blocking findings from the seats that answered, verification green"
6351 )
6352 } else if incomplete {
6353 format!(
6354 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6355 quorum) — no blocking findings from the seats that answered, \
6356 verification green",
6357 expected - answered
6358 )
6359 } else {
6360 format!("round {round}: clean — no blocking findings, verification green")
6361 },
6362 );
6363 self.state.reviews.push(round_record);
6364 self.state.status = RunStatus::Gating;
6365 self.state.save()?;
6366 return Ok(());
6367 }
6368
6369 if incomplete && blocking == 0 && e2e_ok {
6377 self.state.reviews.push(round_record);
6378 self.state.save()?;
6379 if round == max_rounds {
6380 self.state.status = RunStatus::Blocked;
6381 self.state.event(
6382 "review",
6383 format!(
6384 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6385 refusing to call it clean",
6386 expected - answered
6387 ),
6388 );
6389 return Ok(());
6390 }
6391 continue;
6392 }
6393
6394 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6406 self.state.reviews.push(round_record);
6407 return self
6408 .stop_reviewing(
6409 "the round's own verification could not run",
6410 &shell,
6411 &winner.worktree,
6412 )
6413 .await;
6414 }
6415
6416 if round == max_rounds {
6417 self.state.reviews.push(round_record);
6418 return self
6419 .stop_reviewing(
6420 &format!(
6421 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6422 ),
6423 &shell,
6424 &winner.worktree,
6425 )
6426 .await;
6427 }
6428
6429 let blocking_findings: Vec<_> = all_findings
6432 .iter()
6433 .filter(|f| f.severity.blocks())
6434 .cloned()
6435 .collect();
6436 let fix_prompt = prompt::fix_in(
6437 &self.state.instruction,
6438 &blocking_findings,
6439 this_round_verification.as_ref(),
6440 round,
6441 max_rounds,
6442 &language,
6443 &self.state.config.graph.github_language,
6444 );
6445 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6446 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6447 let (job, seat, out) = self
6448 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6449 prompt: fix_prompt.clone(),
6450 spec,
6451 seat,
6452 cwd: winner.worktree.clone(),
6453 timeout,
6454 allow_write: true,
6455 sessions,
6456 artifacts: artifacts.clone(),
6457 stem: format!("fix-{round}"),
6458 handover: None,
6459 })
6460 .await;
6461 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6464 return self.stop_for_resource();
6465 }
6466 let agent_id = seat.agent.clone();
6467
6468 let mut fix = FixRecord {
6469 agent: agent_id,
6470 addressed: Vec::new(),
6471 rejected: Vec::new(),
6472 notes: String::new(),
6473 committed: false,
6474 failed: None,
6475 duration_ms: 0,
6476 continuation: None,
6477 };
6478 let mut continuation = ContinuationRecord::not_needed();
6479 let mut final_seat = seat.clone();
6480 match out {
6481 AgentOutcome::Ok(o) => {
6482 fix.duration_ms = o.duration_ms;
6483 let parsed = verdict::extract_json::<FixReport>(&o.text);
6484 let incomplete_reason = match &parsed {
6491 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6492 "the reply parsed, but it reported a command whose own CLI never \
6493 confirmed an exit status"
6494 .to_owned(),
6495 ),
6496 Ok(_) => None,
6497 Err(e) => Some(e.to_string()),
6498 };
6499 match incomplete_reason {
6500 None => {
6501 let report = parsed.expect("checked Ok above");
6502 fix.addressed = report.addressed;
6503 fix.rejected = report.rejected;
6504 fix.notes =
6505 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6506 }
6507 Some(reason) => {
6508 let (resumed_seat, resolved, failure, cont) = self
6509 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6510 .await;
6511 fix.duration_ms += cont.cumulative_wait_ms;
6512 continuation = cont;
6513 final_seat = resumed_seat;
6514 match resolved {
6515 Some(report) => {
6516 fix.addressed = report.addressed;
6517 fix.rejected = report.rejected;
6518 fix.notes = blind::sanitize_prose(
6519 &report.notes,
6520 &self.state.config.blind,
6521 );
6522 }
6523 None => fix.failed = failure,
6524 }
6525 }
6526 }
6527 }
6528 AgentOutcome::Dropped(o) => {
6530 fix.duration_ms = o.duration_ms;
6531 let why = o
6532 .dropped
6533 .as_ref()
6534 .map(|d| d.why.as_str())
6535 .unwrap_or("the CLI ended the stream without delivering its answer");
6536 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6537 }
6538 AgentOutcome::Quota(o) => {
6539 self.state.quota.push(QuotaLoss {
6540 seat: final_seat.key.clone(),
6541 node: "fix".to_owned(),
6542 at: Timestamp::now(),
6543 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6544 });
6545 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6546 }
6547 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6548 }
6549 fix.continuation = Some(continuation);
6550 self.state.seats.insert(final_seat.key.clone(), final_seat);
6551 let coordination = self.coordinate(&winner.worktree).await?;
6552 if let Ok(r) = git::rescue_commit(
6553 &winner.worktree,
6554 &format!("magi: review round {round} fixes (uncommitted work)"),
6555 )
6556 .await
6557 {
6558 self.state.note_withheld("fix", &r.withheld);
6559 }
6560 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6561 fix.committed = after != before;
6562 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6570 drop(coordination);
6571 let progressed = diff_after != patch;
6572 let commit_note = if fix.committed {
6573 "committed"
6574 } else {
6575 "NO new commit"
6576 };
6577 let tree_note = if progressed {
6578 "changed vs base"
6579 } else {
6580 "unchanged vs base"
6581 };
6582 self.state.event(
6583 "fix",
6584 match &fix.failed {
6585 Some(reason) => {
6591 format!(
6592 "round {round}: fixer's adoption report was lost ({reason}); \
6593 {commit_note}, tree {tree_note}"
6594 )
6595 }
6596 None => format!(
6597 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6598 {tree_note}{}",
6599 fix.addressed.len(),
6600 fix.rejected.len(),
6601 if continuation.outcome == ContinuationOutcome::Resumed {
6602 format!(
6603 " (adoption report recovered after {} continuation(s))",
6604 continuation.attempts
6605 )
6606 } else {
6607 String::new()
6608 },
6609 ),
6610 },
6611 );
6612 round_record.fix = Some(fix);
6613 round_record.progressed = progressed;
6614 self.state.reviews.push(round_record);
6615 self.state.save()?;
6616
6617 if matches!(
6630 continuation.outcome,
6631 ContinuationOutcome::Exhausted
6632 | ContinuationOutcome::QuotaLost
6633 | ContinuationOutcome::NoSession
6634 ) {
6635 return self
6636 .stop_reviewing(
6637 "the fixer's adoption report never came back, even after resuming its \
6638 own seat; refusing to start another round against the same worktree \
6639 while that is unresolved",
6640 &shell,
6641 &winner.worktree,
6642 )
6643 .await;
6644 }
6645
6646 let streak = self
6647 .state
6648 .reviews
6649 .iter()
6650 .rev()
6651 .take_while(|r| !r.progressed)
6652 .count();
6653 if streak >= STAGNANT_LIMIT {
6654 return self
6655 .stop_reviewing(
6656 &format!(
6657 "the tree has not moved against base for {streak} round(s) in a row"
6658 ),
6659 &shell,
6660 &winner.worktree,
6661 )
6662 .await;
6663 }
6664 }
6665 Ok(())
6666 }
6667
6668 fn record_contested_handoff(&mut self) {
6702 if self.state.contested_handoff.is_some() {
6703 return;
6704 }
6705 let Some(contested) = self
6706 .state
6707 .reviews
6708 .last()
6709 .and_then(ReviewRound::contested_handoff)
6710 else {
6711 return;
6712 };
6713 self.state.event(
6714 "review",
6715 format!(
6716 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6717 wait for the owner's approval",
6718 contested.findings.len(),
6719 contested.rejecters.len()
6720 ),
6721 );
6722 self.state.contested_handoff = Some(contested);
6723 }
6724
6725 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6726 let round_idx = self.state.reviews.len() - 1;
6727 let needs_catchup_run = matches!(
6735 self.state.reviews[round_idx].e2e_status(),
6736 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6737 );
6738 if needs_catchup_run {
6739 let round = self.state.reviews[round_idx].round;
6740 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6741 let commands = self.state.config.verify.e2e.clone();
6742 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6743 let cache_dir = self.state.config.cache_dir();
6744 let context = format!(
6745 "round {round}: verification unresolved, catching up before the final decision"
6746 );
6747 let wait = self.repository_wait();
6748 let (outcomes, verify_retried) = with_cache_lease(
6749 &mut self.state,
6750 cache_dir.as_deref(),
6751 "e2e",
6752 "e2e",
6753 worktree,
6754 &attempted_head,
6755 timeout,
6756 &context,
6757 &wait,
6758 |state, budget| {
6759 let shell = shell.to_vec();
6760 let commands = commands.clone();
6761 let context = context.clone();
6762 async move {
6763 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6764 .await
6765 }
6766 },
6767 )
6768 .await?;
6769 let last = &mut self.state.reviews[round_idx];
6770 last.e2e = outcomes;
6771 last.verify_retried = verify_retried;
6772 last.verified_head = Some(attempted_head);
6779 last.verified_at = Some(Timestamp::now());
6780 if verify_inconclusive(&last.e2e) {
6781 self.state.save()?;
6788 return Ok(());
6789 }
6790 last.e2e_deferred = false;
6791 }
6792 let last = &self.state.reviews[round_idx];
6793 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6794
6795 match last.e2e_status() {
6796 E2eStatus::Failed => {
6797 let red: Vec<String> = last
6798 .e2e
6799 .iter()
6800 .filter(|o| !o.ok())
6801 .map(|o| {
6802 format!(
6803 "`{}` -> {:?}\n{}",
6804 o.command,
6805 o.code,
6806 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6807 )
6808 })
6809 .collect();
6810 self.state
6811 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6812 self.state.status = RunStatus::Blocked;
6813 }
6814 E2eStatus::ResourceBlocked => {
6819 self.state.event(
6820 "review",
6821 format!(
6822 "{why}; e2e could not run (shared build cache unavailable); not \
6823 deciding yet"
6824 ),
6825 );
6826 }
6827 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6828 self.state.event(
6829 "review",
6830 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6831 );
6832 self.record_contested_handoff();
6833 self.state.status = RunStatus::Gating;
6834 }
6835 }
6836 self.state.save()?;
6837 Ok(())
6838 }
6839
6840 async fn gate(&mut self) -> Result<()> {
6843 if self.state.status == RunStatus::Failed
6855 || self
6856 .state
6857 .base_sync
6858 .as_ref()
6859 .is_some_and(|s| s.conflict.is_some())
6860 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6861 != Some(RunStatus::Gating)
6862 {
6863 return Ok(());
6864 }
6865 if self.state.gate_ran {
6866 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6877 self.state.status = RunStatus::Blocked;
6878 self.state.save()?;
6879 }
6880 return Ok(());
6881 }
6882 let Some(winner) = self.state.winner().cloned() else {
6883 return Ok(());
6884 };
6885 self.state.status = RunStatus::Gating;
6886 let mut outcomes = self.run_gate(&winner).await?;
6887 loop {
6888 if verify_inconclusive(&outcomes) {
6899 self.state.save()?;
6900 return Ok(());
6901 }
6902 if outcomes.iter().all(CommandOutcome::ok) {
6903 break;
6904 }
6905 match self.gate_fix_round(&winner, &outcomes).await? {
6906 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6907 GateFix::Stop => break,
6908 GateFix::Defer => {
6909 self.state.save()?;
6910 return Ok(());
6911 }
6912 }
6913 }
6914 let passed = outcomes.iter().all(CommandOutcome::ok);
6915 self.state.gate = outcomes;
6916 self.state.gate_ran = true;
6917 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6918 if !passed {
6919 self.state.status = RunStatus::Blocked;
6920 let spent = self.state.gate_fixes.len();
6921 self.state.event(
6922 "gate",
6923 if spent == 0 {
6924 "gate failed; not merging".to_owned()
6925 } else {
6926 format!("gate failed after {spent} gate-fix round(s); not merging")
6927 },
6928 );
6929 }
6930 self.state.save()?;
6931 Ok(())
6932 }
6933
6934 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6944 let commands = state.config.verify.pre_gate.clone();
6945 if commands.is_empty() {
6946 return;
6947 }
6948 let shell = state.config.shell();
6949 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6950 let (outcomes, _) = run_commands(
6951 state,
6952 "pre_gate",
6953 "pre_gate",
6954 0,
6955 &shell,
6956 &commands,
6957 &winner.worktree,
6958 timeout,
6959 )
6960 .await;
6961 for o in &outcomes {
6962 if !o.ok() {
6963 tracing::warn!(
6964 "pre_gate `{}` failed ({:?}); the gate decides",
6965 o.command,
6966 o.code
6967 );
6968 }
6969 state.event(
6970 "pre_gate",
6971 format!(
6972 "`{}` -> {}",
6973 o.command,
6974 if o.ok() {
6975 "pass".to_owned()
6976 } else {
6977 format!(
6978 "FAIL ({:?})\n{}",
6979 o.code,
6980 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6981 )
6982 }
6983 ),
6984 );
6985 }
6986 state.pre_gate = outcomes;
6987 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6988 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6989 Ok(head) => {
6990 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6991 state.pre_gate_commit = Some(head);
6992 }
6993 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6994 },
6995 Ok(false) => {}
6996 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6997 }
6998 if let Err(e) = state.save() {
6999 tracing::warn!("could not persist the pre_gate record: {e:#}");
7000 }
7001 }
7002
7003 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7006 let shell = self.state.config.shell();
7007 let gate_commands = self.state.config.verify.gate.clone();
7008 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7009 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7010 Vec::new()
7011 } else {
7012 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7013 let cache_dir = self.state.config.cache_dir();
7014 let wait = self.repository_wait();
7015 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7016 let (outcomes, _) = with_cache_lease(
7017 &mut self.state,
7018 cache_dir.as_deref(),
7019 "gate",
7020 "gate",
7021 &winner.worktree,
7022 &head,
7023 timeout,
7024 "final gate",
7025 &wait,
7026 |state, budget| {
7027 let shell = shell.clone();
7028 let gate_commands = gate_commands.clone();
7029 let winner = winner.clone();
7030 async move {
7031 Self::run_pre_gate(state, &winner).await;
7032 let (outcomes, timed_out_pids) = run_commands(
7033 state,
7034 "gate",
7035 "gate",
7036 0,
7037 &shell,
7038 &gate_commands,
7039 &winner.worktree,
7040 budget,
7041 )
7042 .await;
7043 (outcomes, false, timed_out_pids)
7044 }
7045 },
7046 )
7047 .await?;
7048 outcomes
7049 };
7050 if outcomes.is_empty() {
7051 self.state
7056 .event("gate", "no gate commands configured; not tested");
7057 }
7058 for o in &outcomes {
7059 self.state.event(
7060 "gate",
7061 format!(
7062 "`{}` -> {}",
7063 o.command,
7064 if o.ok() {
7065 "pass".to_owned()
7066 } else {
7067 format!(
7068 "FAIL ({:?})\n{}",
7069 o.code,
7070 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7071 )
7072 }
7073 ),
7074 );
7075 }
7076 Ok(outcomes)
7077 }
7078
7079 async fn gate_fix_round(
7092 &mut self,
7093 winner: &Candidate,
7094 outcomes: &[CommandOutcome],
7095 ) -> Result<GateFix> {
7096 let cap = self.state.config.graph.gate_fix_rounds;
7097 let spent = self.state.gate_fixes.len();
7098 if spent >= cap {
7099 if cap > 0 {
7100 self.state.event(
7101 "gate",
7102 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7103 );
7104 }
7105 return Ok(GateFix::Stop);
7106 }
7107 if !gate_fixable(outcomes) {
7108 self.state.event(
7109 "gate",
7110 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7111 command or similar); not spending a fix round on it",
7112 );
7113 return Ok(GateFix::Stop);
7114 }
7115 let min_free = self.state.config.disk.min_free_bytes;
7116 if min_free > 0 {
7117 match crate::disk::free_bytes(&winner.worktree) {
7118 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7119 Ok(free) => {
7120 self.state.event(
7121 "gate",
7122 format!(
7123 "only {free} bytes free ({min_free} required by `[disk] \
7124 min_free_bytes`); not spending a fix round on a failure the disk \
7125 may explain"
7126 ),
7127 );
7128 return Ok(GateFix::Stop);
7129 }
7130 Err(e) => {
7131 self.state.event(
7132 "gate",
7133 format!("free disk space could not be measured ({e:#}); no fix round"),
7134 );
7135 return Ok(GateFix::Stop);
7136 }
7137 }
7138 }
7139
7140 let attempt = spent + 1;
7141 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7142 let base = self.landing_base();
7143 let fix_prompt = prompt::gate_fix_in(
7144 &self.state.instruction,
7145 &failed,
7146 attempt,
7147 cap,
7148 &self.state.config.graph.language,
7149 &self.state.config.graph.github_language,
7150 );
7151 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7152 let sessions = self.state.config.graph.sessions;
7153 let artifacts = self.merge_repair_artifacts();
7154 self.state.event(
7155 "gate",
7156 format!("gate failed; gate-fix round {attempt} of {cap}"),
7157 );
7158 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7159 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7160 let (_, seat, out) = self
7161 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7162 prompt: fix_prompt.clone(),
7163 spec,
7164 seat,
7165 cwd: winner.worktree.clone(),
7166 timeout,
7167 allow_write: true,
7168 sessions,
7169 artifacts: artifacts.clone(),
7170 stem: format!("gate-fix-{attempt}"),
7171 handover: None,
7172 })
7173 .await;
7174 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7177 return Ok(GateFix::Defer);
7178 }
7179 let mut record = GateFixRecord {
7180 agent: seat.agent.clone(),
7181 failed,
7182 notes: String::new(),
7183 committed: false,
7184 error: None,
7185 };
7186 match out {
7187 AgentOutcome::Ok(o) => {
7188 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7191 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7192 }
7193 }
7194 AgentOutcome::Dropped(_) => {
7195 record.error = Some("the CLI dropped the stream".to_owned());
7196 }
7197 AgentOutcome::Quota(o) => {
7198 self.state.quota.push(QuotaLoss {
7199 seat: seat.key.clone(),
7200 node: "gate-fix".to_owned(),
7201 at: Timestamp::now(),
7202 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7203 });
7204 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7205 }
7206 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7207 }
7208 self.state.seats.insert(seat.key.clone(), seat);
7209 let coordination = self.coordinate(&winner.worktree).await?;
7210 if let Ok(r) = git::rescue_commit(
7211 &winner.worktree,
7212 &format!("magi: gate fix {attempt} (uncommitted work)"),
7213 )
7214 .await
7215 {
7216 self.state.note_withheld("gate-fix", &r.withheld);
7217 }
7218 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7219 record.committed = after != before;
7220 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7221 let note = record.error.clone();
7222 self.state.gate_fixes.push(record);
7223 self.state.save()?;
7224 if !changed {
7225 self.state.event(
7226 "gate",
7227 match note {
7228 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7229 None => format!("gate-fix round {attempt}: the tree did not change"),
7230 },
7231 );
7232 return Ok(GateFix::Stop);
7233 }
7234 self.state.event(
7235 "gate",
7236 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7237 );
7238
7239 drop(coordination);
7240 let commands = self.state.config.verify.e2e.clone();
7241 if !commands.is_empty() {
7242 let shell = self.state.config.shell();
7243 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7244 let cache_dir = self.state.config.cache_dir();
7245 let context = format!("gate-fix round {attempt}");
7246 let wait = self.repository_wait();
7247 let (e2e, _) = with_cache_lease(
7248 &mut self.state,
7249 cache_dir.as_deref(),
7250 "e2e",
7251 "e2e",
7252 &winner.worktree,
7253 &after,
7254 timeout,
7255 &context,
7256 &wait,
7257 |state, budget| {
7258 let shell = shell.clone();
7259 let commands = commands.clone();
7260 let context = context.clone();
7261 let worktree = winner.worktree.clone();
7262 async move {
7263 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7264 .await
7265 }
7266 },
7267 )
7268 .await?;
7269 if verify_inconclusive(&e2e) {
7270 return Ok(GateFix::Defer);
7271 }
7272 if e2e.iter().any(|o| !o.ok()) {
7273 self.state.event(
7274 "gate",
7275 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7276 );
7277 return Ok(GateFix::Stop);
7278 }
7279 }
7280 Ok(GateFix::Retry)
7281 }
7282
7283 async fn guarded_pr_message(
7290 &mut self,
7291 winner: &Candidate,
7292 facts: Option<&BranchFacts>,
7293 posting: bool,
7294 ) -> Result<Option<PrMessage>> {
7295 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7296 let head = match final_head.as_deref() {
7297 Some(h) => crate::pr_outcome::FinalHead::At(h),
7298 None => crate::pr_outcome::FinalHead::Unknown,
7299 };
7300 let summarized = if posting {
7302 Some(self.summarize_change(winner, final_head.as_deref()).await)
7303 } else {
7304 None
7305 };
7306 if self.state.resource_block.is_some() {
7311 self.state.event(
7312 "pr-summary",
7313 "a shared resource was busy; nothing was published, resume to summarize again",
7314 );
7315 return Ok(None);
7316 }
7317 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7318 anyhow::bail!(
7319 "the final commit moved during public description generation; retry before publishing"
7320 );
7321 }
7322 let note = match (&summarized, &self.state.change_summary) {
7323 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7324 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7325 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7326 (None, _) => ChangeNote::NotAsked,
7327 };
7328 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7329 if !posting {
7330 let identity = crate::scrub::Identity::current();
7331 return Ok(Some(PrMessage {
7332 title: crate::scrub::scrub(&pr.title, &identity),
7333 body: crate::scrub::scrub(&pr.body, &identity),
7334 }));
7335 }
7336 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7339 if let Some(known) = self
7340 .state
7341 .github_text
7342 .clone()
7343 .filter(|g| g.fingerprint == raw_fp)
7344 {
7345 return self.settle_withheld(winner, &pr, known).await;
7346 }
7347 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7348 let mut violations =
7349 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7350 let identity = crate::scrub::Identity::current();
7351 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7352 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7353 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7354 {
7355 violations.push(crate::github_text::Violation::SensitiveData);
7356 }
7357 let has_language = violations
7361 .iter()
7362 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7363 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7364 self.state.event(
7365 "github-text",
7366 if has_language {
7367 format!("description rejected: {violations:?}; requesting one rewrite")
7368 } else {
7369 format!("description rejected: {violations:?}; asking the owner")
7370 },
7371 );
7372 let mut rewritten = false;
7373 let mut rejected_rewrite = None;
7374 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7375 {
7376 let output = text;
7377 if let Some(title) = summary_title(&output) {
7378 let raw_body = summary_without_title(&output);
7381 let clean_title = crate::scrub::scrub(&title, &identity);
7382 let mut body = crate::scrub::scrub(&raw_body, &identity);
7383 let redacted = clean_title != title || body != raw_body;
7384 let title = clean_title;
7385 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7386 if !complete {
7387 self.state.event(
7388 "github-text",
7389 "description rewrite omitted required findings",
7390 );
7391 }
7392 if !body.trim().is_empty() {
7393 body.push_str("\n\n");
7394 body.push_str(&public_pr_evidence(&self.state, head));
7395 }
7396 let marker = format!("magi:run/{}", self.state.id);
7397 if !body.trim().is_empty() && !body.contains(&marker) {
7398 body.push_str(&format!("\n\n{marker}"));
7399 }
7400 let rewrite = PrMessage {
7401 title: title.clone(),
7402 body: body.clone(),
7403 };
7404 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7405 let mut failed = crate::github_text::check_for(
7406 &self.state.config.graph,
7407 &title,
7408 &body,
7409 redecision,
7410 );
7411 if (!crate::github_text::shareable(&title)
7412 || !crate::github_text::shareable(&body))
7413 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7414 {
7415 failed.push(crate::github_text::Violation::SensitiveData);
7416 }
7417 if complete
7418 && !title.trim().is_empty()
7419 && !body.trim().is_empty()
7420 && !failed.is_empty()
7421 {
7422 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7423 title: title.clone(),
7424 body: body.clone(),
7425 failed: failed
7426 .iter()
7427 .map(|v| crate::github_text::category(*v).to_owned())
7428 .collect(),
7429 });
7430 }
7431 if complete
7432 && !title.trim().is_empty()
7433 && !raw_body.trim().is_empty()
7434 && failed.is_empty()
7435 {
7436 decision = redecision;
7437 pr = PrMessage { title, body };
7438 rewritten = true;
7439 self.state
7440 .event("github-text", "description rewrite passed");
7441 if redacted {
7442 self.state.event(
7443 "github-text",
7444 "sensitive data removed from description rewrite",
7445 );
7446 }
7447 }
7448 }
7449 }
7450 if !rewritten {
7451 self.state.event(
7452 "github-text",
7453 "description rewrite unavailable or rejected; asking the owner",
7454 );
7455 if let Some(w) =
7456 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7457 {
7458 self.retire_github_text_question("the withheld text changed");
7461 let record = crate::run::GithubTextAsk {
7462 rewrite: rejected_rewrite,
7463 fingerprint: raw_fp,
7464 title: w.title,
7465 body: w.body,
7466 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7467 question: None,
7468 asks: 0,
7469 resolved: false,
7470 chosen_title: None,
7471 choice: None,
7472 };
7473 return self.settle_withheld(winner, &pr, record).await;
7474 }
7475 }
7476 }
7477 let (title, body) =
7478 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7479 Ok(Some(PrMessage { title, body }))
7480 }
7481
7482 async fn summarize_change(
7493 &mut self,
7494 winner: &Candidate,
7495 head: Option<&str>,
7496 ) -> std::result::Result<(), String> {
7497 let Some(head) = head else {
7498 return Err("the final commit could not be read".to_owned());
7499 };
7500 let remote = self.state.config.merge.remote.clone();
7501 let tracking = format!("{remote}/{}", self.state.base_branch);
7502 if !git::rev_exists(&winner.worktree, &tracking).await {
7503 return Err(format!("the remote base `{tracking}` could not be read"));
7504 }
7505 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7506 Ok(b) if !b.is_empty() => b,
7507 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7508 };
7509 let evidence = crate::pr_outcome::summary_evidence(
7510 &self.state,
7511 crate::pr_outcome::FinalHead::At(head),
7512 );
7513 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7514 if self
7515 .state
7516 .change_summary
7517 .as_ref()
7518 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7519 {
7520 return Ok(());
7521 }
7522 if self.state.change_summary.take().is_some() {
7523 self.state.event(
7524 "pr-summary",
7525 "the final diff or evidence changed; the earlier change summary was discarded",
7526 );
7527 }
7528 if let Err(e) = write_artifact(
7530 &self.state,
7531 &format!("public-description-{fingerprint}.md"),
7532 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7533 ) {
7534 tracing::warn!("could not save public description evidence: {e:#}");
7535 }
7536 let chain = match agent::pick_chain(
7537 &self.state.config.agents,
7538 self.state.config.roles.synthesizer.as_ref(),
7539 &agent::installed,
7540 "synthesizer",
7541 ) {
7542 Ok(chain) => chain,
7543 Err(e) => {
7544 tracing::warn!("no agent for the change summary: {e:#}");
7545 self.state.event("pr-summary", "no summary agent available");
7546 return Err("no agent was available to summarize it".to_owned());
7547 }
7548 };
7549 let subjects = git::subjects(&winner.worktree, &base, head)
7550 .await
7551 .unwrap_or_default();
7552 let stat = git::diff_stat(&winner.worktree, &base, head)
7553 .await
7554 .unwrap_or_default();
7555 let stat: String = stat.chars().take(6_000).collect();
7556 let mut prompt = crate::prompt::change_summary_in(
7557 &base,
7558 head,
7559 &subjects,
7560 stat.trim(),
7561 &self.state.config.graph.github_language,
7562 );
7563 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7564 let sessions = self.state.config.graph.sessions;
7565 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7566 let prompts = self.state.config.prompts.clone();
7567 let cache = self.state.config.cache_dir();
7568 let run = self.state.id.clone();
7569 for (i, spec) in chain.into_iter().enumerate() {
7570 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7571 let job = SeatJob {
7572 spec: spec.clone(),
7573 seat,
7574 cwd: winner.worktree.clone(),
7575 prompt: prompt.clone(),
7576 timeout,
7577 allow_write: false,
7578 sessions,
7579 artifacts: agent::artifacts_dir(&self.state.dir()),
7580 stem: if i == 0 {
7581 "change-summary".to_owned()
7582 } else {
7583 format!("change-summary-{}", spec.id)
7584 },
7585 handover: None,
7586 };
7587 let ctx = WaveCtx {
7588 run: &run,
7589 node: "pr-summary",
7590 prompts: &prompts,
7591 cache: cache.as_deref(),
7592 round: None,
7593 carry_seats: false,
7594 pauses: Some((&self.pause, &self.interrupt)),
7595 };
7596 let (seat, outcome) =
7597 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7598 self.state.seats.insert(seat.key.clone(), seat);
7599 let text = match outcome {
7600 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7601 AgentOutcome::ResourceBlocked(why) => {
7602 if self.state.resource_block.is_none() {
7605 self.state.resource_block = Some(crate::run::ResourceBlock {
7606 node: "pr-summary".to_owned(),
7607 kind: "shared resource".to_owned(),
7608 detail: why,
7609 at: Timestamp::now(),
7610 });
7611 }
7612 return Err("a shared resource was busy".to_owned());
7613 }
7614 _ => continue,
7615 };
7616 let blind = &self.state.config.blind;
7617 let Some(title) = summary_title(&text) else {
7618 continue;
7619 };
7620 let body: String = summary_without_title(&text)
7621 .lines()
7622 .filter(|l| !l.contains("magi:run/"))
7623 .collect::<Vec<_>>()
7624 .join("\n");
7625 let body = blind::sanitize_prose(body.trim(), blind);
7626 let title = blind::sanitize_prose(&title, blind);
7627 if body.trim().is_empty() || title.trim().is_empty() {
7628 continue;
7629 }
7630 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7631 continue;
7632 }
7633 if git::rev_parse(&winner.worktree, "HEAD")
7635 .await
7636 .ok()
7637 .as_deref()
7638 != Some(head)
7639 {
7640 return Err("the branch moved while the summary was written".to_owned());
7641 }
7642 if git::merge_base(&winner.worktree, &tracking, head)
7643 .await
7644 .ok()
7645 .as_deref()
7646 != Some(base.as_str())
7647 || crate::pr_outcome::summary_evidence(
7648 &self.state,
7649 crate::pr_outcome::FinalHead::At(head),
7650 ) != evidence
7651 {
7652 return Err("the summary evidence changed while it was written".to_owned());
7653 }
7654 self.state.change_summary = Some(crate::run::ChangeSummary {
7655 head: head.to_owned(),
7656 base,
7657 evidence: fingerprint,
7658 title,
7659 body,
7660 });
7661 self.state.event(
7662 "pr-summary",
7663 format!("change summary written by {} for the final diff", spec.id),
7664 );
7665 if let Err(e) = self.state.save() {
7666 tracing::warn!("could not save the change summary: {e:#}");
7667 }
7668 return Ok(());
7669 }
7670 self.state
7671 .event("pr-summary", "no agent produced a change summary");
7672 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7673 }
7674
7675 async fn rewrite_pr_text(
7683 &mut self,
7684 winner: &Candidate,
7685 pr: &PrMessage,
7686 violations: &[crate::github_text::Violation],
7687 ) -> Option<String> {
7688 let prompt = format!(
7689 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7690 pr.title, pr.body
7691 );
7692 let sessions = self.state.config.graph.sessions;
7693 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7694 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7696 let key = format!("impl-{}", winner.label);
7697 if let Some(spec) = self
7698 .state
7699 .config
7700 .agents
7701 .iter()
7702 .find(|a| a.id == winner.agent)
7703 .cloned()
7704 && let Some(existing) = self.state.seats.get(&key)
7705 && existing.agent == spec.id
7706 && agent::has_session(spec.kind, existing, sessions)
7707 {
7708 attempts.push((spec, existing.clone(), true));
7709 } else {
7710 let chain = agent::pick_chain(
7711 &self.state.config.agents,
7712 self.state.config.roles.language_judge.as_ref(),
7713 &agent::installed,
7714 "language_judge",
7715 );
7716 match chain {
7717 Ok(chain) => {
7718 for spec in chain {
7719 let seat = crate::fixer::seat_for(
7720 &mut self.state,
7721 "github-text-rewrite",
7722 &spec.id,
7723 );
7724 attempts.push((spec, seat, false));
7725 }
7726 }
7727 Err(e) => {
7728 tracing::warn!("no agent for the description rewrite: {e:#}");
7729 self.state
7730 .event("github-text", "no rewrite agent available");
7731 return None;
7732 }
7733 }
7734 }
7735 let prompts = self.state.config.prompts.clone();
7736 let cache = self.state.config.cache_dir();
7737 let run = self.state.id.clone();
7738 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7739 let job = SeatJob {
7740 spec: spec.clone(),
7741 seat,
7742 cwd: winner.worktree.clone(),
7743 prompt: prompt.clone(),
7744 timeout: retry_budget(timeout_full, resumed),
7745 allow_write: false,
7746 sessions,
7747 artifacts: agent::artifacts_dir(&self.state.dir()),
7748 stem: if i == 0 {
7749 "github-text-rewrite".to_owned()
7750 } else {
7751 format!("github-text-rewrite-{}", spec.id)
7752 },
7753 handover: None,
7754 };
7755 let ctx = WaveCtx {
7756 run: &run,
7757 node: "github-text",
7758 prompts: &prompts,
7759 cache: cache.as_deref(),
7760 round: None,
7761 carry_seats: false,
7762 pauses: Some((&self.pause, &self.interrupt)),
7763 };
7764 let (seat, outcome) =
7765 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7766 self.state.seats.insert(seat.key.clone(), seat);
7767 match outcome {
7768 AgentOutcome::Ok(o)
7769 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7770 {
7771 return Some(o.text);
7772 }
7773 AgentOutcome::ResourceBlocked(_) => return None,
7774 _ => {}
7775 }
7776 }
7777 None
7778 }
7779
7780 fn retire_github_text_question(&mut self, why: &str) {
7782 let Some(id) = self
7783 .state
7784 .github_text
7785 .as_ref()
7786 .and_then(|g| g.question.clone())
7787 else {
7788 return;
7789 };
7790 let _ = ask::Questions::open().update(&id, |q| {
7791 q.abandon(why);
7792 Ok(())
7793 });
7794 }
7795
7796 async fn settle_withheld(
7800 &mut self,
7801 winner: &Candidate,
7802 pr: &PrMessage,
7803 mut g: crate::run::GithubTextAsk,
7804 ) -> Result<Option<PrMessage>> {
7805 use crate::github_text as gt;
7806 let store = ask::Questions::open();
7807 let timeout = self.state.config.graph.answer_timeout;
7808 if !g.resolved {
7809 let mut standing = g
7810 .question
7811 .as_ref()
7812 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7813 if let Some(q) = standing.as_ref()
7814 && q.status.open()
7815 && gt::expired(q, timeout)
7816 {
7817 let why = format!("no answer within {}s of asking", timeout.max(1));
7818 standing = store
7819 .update(&q.id, |q| {
7820 q.abandon(&why);
7821 Ok(())
7822 })
7823 .ok()
7824 .map(|(q, ())| q);
7825 }
7826 let mut ask_again: Option<Option<Vec<String>>> = None;
7827 match standing {
7828 None if g.asks <= 1 => {
7831 g.asks = 0;
7832 ask_again = Some(None);
7833 }
7834 None => {
7835 g.resolved = true;
7836 g.chosen_title = None;
7837 g.choice = Some(GithubTextChoice::Fallback);
7838 }
7839 Some(q) if q.status.open() => {
7840 self.park_on_github_text(&g)?;
7841 return Ok(None);
7842 }
7843 Some(q) => match gt::read_reply(&q) {
7844 gt::Reply::Fallback => {
7845 g.resolved = true;
7846 g.chosen_title = None;
7847 g.choice = Some(GithubTextChoice::Fallback);
7848 }
7849 gt::Reply::Original => {
7850 g.resolved = true;
7851 g.chosen_title = None;
7852 g.choice = Some(GithubTextChoice::Original);
7853 }
7854 gt::Reply::Rewrite => {
7855 g.resolved = true;
7856 g.chosen_title = None;
7857 g.choice = Some(if g.rewrite.is_some() {
7858 GithubTextChoice::Rewrite
7859 } else {
7860 GithubTextChoice::Fallback
7861 });
7862 }
7863 gt::Reply::Title(text) => {
7864 let first = text
7865 .lines()
7866 .map(str::trim)
7867 .find(|l| !l.is_empty())
7868 .unwrap_or("");
7869 let decision = if first.is_empty() {
7870 None
7871 } else {
7872 let candidate = PrMessage {
7873 title: first.to_owned(),
7874 body: String::new(),
7875 };
7876 self.judge_pr_language(&winner.worktree, &candidate).await
7877 };
7878 match gt::vet_title(
7879 &text,
7880 decision,
7881 self.state.config.graph.github_english(),
7882 ) {
7883 Ok(title) => {
7884 g.resolved = true;
7885 g.chosen_title = Some(title);
7886 g.choice = Some(GithubTextChoice::MyTitle);
7887 }
7888 Err(cats) if g.asks < 2 => {
7889 ask_again = Some(Some(
7890 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7891 ))
7892 }
7893 Err(_) => {
7894 g.resolved = true;
7895 g.chosen_title = None;
7896 g.choice = Some(GithubTextChoice::Fallback);
7897 }
7898 }
7899 }
7900 },
7901 }
7902 if let Some(rejected) = ask_again {
7903 let w = gt::Withheld {
7906 title: g.title,
7907 body: g.body,
7908 categories: g.categories.clone(),
7909 };
7910 let artifact = match crate::run::write_artifact(
7913 &self.state,
7914 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7915 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7916 ) {
7917 Ok(p) => Some(p),
7918 Err(e) => {
7919 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7920 None
7921 }
7922 };
7923 let (detail, comparison) = gt::question_content(
7924 &w,
7925 &pr.title,
7926 &pr.body,
7927 &self.state.id,
7928 rejected.as_deref(),
7929 artifact.as_deref(),
7930 g.rewrite.as_ref(),
7931 );
7932 let mut q = ask::Question::new(
7933 self.state.id.clone(),
7934 gt::ASK_NODE.to_owned(),
7935 gt::ASK_SEAT.to_owned(),
7936 gt::question_summary(&self.state.config.graph.language, &w),
7937 detail,
7938 gt::question_choices(&w, g.rewrite.as_ref()),
7939 );
7940 q.posting_comparison = Some(comparison);
7941 q.answer_timeout = timeout;
7942 g.question = Some(q.id.clone());
7945 g.asks += 1;
7946 self.state.github_text = Some(g.clone());
7947 self.state.event(
7948 "github-text",
7949 format!("asking the owner about the withheld text ({})", q.short()),
7950 );
7951 self.state.save()?;
7952 store
7953 .put_executable(&mut q)
7954 .context("file the posting-gate question")?;
7955 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7956 tracing::warn!(
7957 "could not notify about posting-gate question {}: {e:#}",
7958 q.short()
7959 );
7960 }
7961 self.park_on_github_text(&g)?;
7962 return Ok(None);
7963 }
7964 }
7965 let resolved = g.chosen_title.clone();
7966 let original = g.choice == Some(GithubTextChoice::Original);
7967 self.state.event(
7969 "github-text",
7970 if original {
7971 format!(
7972 "posting the original text for: {}",
7973 match (g.title, g.body) {
7974 (true, true) => "title, description",
7975 (true, false) => "title",
7976 _ => "description",
7977 }
7978 )
7979 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7980 "posting the saved rewrite title and description".to_owned()
7981 } else if resolved.is_some() {
7982 "posting the owner's replacement title".to_owned()
7983 } else {
7984 "posting the neutral text for the withheld fields".to_owned()
7985 },
7986 );
7987 self.state.github_text = Some(g.clone());
7988 self.state.save()?;
7989 if g.choice == Some(GithubTextChoice::Rewrite)
7990 && let Some(r) = g.rewrite.as_ref()
7991 {
7992 return Ok(Some(PrMessage {
7993 title: r.title.clone(),
7994 body: r.body.clone(),
7995 }));
7996 }
7997 let w = gt::Withheld {
7998 title: g.title,
7999 body: g.body,
8000 categories: g.categories.clone(),
8001 };
8002 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8003 if let Some(t) = resolved {
8004 title = t;
8005 }
8006 let english = gt::LanguageDecision {
8008 title_english: true,
8009 body_english: true,
8010 body_complete: true,
8011 };
8012 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8013 if original {
8014 if g.title {
8017 title = pr.title.clone();
8018 }
8019 if g.body {
8020 body = pr.body.clone();
8021 }
8022 }
8023 Ok(Some(PrMessage { title, body }))
8024 }
8025
8026 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8027 self.state.github_text = Some(g.clone());
8028 self.state.end_pass_parked(ParkCause::Question);
8029 self.state.parked = true;
8030 self.state.event(
8031 "github-text",
8032 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8033 );
8034 self.state.save()
8035 }
8036
8037 async fn judge_pr_language(
8040 &mut self,
8041 cwd: &Path,
8042 pr: &PrMessage,
8043 ) -> Option<crate::github_text::LanguageDecision> {
8044 self.state.config.roles.language_judge.as_ref()?;
8045 if !self.state.config.graph.language_gate() {
8046 return None;
8047 }
8048 let decision =
8049 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8050 self.state.event(
8051 "github-text",
8052 if decision.is_some() {
8053 "language decided by the language judge"
8054 } else {
8055 "language judge unavailable; using built-in heuristics"
8056 },
8057 );
8058 decision
8059 }
8060
8061 async fn merge(&mut self) -> Result<()> {
8062 if self
8077 .state
8078 .base_sync
8079 .as_ref()
8080 .is_some_and(|s| s.conflict.is_some())
8081 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8082 != Some(RunStatus::Gating)
8083 || !self.state.gate_allows_progress()
8091 {
8092 return Ok(());
8093 }
8094 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8103 return Ok(());
8104 }
8105 let Some(winner) = self.state.winner().cloned() else {
8106 return Ok(());
8107 };
8108 let repo = self.state.repo.clone();
8109 let base = self.state.base_branch.clone();
8110 let mode = self.state.config.merge.mode;
8111 let style = self.state.config.merge.style;
8112 let facts = if is_review_run(&self.state) {
8113 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8114 let start = review_base(
8115 &repo,
8116 &self.state.config.merge.remote,
8117 &base,
8118 &self.state.base_commit,
8119 &winner.branch,
8120 )
8121 .await;
8122 branch_facts(&repo, &start, &winner.branch).await
8123 } else {
8124 None
8125 };
8126 let leaked = if is_review_run(&self.state) {
8129 leaked_subjects(&self.state, &winner.branch).await
8130 } else {
8131 Some(Vec::new())
8132 };
8133 let Some(pr) = self
8137 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8138 .await?
8139 else {
8140 return Ok(());
8141 };
8142 let message = pr.commit_message();
8143
8144 let outcome = match mode {
8145 MergeMode::None => MergeOutcome {
8146 mode,
8147 ok: true,
8148 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8149 empty: false,
8150 },
8151 MergeMode::Pr | MergeMode::Local
8152 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8153 {
8154 MergeOutcome {
8155 mode,
8156 ok: false,
8157 detail: empty_candidate_detail(&self.state, &base),
8158 empty: true,
8159 }
8160 }
8161 MergeMode::Local => {
8162 let on = git::current_branch(&repo).await?;
8163 if on.as_deref() != Some(base.as_str()) {
8164 MergeOutcome {
8165 mode,
8166 ok: false,
8167 detail: format!(
8168 "{} has {} checked out, not the base branch {base}",
8169 repo.display(),
8170 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8171 ),
8172 empty: false,
8173 }
8174 } else if !git::is_clean(&repo).await? {
8175 MergeOutcome {
8176 mode,
8177 ok: false,
8178 detail: format!("{} is dirty; refusing to merge", repo.display()),
8179 empty: false,
8180 }
8181 } else {
8182 let out = match style {
8183 MergeStyle::Merge => {
8184 git::merge_no_ff(&repo, &winner.branch, &message).await?
8185 }
8186 MergeStyle::Squash => {
8187 git::merge_squash(&repo, &winner.branch, &message).await?
8188 }
8189 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8190 };
8191 MergeOutcome {
8192 mode,
8193 ok: out.ok(),
8194 detail: if out.ok() { out.stdout } else { out.stderr },
8195 empty: false,
8196 }
8197 }
8198 }
8199 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8200 self.state.pr_publication_pending = true;
8201 self.state.save()?;
8202 let out =
8203 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8204 self.state.pr_publication_pending = out.is_err();
8205 MergeOutcome {
8206 mode,
8207 ok: out.is_ok(),
8208 detail: match out {
8209 Ok(url) => url,
8210 Err(e) => format!("{e:#}"),
8211 },
8212 empty: false,
8213 }
8214 }
8215 MergeMode::Pr => {
8216 let remote = self.state.config.merge.remote.clone();
8217 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8218 if !pushed.ok() {
8219 MergeOutcome {
8220 mode,
8221 ok: false,
8222 detail: pushed.stderr,
8223 empty: false,
8224 }
8225 } else {
8226 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8232 let out = match pr_merge_plan(found) {
8233 PrPlan::Create => {
8234 gh_pr_create(
8235 &winner.worktree,
8236 &base,
8237 &winner.branch,
8238 &pr.title,
8239 &pr.body,
8240 )
8241 .await
8242 }
8243 PrPlan::Adopt { url, title } => {
8244 self.state
8245 .event("merge", format!("Pr: adopted open pull request {url}"));
8246 if self.state.github_text.as_ref().is_some_and(|g| {
8247 g.resolved
8248 && g.choice == Some(GithubTextChoice::Rewrite)
8249 && g.rewrite.is_some()
8250 }) {
8251 land::set_pr_text_verbatim(
8254 &winner.worktree,
8255 &url,
8256 &pr.title,
8257 &pr.body,
8258 )
8259 .await
8260 .map(|()| url)
8261 } else {
8262 if title != pr.title
8263 && (!is_review_run(&self.state)
8264 || leaked
8265 .as_deref()
8266 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8267 && let Err(e) =
8268 if self.state.github_text.as_ref().is_some_and(|g| {
8269 g.resolved
8270 && g.title
8271 && g.choice == Some(GithubTextChoice::Original)
8272 }) {
8273 land::set_pr_title_verbatim(
8275 &winner.worktree,
8276 &url,
8277 &pr.title,
8278 )
8279 .await
8280 } else {
8281 land::set_pr_title(
8282 &mut self.state,
8283 &winner.worktree,
8284 &url,
8285 &pr.title,
8286 )
8287 .await
8288 }
8289 {
8290 tracing::warn!("could not refresh title of {url}: {e:#}");
8291 self.state
8292 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8293 }
8294 Ok(url)
8295 }
8296 }
8297 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8298 };
8299 match out {
8300 Ok(url) => MergeOutcome {
8301 mode,
8302 ok: true,
8303 detail: url,
8304 empty: false,
8305 },
8306 Err(e) => MergeOutcome {
8307 mode,
8308 ok: false,
8309 detail: e.to_string(),
8310 empty: false,
8311 },
8312 }
8313 }
8314 }
8315 };
8316
8317 self.state.status = match (mode, outcome.ok) {
8318 (MergeMode::None, _) => RunStatus::Ready,
8319 (_, true) if outcome.empty => RunStatus::Ready,
8320 (MergeMode::Pr, true) => RunStatus::Ready,
8321 (_, true) => RunStatus::Merged,
8322 (_, false) => RunStatus::Blocked,
8323 };
8324 self.state.event(
8325 "merge",
8326 format!(
8327 "{:?}: {}",
8328 mode,
8329 outcome.detail.lines().next().unwrap_or("")
8330 ),
8331 );
8332 self.state.merge = Some(outcome);
8333 self.state.save()?;
8334
8335 if self.state.config.graph.land
8341 && mode == MergeMode::Pr
8342 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8343 {
8344 self.run_land().await?;
8345 }
8346 self.settle_questions();
8351 Ok(())
8352 }
8353
8354 async fn run_land(&mut self) -> Result<()> {
8365 crate::pr_head::ensure_owner(&self.state)?;
8366 let url = self
8367 .state
8368 .merge
8369 .as_ref()
8370 .map(|m| m.detail.clone())
8371 .unwrap_or_default();
8372 let url = url.lines().next().unwrap_or("").trim().to_owned();
8373 if !url.starts_with("http") {
8374 return Ok(());
8375 }
8376 loop {
8379 match land::land(&mut self.state, &url).await {
8380 Ok(_)
8381 if self
8382 .state
8383 .merge_repairs
8384 .last()
8385 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8386 {
8387 if self.repair_merge_review().await? {
8388 continue;
8389 }
8390 }
8391 Ok(pr) if self.state.parked => {
8392 let _ = pr;
8396 }
8397 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8398 self.state.status = RunStatus::Superseded;
8399 self.state.save()?;
8400 }
8401 Ok(pr) => {
8402 self.state.status = match pr.state {
8403 land::PrLifecycle::Merged => RunStatus::Merged,
8404 _ => RunStatus::Blocked,
8405 };
8406 if bump::should_release_bump(self.state.status)
8413 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8414 {
8415 self.state
8421 .event("bump", format!("release bump skipped: {e:#}"));
8422 }
8423 if self.state.status == RunStatus::Merged {
8426 crate::followup::after_merge(&mut self.state, &pr.url).await;
8427 }
8428 self.state.save()?;
8429 }
8430 Err(e) => {
8431 if e.is::<crate::repo_lock::ResourceBlocked>() {
8432 return Err(e);
8433 }
8434 self.state.status = RunStatus::Blocked;
8435 self.state.event("land", format!("gave up: {e}"));
8436 self.state.save()?;
8437 }
8438 }
8439 break;
8440 }
8441 Ok(())
8442 }
8443
8444 async fn repair_merge_review(&mut self) -> Result<bool> {
8447 use crate::run::MergeRepairPhase as Phase;
8448 crate::pr_head::ensure_owner(&self.state)?;
8449 let i = self.state.merge_repairs.len() - 1;
8450 let request = self.state.merge_repairs[i].clone();
8451 if request.phase == Phase::Fixing {
8452 let restored = match self.state.winner().cloned() {
8453 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8454 None => Err(anyhow::anyhow!("repair has no winning branch")),
8455 };
8456 let why = match restored {
8457 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8458 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8459 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8460 };
8461 self.hold_merge_repair(i, why)?;
8462 return Ok(false);
8463 }
8464 let failure = if i >= self.state.config.graph.land_rounds
8465 || self.state.config.graph.review_rounds == 0
8466 {
8467 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8468 } else {
8469 None
8470 };
8471 if let Some(why) = failure {
8472 self.hold_merge_repair(i, why)?;
8473 return Ok(false);
8474 }
8475 if request.phase == Phase::Requested {
8476 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8477 Ok(pr) => pr,
8478 Err(e) => {
8479 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8480 return Ok(false);
8481 }
8482 };
8483 self.state.merge_repairs[i].phase = Phase::Fixing;
8484 self.state.land_approval = None;
8485 self.state.land_armed_head = None;
8486 self.state.save()?;
8487 let evidence = serde_json::to_string_pretty(&request)?;
8490 let reason = format!(
8491 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8492 );
8493 let budget = self.state.config.graph.land_rounds;
8494 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8495 match result {
8496 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8497 Ok(land::Fixed::Failed(why)) => {
8498 self.hold_merge_repair(
8499 i,
8500 format!(
8501 "repair failed: {why}; inspect the branch and run an explicit review"
8502 ),
8503 )?;
8504 return Ok(false);
8505 }
8506 Err(e) => {
8507 self.hold_merge_repair(
8508 i,
8509 format!(
8510 "repair failed: {e:#}; inspect the branch and run an explicit review"
8511 ),
8512 )?;
8513 return Ok(false);
8514 }
8515 }
8516 self.begin_merge_repair_review(i)?;
8520 }
8521 self.review_loop().await?;
8522 self.gate().await?;
8523 if self.resource_halt()? || self.state.parked {
8524 return Ok(false);
8525 }
8526 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8527 != Some(RunStatus::Gating)
8528 || !self.state.gate_allows_progress()
8529 {
8530 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8531 return Ok(false);
8532 }
8533 let winner = self
8536 .state
8537 .winner()
8538 .cloned()
8539 .context("repair has no winning branch")?;
8540 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8541 if self.state.gate_head.as_deref() != Some(after.as_str())
8542 || !git::is_clean(&winner.worktree).await?
8543 {
8544 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8545 return Ok(false);
8546 }
8547 let expected = self
8548 .state
8549 .pr_pushed_head
8550 .as_deref()
8551 .unwrap_or(&request.head)
8552 .to_owned();
8553 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8554 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8557 let refspec = format!("{after}:refs/heads/{public}");
8558 let push = git::git_raw(
8559 &winner.worktree,
8560 &["push", &lease, &self.state.config.merge.remote, &refspec],
8561 )
8562 .await;
8563 match push {
8564 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8565 Ok(out) => {
8566 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8567 return Ok(false);
8568 }
8569 Err(e) => {
8570 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8571 return Ok(false);
8572 }
8573 }
8574 self.state.merge_repairs[i].phase = Phase::Complete;
8575 self.state.status = RunStatus::Landing;
8576 self.state.event(
8577 "repair",
8578 "new review and gate completed; fresh owner merge decision required",
8579 );
8580 self.state.save()?;
8581 Ok(true)
8582 }
8583
8584 fn merge_repair_artifacts(&self) -> PathBuf {
8585 let artifacts = agent::artifacts_dir(&self.state.dir());
8586 match self
8587 .state
8588 .merge_repairs
8589 .last()
8590 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8591 {
8592 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8593 None => artifacts,
8594 }
8595 }
8596
8597 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8598 self.state.reviews.clear();
8599 self.state.gate.clear();
8600 self.state.gate_ran = false;
8601 self.state.gate_head = None;
8602 self.state.gate_fixes.clear();
8603 self.state.pre_gate.clear();
8604 self.state.pre_gate_commit = None;
8605 self.state.contested_handoff = None;
8606 self.state.land_reruns = None;
8607 self.state.land_approval = None;
8608 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8609 self.state.status = RunStatus::Reviewing;
8610 self.state.save()
8611 }
8612
8613 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8614 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8615 self.state.merge_repairs[index].failure = Some(why.clone());
8616 self.state.status = RunStatus::Blocked;
8617 self.state.event("repair", why);
8618 self.state.save()
8619 }
8620
8621 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8625 if let Some(existing) = self.state.seats.get(key)
8626 && existing.agent == agent
8627 {
8628 return existing.clone();
8629 }
8630 let fresh = if self.state.seats.contains_key(key) {
8633 handover_seat(key, agent, self.state.next_seat_seed())
8634 } else {
8635 SeatState::new(key, agent, self.state.seed)
8636 };
8637 self.state.seats.insert(key.to_owned(), fresh.clone());
8638 fresh
8639 }
8640
8641 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8647 match self.state.seats.get(key) {
8648 Some(s) if s.agent != spec.id => {
8649 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8650 }
8651 _ => spec,
8652 }
8653 }
8654
8655 fn view(&self, c: &Candidate) -> CandidateView {
8657 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8658 .unwrap_or_default();
8659 let (patch, _) = blind::sanitize_patch(
8660 &format!("candidate {} patch", c.label),
8661 &raw,
8662 &self.state.config.blind,
8663 );
8664 CandidateView {
8665 label: c.label,
8666 branch: c.branch.clone(),
8667 summary: c.summary.clone(),
8668 stat: c.stat.clone(),
8669 patch,
8670 }
8671 }
8672
8673 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8675 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8676 prompt::judge(
8677 "(see above)",
8678 &views,
8679 self.roles.judges.len(),
8680 base_short,
8681 "en",
8682 )
8683 }
8684
8685 fn vote_prompt_full(
8691 &self,
8692 j: usize,
8693 labels: &[char],
8694 language: &str,
8695 candidates: &[Candidate],
8696 base_short: &str,
8697 ) -> String {
8698 let mut text = format!(
8699 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8700 prompt::final_vote(labels, language),
8701 self.state.instruction,
8702 self.candidate_block(candidates, base_short)
8703 );
8704 if let Some(own) = self
8705 .state
8706 .judgements
8707 .get(j)
8708 .filter(|r| !r.ranking.is_empty())
8709 {
8710 let reasons = own
8711 .reasons
8712 .iter()
8713 .map(|(k, v)| format!("- {k}: {v}"))
8714 .collect::<Vec<_>>()
8715 .join("\n");
8716 text.push_str(&format!(
8717 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8718 own.ranking.iter().collect::<String>(),
8719 if reasons.is_empty() {
8720 ""
8721 } else {
8722 ", because:\n"
8723 }
8724 ));
8725 }
8726 if !self.state.deliberation.is_empty() {
8727 text.push_str("\n# What was argued before this vote\n");
8728 for t in self.transcript(&[], j) {
8729 text.push_str(&format!(
8730 "\n## {}{}\n\n{}\n",
8731 t.who,
8732 if t.is_self { " (you)" } else { "" },
8733 t.body.trim()
8734 ));
8735 }
8736 }
8737 text
8738 }
8739
8740 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8747 let mut turns = Vec::new();
8748 for j in &self.state.judgements {
8749 if j.ranking.is_empty() {
8750 continue;
8751 }
8752 let reasons = j
8753 .reasons
8754 .iter()
8755 .map(|(k, v)| format!("- {k}: {v}"))
8756 .collect::<Vec<_>>()
8757 .join("\n");
8758 turns.push(Turn {
8759 who: format!("Judge {} (opening ranking)", j.judge),
8760 is_self: j.judge == self_idx + 1,
8761 body: format!(
8762 "Ranked {}{}{reasons}",
8763 j.ranking.iter().collect::<String>(),
8764 if reasons.is_empty() {
8765 ""
8766 } else {
8767 ", because:\n"
8768 }
8769 ),
8770 });
8771 }
8772 for t in self
8773 .state
8774 .deliberation
8775 .iter()
8776 .flat_map(|r| r.turns.iter())
8777 .chain(current)
8778 {
8779 turns.push(Turn {
8780 who: format!("Judge {}", t.judge),
8781 is_self: t.judge == self_idx + 1,
8782 body: t.body.clone(),
8783 });
8784 }
8785 turns
8786 }
8787}
8788
8789fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8791 agent::has_session(spec.kind, seat, sessions)
8792}
8793
8794fn next_untried_in_roster<'a>(
8815 roster: &'a [AgentSpec],
8816 start: usize,
8817 tried: &BTreeSet<String>,
8818) -> Option<&'a AgentSpec> {
8819 roster
8820 .get(start + 1..)?
8821 .iter()
8822 .find(|s| !tried.contains(&s.id))
8823}
8824
8825fn pick_successor<'a>(
8846 roster: &'a [AgentSpec],
8847 start: usize,
8848 tried: &BTreeSet<String>,
8849 carried: Option<&BTreeSet<String>>,
8850 others: &BTreeSet<String>,
8851) -> Option<&'a AgentSpec> {
8852 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8853 roster
8854 .get(start + 1..)
8855 .and_then(|tail| {
8856 tail.iter()
8857 .filter(free)
8858 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8859 })
8860 .or_else(|| {
8861 carried
8862 .is_some()
8863 .then(|| roster.iter().find(free))
8864 .flatten()
8865 })
8866 .or_else(|| match carried {
8867 Some(c) => next_for_seat(roster, start, tried, c),
8868 None => next_untried_in_roster(roster, start, tried),
8869 })
8870}
8871
8872fn next_for_seat<'a>(
8882 roster: &'a [AgentSpec],
8883 start: usize,
8884 round_tried: &BTreeSet<String>,
8885 carried_failed: &BTreeSet<String>,
8886) -> Option<&'a AgentSpec> {
8887 roster
8888 .get(start + 1..)?
8889 .iter()
8890 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8891 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8892}
8893
8894fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8900 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8901 return spec;
8902 };
8903 let ok = |id: &str| !h.failed.contains(id);
8904 if let Some(last) = h.last_ok.as_deref()
8905 && ok(last)
8906 && let Some(s) = roster.iter().find(|s| s.id == last)
8907 {
8908 return s.clone();
8909 }
8910 if ok(&spec.id) {
8911 return spec;
8912 }
8913 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8914 roster
8915 .get(from + 1..)
8916 .into_iter()
8917 .flatten()
8918 .chain(roster.iter())
8919 .find(|s| ok(&s.id))
8920 .cloned()
8921 .unwrap_or(spec)
8922}
8923
8924pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8928 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8929}
8930
8931const TIMED_OUT: &str = "timed out";
8934
8935impl FailClass {
8936 fn of(out: &AgentOutcome) -> Option<Self> {
8938 match out {
8939 AgentOutcome::Ok(_) => None,
8940 AgentOutcome::Quota(_) => Some(Self::Quota),
8941 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8942 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8943 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8944 AgentOutcome::ResourceBlocked(_) => None,
8947 }
8948 }
8949
8950 fn stem_word(&self) -> &'static str {
8952 match self {
8953 Self::Quota => "quota",
8954 _ => "handover",
8955 }
8956 }
8957}
8958
8959fn failure_signature(msg: &str) -> String {
8963 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8964 if let Some(code) = line
8967 .strip_prefix("process exited with no usable output (exit code ")
8968 .and_then(|s| s.strip_suffix(')'))
8969 .and_then(|s| s.parse::<i32>().ok())
8970 {
8971 line = format!("exited with some({code}) and no usable output");
8972 } else if matches!(
8973 line.as_str(),
8974 "process exited with no usable output (no exit code reported)"
8975 | "process exited with no usable output (exit code unavailable)"
8976 ) {
8977 line = "exited with none and no usable output".to_owned();
8978 }
8979 let mut out = Vec::new();
8980 for word in line.split_whitespace() {
8981 if word.contains('/') || word.contains('\\') {
8982 out.push("<path>".to_owned());
8983 continue;
8984 }
8985 let mut w = String::new();
8986 let mut in_digits = false;
8987 for c in word.chars() {
8988 if c.is_ascii_digit() {
8989 if !in_digits {
8990 w.push('#');
8991 }
8992 in_digits = true;
8993 } else {
8994 in_digits = false;
8995 w.push(c);
8996 }
8997 }
8998 out.push(w);
8999 }
9000 out.join(" ").chars().take(120).collect()
9001}
9002
9003fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9010 match cur {
9011 FailClass::Quota | FailClass::Timeout => true,
9012 FailClass::Other(_) => prev != Some(cur),
9013 }
9014}
9015
9016fn fail_reason(out: &AgentOutcome) -> String {
9018 match out {
9019 AgentOutcome::Ok(_) => String::new(),
9020 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9021 AgentOutcome::Dropped(o) => format!(
9022 "the CLI dropped the stream ({})",
9023 o.dropped
9024 .as_ref()
9025 .map(|d| d.why.as_str())
9026 .unwrap_or("it ended without delivering its answer")
9027 ),
9028 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9029 e.lines().next().unwrap_or("").chars().take(160).collect()
9030 }
9031 }
9032}
9033
9034pub(crate) fn record_handover(
9037 state: &mut RunState,
9038 node: &str,
9039 seat: &str,
9040 from: &str,
9041 to: &str,
9042 class: &FailClass,
9043 reason: &str,
9044) {
9045 let message = if *class == FailClass::Quota {
9046 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9047 } else {
9048 format!("{seat}: handed over {from} -> {to} ({reason})")
9049 };
9050 state.event(node, message);
9051 state.handovers.push(Handover {
9052 at: Timestamp::now(),
9053 node: node.to_owned(),
9054 seat: seat.to_owned(),
9055 from: from.to_owned(),
9056 to: to.to_owned(),
9057 reason: reason.to_owned(),
9058 });
9059}
9060
9061fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9075 commands.iter().any(|c| c.exit_code.is_none())
9076}
9077
9078fn verified_noop_claim(
9091 usable: bool,
9092 commands: &[agent::CommandEvidence],
9093 text: &str,
9094) -> Option<String> {
9095 (usable && !has_unconfirmed_command(commands))
9096 .then(|| verdict::verified_noop(text))
9097 .flatten()
9098}
9099
9100fn short(commit: &str) -> String {
9101 commit.chars().take(7).collect()
9102}
9103
9104fn make_executable(path: &Path) -> Result<()> {
9105 #[cfg(unix)]
9106 {
9107 use std::os::unix::fs::PermissionsExt as _;
9108 let mut perms = std::fs::metadata(path)?.permissions();
9109 perms.set_mode(0o755);
9110 std::fs::set_permissions(path, perms)?;
9111 }
9112 #[cfg(not(unix))]
9113 {
9114 let _ = path;
9115 }
9116 Ok(())
9117}
9118
9119struct WaveCtx<'a> {
9126 run: &'a str,
9129 node: &'a str,
9131 prompts: &'a Prompts,
9132 cache: Option<&'a Path>,
9134 round: Option<usize>,
9137 carry_seats: bool,
9141 pauses: Option<(&'a Pause, &'a Pause)>,
9144}
9145
9146fn repository_wait(
9148 state: &RunState,
9149 pauses: Option<(&Pause, &Pause)>,
9150) -> crate::repo_lock::WaitPolicy {
9151 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9152 let inherited = crate::repo_lock::WaitPolicy::current();
9153 crate::repo_lock::WaitPolicy::new(
9154 Duration::from_secs(state.config.graph.timeout_resource_wait),
9155 move || {
9156 inherited.cancelled()
9157 || pauses
9158 .as_ref()
9159 .is_some_and(|(a, b)| a.parked() || b.parked())
9160 },
9161 )
9162}
9163
9164fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9165 if error.is::<crate::repo_lock::ResourceBlocked>() {
9166 state.resource_block = Some(crate::run::ResourceBlock {
9167 node: "repository".into(),
9168 kind: "repository coordination".into(),
9169 detail: error.to_string(),
9170 at: Timestamp::now(),
9171 });
9172 state.event("resource", error.to_string());
9173 }
9174}
9175
9176async fn coordinate_operation(
9177 state: &mut RunState,
9178 cwd: &Path,
9179 wait: &crate::repo_lock::WaitPolicy,
9180) -> Result<Option<crate::repo_lock::SeatLease>> {
9181 let result =
9182 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9183 .await;
9184 if let Err(e) = &result {
9185 record_repository_block(state, e);
9186 }
9187 result
9188}
9189
9190async fn run_one(
9192 job: SeatJob,
9193 sem: Arc<Semaphore>,
9194 ctx: &WaveCtx<'_>,
9195 state: &mut RunState,
9196 attempt: usize,
9197) -> (SeatState, AgentOutcome) {
9198 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9199 .await
9200 .pop()
9201 .expect("one job in, one result out");
9202 (seat, out)
9203}
9204
9205async fn wave(
9211 jobs: Vec<SeatJob>,
9212 sem: Arc<Semaphore>,
9213 ctx: &WaveCtx<'_>,
9214 state: &mut RunState,
9215 attempt: usize,
9216) -> Vec<(usize, SeatState, AgentOutcome)> {
9217 let WaveCtx {
9218 run,
9219 node,
9220 prompts,
9221 cache,
9222 round,
9223 carry_seats: _,
9224 pauses,
9225 } = *ctx;
9226 for job in &jobs {
9227 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9228 }
9229 if let Err(e) = state.save() {
9230 tracing::warn!("could not persist in-progress seats: {e:#}");
9235 }
9236 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9252 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9256 let cache_guard = if let Some(cache_dir) = cache {
9257 if jobs_had_a_writer {
9258 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9259 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9260 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9261 {
9262 Ok(Some(g)) => Some(g),
9263 Ok(None) => {
9266 state.resource_block = Some(crate::run::ResourceBlock {
9267 node: node.to_owned(),
9268 kind: "shared build cache".to_owned(),
9269 detail: "the run is parking".to_owned(),
9270 at: Timestamp::now(),
9271 });
9272 return block_wave(
9273 jobs,
9274 state,
9275 node,
9276 attempt,
9277 "parked while waiting for the shared build cache",
9278 );
9279 }
9280 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9283 Err(e) => {
9284 let why = format!("{e:#}");
9288 state.resource_block = Some(crate::run::ResourceBlock {
9289 node: node.to_owned(),
9290 kind: "shared build cache".to_owned(),
9291 detail: why.clone(),
9292 at: Timestamp::now(),
9293 });
9294 state.event(
9295 "resource",
9296 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9297 );
9298 return block_wave(jobs, state, node, attempt, &why);
9299 }
9300 }
9301 } else {
9302 None
9303 }
9304 } else {
9305 None
9306 };
9307 state.seats_spawning(node, attempt);
9310 if let Err(e) = state.save() {
9311 tracing::warn!("could not persist the start of a wave: {e:#}");
9312 }
9313 let slot_limit = resource_limit
9317 + jobs
9318 .iter()
9319 .map(|j| j.timeout)
9320 .max()
9321 .unwrap_or_default()
9322 .saturating_mul(jobs.len() as u32);
9323 let mut set = tokio::task::JoinSet::new();
9324 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9327 let overlay = prompts.overlay(node);
9328 let repository_wait = repository_wait(state, pauses);
9329 for (i, mut job) in jobs.into_iter().enumerate() {
9330 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9331 if cache.is_some() {
9332 job.prompt.push('\n');
9333 job.prompt
9334 .push_str(&prompt::build_cache_note(node, job.allow_write));
9335 }
9336 let sem = Arc::clone(&sem);
9337 let run = run.to_owned();
9338 let node = node.to_owned();
9339 let attachments = if node == "implement" {
9342 state.attachments.clone()
9343 } else {
9344 Vec::new()
9345 };
9346 let cache = cache
9357 .filter(|_| job.allow_write && cache_guard.is_some())
9358 .map(Path::to_path_buf);
9359 let spawned_tx = spawned_tx.clone();
9360 let repository_wait = repository_wait.clone();
9361 let coordination = state.config.coordination.clone();
9362 let fetched_base = state.fetched_base.clone();
9363 set.spawn(async move {
9364 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9368 Ok(p) => p,
9369 Err(_) => {
9370 return (
9371 i,
9372 job.seat,
9373 AgentOutcome::ResourceBlocked(format!(
9374 "no parallel slot became free within {}s",
9375 slot_limit.as_secs()
9376 )),
9377 );
9378 }
9379 };
9380 spawned_tx.send(job.seat.key.clone()).ok();
9381 let mut seat = job.seat;
9382 let out = repository_wait
9383 .scope(agent::invoke(
9384 &job.spec,
9385 &mut seat,
9386 &Invocation {
9387 scope: crate::agent::InvocationScope::Repository,
9388 coordination: coordination.as_ref(),
9389 fetched_base: fetched_base.as_ref(),
9390 cwd: &job.cwd,
9391 prompt: &job.prompt,
9392 timeout: job.timeout,
9393 allow_write: job.allow_write,
9394 unsandboxed: false,
9395 sessions: job.sessions,
9396 artifacts: &job.artifacts,
9397 stem: &job.stem,
9398 run: &run,
9399 node: &node,
9400 cache_dir: cache.as_deref(),
9401 attachments: &attachments,
9402 writable: &[],
9403 },
9404 ))
9405 .await;
9406 let out = match out {
9407 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9408 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9409 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9417 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9418 Ok(o) => AgentOutcome::Failed(format!(
9419 "process exited with no usable output ({})",
9420 crate::notices::exit_status(o.exit_code)
9421 )),
9422 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9423 AgentOutcome::ResourceBlocked(e.to_string())
9424 }
9425 Err(e) => AgentOutcome::Failed(e.to_string()),
9426 };
9427 (i, seat, out)
9428 });
9429 }
9430 drop(spawned_tx);
9431 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9432 loop {
9433 let joined = tokio::select! {
9434 Some(key) = spawned_rx.recv() => {
9435 state.seat_spawned(&key);
9437 if let Err(e) = state.save() {
9438 tracing::warn!("could not persist a seat's start: {e:#}");
9439 }
9440 continue;
9441 }
9442 joined = set.join_next() => joined,
9443 };
9444 let Some(joined) = joined else {
9445 break;
9446 };
9447 let (i, seat, out) = match joined {
9448 Ok(v) => v,
9449 Err(e) => {
9453 tracing::error!("agent task panicked: {e}");
9454 continue;
9455 }
9456 };
9457 state.seat_finished(&seat.key);
9458 if let AgentOutcome::ResourceBlocked(why) = &out
9459 && state.resource_block.is_none()
9460 {
9461 state.resource_block = Some(crate::run::ResourceBlock {
9462 node: node.to_owned(),
9463 kind: if why.contains("repository coordination") {
9464 "repository coordination"
9465 } else {
9466 "parallel slot"
9467 }
9468 .to_owned(),
9469 detail: why.clone(),
9470 at: Timestamp::now(),
9471 });
9472 state.event("resource", format!("{node}: no agent was started - {why}"));
9473 }
9474 record_jobs(state, node, round, &seat.key, &out);
9475 if let Err(e) = state.save() {
9476 tracing::warn!("could not persist a seat's completion: {e:#}");
9477 }
9478 if collected.len() <= i {
9479 collected.resize_with(i + 1, || None);
9480 }
9481 collected[i] = Some((i, seat, out));
9482 }
9483 if state
9489 .active
9490 .values()
9491 .any(|a| a.node == node && a.attempt == attempt)
9492 {
9493 state
9494 .active
9495 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9496 if let Err(e) = state.save() {
9497 tracing::warn!("could not persist the end of a wave: {e:#}");
9498 }
9499 }
9500 if let Some(cache_dir) = cache
9507 && jobs_had_a_writer
9508 {
9509 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9510 }
9511 if let Some(guard) = cache_guard {
9512 guard.release();
9513 }
9514 collected.into_iter().flatten().collect()
9515}
9516
9517fn record_jobs(
9528 state: &mut RunState,
9529 node: &str,
9530 round: Option<usize>,
9531 seat: &str,
9532 out: &AgentOutcome,
9533) {
9534 let commands: &[agent::CommandEvidence] = match out {
9535 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9536 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9537 };
9538 let checked_at = Timestamp::now();
9539 for c in commands {
9540 state.jobs.push(JobRecord {
9541 node: node.to_owned(),
9542 round,
9543 seat: seat.to_owned(),
9544 id: c.id.clone(),
9545 description: c.description.clone(),
9546 checked_at,
9547 status: match c.exit_code {
9548 Some(0) => JobStatus::Completed,
9549 Some(_) => JobStatus::Failed,
9550 None => JobStatus::Unknown,
9551 },
9552 exit_code: c.exit_code,
9553 result_summary: c.result_summary.clone(),
9554 source: c.source.clone(),
9555 });
9556 }
9557}
9558
9559fn round_is_clean(
9580 blocking: usize,
9581 e2e_ok: bool,
9582 answered: usize,
9583 expected: usize,
9584 quota_missing: usize,
9585 policy: IncompleteReviewPolicy,
9586) -> bool {
9587 if blocking != 0 || !e2e_ok {
9588 return false;
9589 }
9590 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9591 return true;
9592 }
9593 answered > 0 && expected - answered <= quota_missing
9594}
9595
9596fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9620 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9621 return Some(RunStatus::Gating);
9622 }
9623 let last = reviews.last()?;
9624 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9625 if reviews.len() < max_rounds && !stagnant {
9626 return None;
9627 }
9628 if last.incomplete() && last.blocking == 0 {
9629 return Some(RunStatus::Blocked);
9630 }
9631 if last.e2e_status() == E2eStatus::ResourceBlocked {
9632 return None;
9633 }
9634 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9635 RunStatus::Gating
9636 } else {
9637 RunStatus::Blocked
9638 })
9639}
9640
9641pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9656 if nudged {
9657 (full / 4).max(Duration::from_secs(120)).min(full)
9658 } else {
9659 full
9660 }
9661}
9662
9663#[allow(clippy::too_many_arguments)]
9692async fn ask_json_wave<T>(
9693 jobs: Vec<SeatJob>,
9694 sem: Arc<Semaphore>,
9695 retries: usize,
9696 roster: &[AgentSpec],
9697 ctx: &WaveCtx<'_>,
9698 losses: &mut Vec<QuotaLoss>,
9699 state: &mut RunState,
9700 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9701) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9702where
9703 T: serde::de::DeserializeOwned + Send + 'static,
9704{
9705 ask_wave_with(
9706 jobs,
9707 sem,
9708 retries,
9709 roster,
9710 ctx,
9711 losses,
9712 state,
9713 &|text: &str| {
9714 let v = verdict::extract_json::<T>(text)?;
9715 validate(&v)?;
9716 Ok(v)
9717 },
9718 )
9719 .await
9720}
9721
9722#[allow(clippy::too_many_arguments)]
9731async fn ask_wave_with<T>(
9732 jobs: Vec<SeatJob>,
9733 sem: Arc<Semaphore>,
9734 retries: usize,
9735 roster: &[AgentSpec],
9736 ctx: &WaveCtx<'_>,
9737 losses: &mut Vec<QuotaLoss>,
9738 state: &mut RunState,
9739 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9740) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9741where
9742 T: Send + 'static,
9743{
9744 let n = jobs.len();
9745 let originals: Vec<SeatJob> = jobs;
9746 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9747 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9748 let mut nudges: Vec<usize> = vec![0; n];
9755 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9759 let mut tried: Vec<BTreeSet<String>> = specs
9760 .iter()
9761 .map(|s| BTreeSet::from([s.id.clone()]))
9762 .collect();
9763 let starts: Vec<usize> = specs
9764 .iter()
9765 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9766 .collect();
9767 let carry = ctx.carry_seats && !roster.is_empty();
9768 let carried: Vec<BTreeSet<String>> = originals
9771 .iter()
9772 .map(|j| {
9773 state
9774 .seat_history
9775 .get(&j.seat.key)
9776 .filter(|_| carry)
9777 .map(|h| h.failed.clone())
9778 .unwrap_or_default()
9779 })
9780 .collect();
9781 let mut prev: Vec<Option<FailClass>> = originals
9782 .iter()
9783 .map(|j| {
9784 state
9785 .seat_history
9786 .get(&j.seat.key)
9787 .filter(|_| carry)
9788 .and_then(|h| h.last_fail.clone())
9789 })
9790 .collect();
9791 let next_agent =
9792 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9793 let others: BTreeSet<String> = specs
9794 .iter()
9795 .enumerate()
9796 .filter(|(j, _)| *j != i)
9797 .map(|(_, s)| s.id.clone())
9798 .collect();
9799 pick_successor(
9800 roster,
9801 starts[i],
9802 tried,
9803 carry.then(|| &carried[i]),
9804 &others,
9805 )
9806 .cloned()
9807 };
9808 let mut fresh: Vec<Option<String>> = vec![None; n];
9809 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9810 let mut pending: Vec<usize> = (0..n).collect();
9811
9812 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9817 for round in 0..max_rounds {
9818 if pending.is_empty() {
9819 break;
9820 }
9821 let mut batch = Vec::with_capacity(pending.len());
9822 let mut renudged: Vec<&str> = Vec::new();
9823 for &i in &pending {
9824 let src = &originals[i];
9825 let full: &str = match &src.handover {
9830 Some(h) if specs[i].id != src.spec.id => h,
9831 _ => &src.prompt,
9832 };
9833 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9836 (
9837 full.to_owned(),
9838 src.timeout,
9839 format!("{}-{word}-{}", src.stem, specs[i].id),
9840 )
9841 } else if nudges[i] == 0 {
9842 (full.to_owned(), src.timeout, src.stem.clone())
9843 } else {
9844 renudged.push(src.seat.key.as_str());
9845 let why = done[i]
9846 .as_ref()
9847 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9848 .unwrap_or_else(|| "no parsable answer".to_owned());
9849 let nudge = prompt::nudge(&why);
9850 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9851 let prompt = if nudged {
9852 nudge
9853 } else {
9854 format!("{full}\n\n---\n\n{nudge}")
9855 };
9856 (
9857 prompt,
9858 retry_budget(src.timeout, nudged),
9859 format!("{}-retry{}", src.stem, nudges[i]),
9860 )
9861 };
9862 batch.push(SeatJob {
9863 spec: specs[i].clone(),
9864 seat: seats[i].clone(),
9865 cwd: src.cwd.clone(),
9866 prompt,
9867 timeout,
9868 allow_write: src.allow_write,
9869 sessions: src.sessions,
9870 artifacts: src.artifacts.clone(),
9871 stem,
9872 handover: None,
9873 });
9874 }
9875
9876 if !renudged.is_empty() {
9877 state.event(
9878 ctx.node,
9879 format!("retry {round}: re-asking {}", renudged.join(", ")),
9880 );
9881 }
9882 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9883 let mut still = Vec::new();
9884 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9885 seats[i] = seat;
9886 if let AgentOutcome::ResourceBlocked(why) = &out {
9889 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9890 continue;
9891 }
9892 let class = FailClass::of(&out);
9893 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9896 && nudges[i] < retries
9897 && !roster.is_empty();
9898 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9899 let next = should_hand_over(prev[i].as_ref(), &cur)
9900 .then(|| next_agent(i, &tried[i], &specs))
9901 .flatten();
9902 if carry {
9903 let h = state
9904 .seat_history
9905 .entry(originals[i].seat.key.clone())
9906 .or_default();
9907 h.failed.insert(specs[i].id.clone());
9908 h.last_fail = Some(cur.clone());
9909 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9910 h.last_ok = None;
9911 }
9912 if let Err(e) = state.save() {
9915 tracing::warn!("could not persist a seat's failure history: {e:#}");
9916 }
9917 }
9918 if let Some(next) = next {
9919 record_handover(
9920 state,
9921 ctx.node,
9922 &originals[i].seat.key,
9923 &specs[i].id,
9924 &next.id,
9925 &cur,
9926 &fail_reason(&out),
9927 );
9928 tried[i].insert(next.id.clone());
9929 prev[i] = Some(cur.clone());
9930 seats[i] =
9931 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9932 specs[i] = next;
9933 fresh[i] = Some(cur.stem_word().to_owned());
9934 nudges[i] = 0;
9935 done[i] = Some(Err(anyhow::anyhow!(
9936 "handed over after: {}",
9937 fail_reason(&out)
9938 )));
9939 still.push(i);
9940 continue;
9941 }
9942 }
9943 if carry
9944 && !nudge_first
9945 && let Some(cur) = class.clone()
9946 {
9947 let h = state
9949 .seat_history
9950 .entry(originals[i].seat.key.clone())
9951 .or_default();
9952 h.failed.insert(specs[i].id.clone());
9953 h.last_fail = Some(cur);
9954 }
9955 let parsed = match out {
9956 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9957 AgentOutcome::Quota(o) => {
9958 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9959 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9960 }
9961 AgentOutcome::Dropped(o) => {
9965 let why = o
9966 .dropped
9967 .as_ref()
9968 .map(|d| d.why.as_str())
9969 .unwrap_or("the CLI ended the stream without delivering its answer");
9970 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9971 }
9972 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9973 Err(anyhow::anyhow!(e))
9974 }
9975 };
9976 let quota = class == Some(FailClass::Quota);
9977 let failed = parsed.is_err();
9978 done[i] = Some(parsed);
9979 if carry && !failed {
9980 let h = state
9981 .seat_history
9982 .entry(originals[i].seat.key.clone())
9983 .or_default();
9984 h.failed.remove(&specs[i].id);
9985 h.last_ok = Some(specs[i].id.clone());
9986 h.last_fail = None;
9987 }
9988 let agent_failure = class.is_some()
9996 && !nudge_first
9997 && !roster.is_empty()
9998 && next_agent(i, &tried[i], &specs).is_some();
9999 if failed && !quota && !agent_failure && nudges[i] < retries {
10000 nudges[i] += 1;
10001 still.push(i);
10002 }
10003 }
10004 pending = still;
10005 }
10006
10007 for (i, q) in last_quota.into_iter().enumerate() {
10010 if let Some(reset) = q {
10011 losses.push(QuotaLoss {
10012 seat: originals[i].seat.key.clone(),
10013 node: ctx.node.to_owned(),
10014 at: Timestamp::now(),
10015 reset,
10016 });
10017 }
10018 }
10019
10020 seats
10021 .into_iter()
10022 .zip(done)
10023 .zip(nudges)
10024 .map(|((seat, res), attempts)| {
10025 (
10026 seat,
10027 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10028 attempts,
10029 )
10030 })
10031 .collect()
10032}
10033
10034fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10036 results
10037 .iter()
10038 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10039}
10040
10041fn block_wave(
10045 jobs: Vec<SeatJob>,
10046 state: &mut RunState,
10047 node: &str,
10048 attempt: usize,
10049 why: &str,
10050) -> Vec<(usize, SeatState, AgentOutcome)> {
10051 state.resource_wait = None;
10052 state
10053 .active
10054 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10055 if let Err(e) = state.save() {
10056 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10057 }
10058 jobs.into_iter()
10059 .enumerate()
10060 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10061 .collect()
10062}
10063
10064async fn acquire_cache_lease(
10077 state: &mut RunState,
10078 cache_dir: &Path,
10079 owner: &crate::cache::Owner,
10080 budget: Duration,
10081 context: &str,
10082 cancel: &(dyn Fn() -> bool + Sync),
10083) -> Result<Option<crate::cache::Guard>> {
10084 let home = crate::run::home();
10085 let started = Instant::now();
10086 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10087 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10088 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10089 Err(e) => {
10090 state.event(
10091 "verify",
10092 format!("{context}: could not check the shared build cache: {e:#}"),
10093 );
10094 if let Err(e2) = state.save() {
10095 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10096 }
10097 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10098 }
10099 };
10100 state.event(
10101 "verify",
10102 format!(
10103 "{context}: waiting for the shared build cache at {} ({})",
10104 cache_dir.display(),
10105 busy.describe()
10106 ),
10107 );
10108 state.resource_wait = Some(crate::run::ResourceWait {
10111 node: context.to_owned(),
10112 kind: "shared build cache".to_owned(),
10113 holder: busy.describe(),
10114 since: Timestamp::now(),
10115 limit_secs: budget.as_secs(),
10116 });
10117 if let Err(e) = state.save() {
10118 tracing::warn!("could not persist a cache wait: {e:#}");
10119 }
10120 let remaining = budget.saturating_sub(started.elapsed());
10121 let waited = crate::cache::wait_for_unless(
10122 &home,
10123 cache_dir,
10124 owner,
10125 remaining,
10126 Duration::from_secs(5),
10127 cancel,
10128 )
10129 .await;
10130 state.resource_wait = None;
10131 match waited {
10132 Ok(g) => Ok(g),
10133 Err(e) => {
10134 state.event("verify", format!("{context}: {e:#}"));
10135 if let Err(e2) = state.save() {
10136 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10137 }
10138 Err(e)
10139 }
10140 }
10141}
10142
10143#[derive(Debug)]
10148struct CacheCheckError(String);
10149
10150impl std::fmt::Display for CacheCheckError {
10151 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10152 f.write_str(&self.0)
10153 }
10154}
10155
10156impl std::error::Error for CacheCheckError {}
10157
10158#[allow(clippy::too_many_arguments)]
10179async fn with_cache_lease<'s, F, Fut>(
10180 state: &'s mut RunState,
10181 cache_dir: Option<&Path>,
10182 node: &str,
10183 seat: &str,
10184 worktree: &Path,
10185 head: &str,
10186 budget: Duration,
10187 context: &str,
10188 wait: &crate::repo_lock::WaitPolicy,
10189 body: F,
10190) -> Result<(Vec<CommandOutcome>, bool)>
10191where
10192 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10193 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10194{
10195 let started = Instant::now();
10198 let home = crate::run::home();
10199 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10200 let guard = if let Some(cache_dir) = cache_dir {
10201 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10202 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10203 wait.cancelled()
10204 })
10205 .await
10206 .and_then(|g| {
10207 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10208 }) {
10209 Ok(g) => Some(g),
10210 Err(e) => {
10211 return Ok((
10212 vec![CommandOutcome {
10213 command: "(waiting for the shared build cache)".into(),
10214 code: None,
10215 output_tail: e.to_string(),
10216 duration_ms: started.elapsed().as_millis() as u64,
10217 resource_blocked: true,
10218 }],
10219 false,
10220 ));
10221 }
10222 }
10223 } else {
10224 None
10225 };
10226 let _coordination = coordinate_operation(state, worktree, wait).await?;
10227 if let Some(cache_dir) = cache_dir {
10228 let identity = crate::cache::Identity::new(worktree, head);
10229 if let Err(e) = crate::cache::ensure_fresh(
10230 &home,
10231 cache_dir,
10232 &identity,
10233 &state.config.verify.cargo_manifests(),
10234 ) {
10235 state.event(
10236 "verify",
10237 format!(
10238 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10239 worktree.display(),
10240 short(head)
10241 ),
10242 );
10243 if let Some(guard) = guard {
10244 guard.release();
10245 }
10246 return Ok((
10247 vec![CommandOutcome {
10248 command: "(confirming the shared build cache is fresh)".into(),
10249 code: None,
10250 output_tail: e.to_string(),
10251 duration_ms: started.elapsed().as_millis() as u64,
10252 resource_blocked: true,
10253 }],
10254 false,
10255 ));
10256 }
10257 }
10258 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10259 if !timed_out_pids.is_empty() {
10260 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10261 }
10262 drop(_coordination);
10265 if let Some(guard) = guard {
10266 guard.release();
10267 }
10268 Ok((outcomes, retried))
10269}
10270
10271async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10283 wait_for_pids_with(
10284 pids,
10285 crate::proc::pid_alive,
10286 LEASE_RELEASE_POLL,
10287 LEASE_RELEASE_MAX_WAIT,
10288 )
10289 .await;
10290}
10291
10292async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10298 pids: &[u32],
10299 alive: F,
10300 poll: Duration,
10301 max_wait: Duration,
10302) {
10303 let deadline = Instant::now() + max_wait;
10304 loop {
10305 if pids.iter().all(|&pid| !alive(pid)) {
10306 return;
10307 }
10308 if Instant::now() >= deadline {
10309 return;
10310 }
10311 tokio::time::sleep(poll).await;
10312 }
10313}
10314
10315fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10322 outcomes.iter().any(|o| o.resource_blocked)
10323}
10324
10325enum GateFix {
10327 Retry,
10329 Stop,
10332 Defer,
10335}
10336
10337fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10345 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10346 red.peek().is_some()
10347 && red.all(|o| {
10348 !o.resource_blocked
10349 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10350 && !o.output_tail.trim().is_empty()
10351 })
10352}
10353
10354fn e2e_outcome_label(o: &CommandOutcome) -> String {
10358 if o.ok() {
10359 return "pass".to_owned();
10360 }
10361 let reason = if o.build_failed() {
10362 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10363 } else {
10364 format!("FAIL ({:?})", o.code)
10365 };
10366 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10367}
10368
10369async fn run_e2e_with_retry(
10377 state: &mut RunState,
10378 shell: &[String],
10379 commands: &[String],
10380 worktree: &Path,
10381 timeout: Duration,
10382 context: &str,
10383) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10384 let (mut e2e, mut timed_out_pids) = run_commands(
10385 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10386 )
10387 .await;
10388 for o in &e2e {
10389 state.event(
10390 "verify",
10391 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10392 );
10393 }
10394 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10398 if verify_retried {
10399 state.event(
10400 "verify",
10401 format!(
10402 "{context}: verify could not build/link, not a test result — retrying once \
10403 before concluding"
10404 ),
10405 );
10406 let retried = run_commands(
10407 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10408 )
10409 .await;
10410 e2e = retried.0;
10411 timed_out_pids.extend(retried.1);
10414 for o in &e2e {
10415 state.event(
10416 "verify",
10417 format!(
10418 "{context}: retry `{}` -> {}",
10419 o.command,
10420 e2e_outcome_label(o)
10421 ),
10422 );
10423 }
10424 }
10425 (e2e, verify_retried, timed_out_pids)
10426}
10427
10428#[allow(clippy::too_many_arguments)]
10444async fn run_commands(
10445 state: &mut RunState,
10446 node: &str,
10447 task: &str,
10448 attempt: usize,
10449 shell: &[String],
10450 commands: &[String],
10451 cwd: &Path,
10452 timeout: Duration,
10453) -> (Vec<CommandOutcome>, Vec<u32>) {
10454 if commands.is_empty() {
10455 return (Vec::new(), Vec::new());
10460 }
10461 let mut out = Vec::new();
10462 let mut timed_out_pids = Vec::new();
10463 let total = commands.len();
10464 for (idx, command) in commands.iter().enumerate() {
10465 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10466 if let Err(e) = state.save() {
10467 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10468 }
10469 let started = Instant::now();
10470 let mut cmd = tokio::process::Command::new(&shell[0]);
10471 cmd.quiet();
10472 cmd.args(&shell[1..])
10473 .arg(command)
10474 .current_dir(cwd)
10475 .stdin(std::process::Stdio::null())
10476 .stdout(std::process::Stdio::piped())
10477 .stderr(std::process::Stdio::piped())
10478 .kill_on_drop(true);
10479 let spawned = cmd.spawn();
10480 let (code, body) = match spawned {
10481 Ok(child) => {
10482 let pid = child.id();
10487 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10488 Ok(Ok(o)) => {
10489 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10490 body.push_str(&String::from_utf8_lossy(&o.stderr));
10491 (o.status.code(), body)
10492 }
10493 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10494 Err(_) => {
10495 if let Some(pid) = pid {
10496 timed_out_pids.push(pid);
10497 }
10498 (None, format!("timed out after {}s", timeout.as_secs()))
10499 }
10500 }
10501 }
10502 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10503 };
10504 out.push(CommandOutcome {
10505 command: command.clone(),
10506 code,
10507 output_tail: tail(&body, OUTPUT_TAIL),
10508 duration_ms: started.elapsed().as_millis() as u64,
10509 resource_blocked: false,
10510 });
10511 }
10512 state.task_finished(task);
10513 if let Err(e) = state.save() {
10514 tracing::warn!("could not persist the end of {task}: {e:#}");
10515 }
10516 (out, timed_out_pids)
10517}
10518
10519fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10531 let repo = repo.display();
10532 match style {
10533 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10534 MergeStyle::Squash => {
10535 let subject = message
10538 .lines()
10539 .next()
10540 .unwrap_or(branch)
10541 .replace(['\\', '"', '$', '`'], "");
10542 format!(
10543 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10544 )
10545 }
10546 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10547 }
10548}
10549
10550const PR_TITLE_MAX: usize = 240;
10561
10562fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10573 let line = queue::first_line(text)?;
10574 let chars: Vec<char> = line.chars().collect();
10575 let (kept, mark) = if chars.len() <= max {
10576 (line.to_owned(), "")
10577 } else if let Some(end) = sentence_end(&chars, max) {
10578 (chars[..end].iter().collect::<String>(), "")
10579 } else {
10580 let room = max.saturating_sub(3);
10581 let cut = if chars[room].is_whitespace() {
10584 room
10585 } else {
10586 chars[..room]
10587 .iter()
10588 .rposition(|c| c.is_whitespace())
10589 .unwrap_or(room)
10590 };
10591 let head: String = chars[..cut].iter().collect();
10592 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10593 (head.to_owned(), "...")
10594 };
10595 if kept.is_empty() {
10596 return None;
10597 }
10598 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10599 return None;
10600 }
10601 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10602 return None;
10603 }
10604 Some(format!("{kept}{mark}"))
10605}
10606
10607fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10611 const MIN: usize = 20;
10612 for i in MIN..max.min(chars.len()) {
10613 if !matches!(chars[i], '.' | '!' | '?') {
10614 continue;
10615 }
10616 let Some(&next) = chars.get(i + 1) else {
10617 continue;
10618 };
10619 if !next.is_whitespace() {
10620 continue;
10621 }
10622 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10623 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10624 continue;
10625 }
10626 let word: String = chars[..i]
10627 .iter()
10628 .rev()
10629 .take_while(|c| !c.is_whitespace())
10630 .collect::<Vec<_>>()
10631 .into_iter()
10632 .rev()
10633 .collect();
10634 let word = word.to_ascii_lowercase();
10635 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10636 continue;
10637 }
10638 let end = chars[..i]
10639 .iter()
10640 .rposition(|c| !c.is_whitespace())
10641 .map_or(i, |p| p + 1);
10642 return Some(end);
10643 }
10644 None
10645}
10646
10647struct PrMessage {
10652 title: String,
10653 body: String,
10654}
10655
10656impl PrMessage {
10657 fn commit_message(&self) -> String {
10661 format!("{}\n\n{}", self.title, self.body)
10662 }
10663}
10664
10665fn title_marker(line: &str) -> Option<&str> {
10667 let line = line.trim();
10668 let head = line.get(..6)?;
10669 head.eq_ignore_ascii_case("title:")
10670 .then(|| line[6..].trim())
10671}
10672
10673fn summary_title(summary: &str) -> Option<String> {
10678 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10679 let raw = title_marker(first)?;
10680 if raw.is_empty() {
10681 return None;
10682 }
10683 let title = queue::title_from(raw, PR_TITLE_MAX);
10684 let lower = title.to_ascii_lowercase();
10685 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10686 return None;
10687 }
10688 Some(title)
10689}
10690
10691const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10693
10694pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10696
10697fn is_review_run(state: &RunState) -> bool {
10701 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10702}
10703
10704fn review_title(state: &RunState) -> Option<String> {
10709 english_subject(
10710 state.reviewed_commits.as_ref()?.first()?,
10711 state.config.graph.github_english(),
10712 )
10713}
10714
10715fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10718 let raw = raw.trim();
10719 if raw.is_empty() {
10720 return None;
10721 }
10722 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10723 return None;
10724 }
10725 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10726 return None;
10727 }
10728 let title = queue::title_from(raw, PR_TITLE_MAX);
10729 let lower = title.to_ascii_lowercase();
10730 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10731 return None;
10732 }
10733 Some(title)
10734}
10735
10736#[derive(Debug, Clone, PartialEq, Eq)]
10739struct BranchFacts {
10740 commits: Vec<(String, String)>,
10742 stat: String,
10744}
10745
10746const STAT_FILE_LINES: usize = 25;
10748const COMMITS_MAX_CHARS: usize = 20_000;
10750
10751async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10754 let commits = git::commit_log(repo, base, branch).await.ok()?;
10755 if commits.is_empty() {
10756 return None;
10757 }
10758 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10759 let lines: Vec<&str> = stat.lines().collect();
10760 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10761 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10762 let more = format!(" ... {omitted} more file(s)");
10763 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10764 kept.push(&more);
10765 kept.push(lines[lines.len() - 1]);
10766 kept.join("\n")
10767 } else {
10768 lines.join("\n")
10769 };
10770 Some(BranchFacts { commits, stat })
10771}
10772
10773fn markdown_safe(text: &str) -> String {
10776 text.replace("</details>", "</details>")
10777 .replace("\x60\x60\x60", "~~~")
10778}
10779
10780fn neutral_title(state: &RunState, winner: char) -> String {
10781 format!(
10782 "chore: land candidate {} of run {}",
10783 winner.to_ascii_uppercase(),
10784 state.id
10785 )
10786}
10787
10788pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10794 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10795 ""
10796 } else {
10797 pr_title
10798 }
10799}
10800
10801pub fn landing_subject_source(state: &RunState) -> String {
10805 if is_review_run(state) {
10806 let winner = state.candidates.first().map_or('A', |c| c.label);
10807 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10808 }
10809 state.instruction.clone()
10810}
10811
10812fn summary_without_title(summary: &str) -> String {
10815 let mut lines = summary.trim().lines().peekable();
10816 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10817 lines.next();
10818 }
10819 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10820}
10821
10822#[cfg(test)]
10834fn pr_message(state: &RunState, winner: char) -> PrMessage {
10835 pr_message_with(state, winner, None)
10836}
10837
10838#[cfg(test)]
10841fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10842 let raw = pr_message_raw(
10843 state,
10844 winner,
10845 facts,
10846 crate::pr_outcome::FinalHead::Unreported,
10847 ChangeNote::NotAsked,
10848 );
10849 let id = crate::scrub::Identity::current();
10850 PrMessage {
10851 title: crate::scrub::scrub(&raw.title, &id),
10852 body: crate::scrub::scrub(&raw.body, &id),
10853 }
10854}
10855
10856#[derive(Debug, Clone, Copy)]
10858enum ChangeNote<'a> {
10859 NotAsked,
10861 Summary(&'a crate::run::ChangeSummary),
10863 Unavailable(&'a str),
10865}
10866
10867fn pr_message_raw(
10868 state: &RunState,
10869 winner: char,
10870 facts: Option<&BranchFacts>,
10871 head: crate::pr_outcome::FinalHead<'_>,
10872 change: ChangeNote<'_>,
10873) -> PrMessage {
10874 if !matches!(change, ChangeNote::NotAsked) {
10875 let (title, mut body) = match change {
10876 ChangeNote::Summary(c) => (
10877 c.title.clone(),
10878 format!("## Summary\n\n{}\n\n", c.body.trim()),
10879 ),
10880 ChangeNote::Unavailable(why) => (
10881 neutral_title(state, winner),
10882 format!(
10883 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10884 ),
10885 ),
10886 ChangeNote::NotAsked => unreachable!(),
10887 };
10888 body.push_str(&public_pr_evidence(state, head));
10889 body.push_str(&format!(
10890 "\n---\nmagi:run/{} magi:candidate-{}\n",
10891 state.id,
10892 winner.to_ascii_lowercase()
10893 ));
10894 return PrMessage { title, body };
10895 }
10896 pr_preview(state, winner, facts, head)
10897}
10898
10899fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10902 let mut body = crate::pr_outcome::review_rounds(state);
10903 body.push_str(&crate::pr_outcome::final_verification(state, head));
10904 let open = state.last_round_findings();
10905 if !open.is_empty() {
10906 body.push_str("## Open review findings\n\n");
10907 for f in open {
10908 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10909 }
10910 body.push('\n');
10911 }
10912 if let Some(c) = state.change_summary.as_ref()
10913 && head == crate::pr_outcome::FinalHead::At(&c.head)
10914 {
10915 body.push_str(&format!(
10916 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10917 c.base, c.head
10918 ));
10919 }
10920 body
10921}
10922
10923fn pr_preview(
10925 state: &RunState,
10926 winner: char,
10927 facts: Option<&BranchFacts>,
10928 head: crate::pr_outcome::FinalHead<'_>,
10929) -> PrMessage {
10930 let summary = state
10931 .candidates
10932 .iter()
10933 .find(|c| c.label == winner)
10934 .map(|c| c.summary.as_str())
10935 .unwrap_or_default();
10936 let review = is_review_run(state);
10939 let title = if review {
10940 facts
10941 .and_then(|f| {
10942 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10943 })
10944 .or_else(|| review_title(state))
10945 .or_else(|| {
10946 state
10947 .candidates
10948 .iter()
10949 .find(|c| c.label == winner)
10950 .filter(|c| !c.branch.starts_with("magi/"))
10951 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10952 })
10953 .unwrap_or_else(|| neutral_title(state, winner))
10954 } else {
10955 summary_title(summary).unwrap_or_else(|| {
10956 english_title(
10957 &state.instruction,
10958 PR_TITLE_MAX,
10959 state.config.graph.github_english(),
10960 )
10961 .unwrap_or_else(|| neutral_title(state, winner))
10962 })
10963 };
10964
10965 let mut body = String::new();
10966 let what = summary_without_title(summary);
10967 if !what.is_empty() {
10968 body.push_str("## Summary\n\n");
10969 body.push_str(&what);
10970 body.push_str("\n\n");
10971 }
10972
10973 body.push_str(&crate::pr_outcome::review_rounds(state));
10975 body.push_str(&crate::pr_outcome::final_verification(state, head));
10976 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10977
10978 let open = state.open_findings();
10979 if !open.is_empty() {
10980 body.push_str("## Open review findings\n\n");
10981 for f in &open {
10982 body.push_str(&format!(
10983 "- `{}` [{:?}] {}\n",
10984 f.id, f.severity, "Context retained in local review records."
10985 ));
10986 }
10987 body.push('\n');
10988 }
10989
10990 if let Some(fix) = fix
10991 && !fix.rejected.is_empty()
10992 {
10993 body.push_str("## Declined by the fixer\n\n");
10994 for r in &fix.rejected {
10995 body.push_str(&format!(
10996 "- `{}`: {}\n",
10997 r.id, "Reason retained in local review records."
10998 ));
10999 }
11000 body.push('\n');
11001 }
11002
11003 if review {
11004 body.push_str("## Commits under review\n\n");
11006 if let Some(facts) = facts {
11007 let mut left = COMMITS_MAX_CHARS;
11008 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11009 let mut entry = format!("- {}\n", markdown_safe(subject));
11010 for l in markdown_safe(text).lines() {
11011 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11012 }
11013 if left == 0 {
11014 body.push_str(&format!(
11015 "- ... {} more commit(s)\n",
11016 facts.commits.len() - i
11017 ));
11018 break;
11019 }
11020 if entry.len() > left {
11021 let mut end = left;
11024 while !entry.is_char_boundary(end) {
11025 end -= 1;
11026 }
11027 entry.truncate(end);
11028 entry.push_str("\n ... (truncated)\n");
11029 left = 0;
11030 } else {
11031 left -= entry.len();
11032 }
11033 body.push_str(&entry);
11034 }
11035 if !facts.stat.trim().is_empty() {
11036 body.push_str(&format!(
11037 "\n## Diff stat\n\n```\n{}\n```\n",
11038 markdown_safe(facts.stat.trim())
11039 ));
11040 }
11041 } else {
11042 match &state.reviewed_commits {
11043 Some(subjects) => {
11044 for s in subjects {
11045 body.push_str(&format!("- {}\n", s.trim()));
11046 }
11047 }
11048 None => {
11049 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11052 body.push_str(rest.trim());
11053 body.push('\n');
11054 }
11055 }
11056 }
11057 } else {
11058 let task = state.instruction.trim();
11059 let task = if task.is_empty() {
11060 "(empty task)"
11061 } else {
11062 task
11063 };
11064 body.push_str(&format!(
11065 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11066 task.replace("</details>", "</details>")
11067 ));
11068 }
11069
11070 body.push_str(&format!(
11071 "\n---\nmagi:run/{} magi:candidate-{}\n",
11072 state.id,
11073 winner.to_ascii_lowercase()
11074 ));
11075
11076 PrMessage { title, body }
11077}
11078
11079fn seeded_instruction(state: &RunState) -> String {
11083 match refs::describe(&state.seeds) {
11084 Some(facts) => format!(
11085 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11086 Candidates start from the unmerged branch named above, when there \
11087 is one, and carry any unmerged commit named by sha as a \
11088 cherry-pick. Check that this is what the task meant before \
11089 building on it.",
11090 state.instruction
11091 ),
11092 None => state.instruction.clone(),
11093 }
11094}
11095
11096async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11100 let base = &state.base_branch;
11101 let mut against = base.clone();
11102 if mode == MergeMode::Pr {
11103 let remote = &state.config.merge.remote;
11107 let tracking = format!("{remote}/{base}");
11108 let fetched = git::fetch(repo, remote, base).await;
11109 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11110 against = tracking;
11111 } else {
11112 return false;
11113 }
11114 }
11115 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11116}
11117
11118fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11121 let mut detail = format!(
11122 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11123 nothing to open a pull request for"
11124 );
11125 match refs::describe(&state.seeds) {
11126 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11127 None => detail.push_str(
11128 "\nThe task names no existing branch or commit; if it means to land work \
11129 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11130 ),
11131 }
11132 detail
11133}
11134
11135#[derive(Debug, PartialEq, Eq)]
11138enum PrPlan {
11139 Create,
11140 Adopt { url: String, title: String },
11141 Stop(String),
11142}
11143
11144fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11147 match found {
11148 Ok(land::OpenPr::None) => PrPlan::Create,
11149 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11150 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11151 "several open pull requests exist for this branch, not picking one: {}",
11152 urls.join(" ")
11153 )),
11154 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11155 }
11156}
11157
11158pub(crate) async fn gh_pr_create(
11160 cwd: &Path,
11161 base: &str,
11162 head: &str,
11163 title: &str,
11164 body: &str,
11165) -> Result<String> {
11166 let out = tokio::process::Command::new("gh")
11167 .args([
11168 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11169 ])
11170 .current_dir(cwd)
11171 .quiet()
11172 .stdin(std::process::Stdio::null())
11173 .output()
11174 .await
11175 .context("spawn gh")?;
11176 if out.status.success() {
11177 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11178 } else {
11179 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11180 }
11181}
11182
11183async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11185 match std::fs::symlink_metadata(path) {
11186 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11187 git::worktree_remove_clean(repo, path).await?;
11190 return Ok(());
11191 }
11192 Err(e) => return Err(e.into()),
11193 Ok(_) => {}
11194 }
11195 if !git::is_clean(path).await? {
11196 bail!(
11197 "worktree has uncommitted work; refusing removal: {}",
11198 path.display()
11199 );
11200 }
11201 if !git::worktree_remove_clean(repo, path).await? {
11202 bail!("worktree removal refused: {}", path.display());
11203 }
11204 Ok(())
11205}
11206
11207pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11216 if crate::pr_recovery::sources_retained(home, &state.id) {
11217 return Ok(Vec::new());
11218 }
11219 if !state.candidate_preservation.is_empty() {
11220 bail!("candidate preservation incomplete; refusing cleanup");
11221 }
11222 let repo = state.repo.clone();
11223 let wait = repository_wait(state, None);
11226 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11227 Ok(lease) => lease,
11228 Err(e) => {
11229 state.save_under(home)?;
11230 return Err(e);
11231 }
11232 };
11233 let root = state.worktree_root();
11234 let winner = state.tally.as_ref().map(|t| t.winner);
11235 let mut removed = Vec::new();
11236
11237 for i in 0..state.candidates.len() {
11238 let c = state.candidates[i].clone();
11239 let is_winner = Some(c.label) == winner;
11240 if is_winner && !drop_winner {
11241 continue;
11242 }
11243 let existed = c.worktree.exists();
11244 remove_preserved_worktree(&repo, &c.worktree).await?;
11247 if existed {
11248 removed.push(c.worktree.to_string_lossy().into_owned());
11249 }
11250 let handed_over = state.released_branches.contains(&c.branch);
11253 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11254 if !git::branch_delete(&repo, &c.branch).await? {
11255 bail!("candidate branch removal refused");
11256 }
11257 removed.push(c.branch.clone());
11258 }
11259 state.candidates[i].folded = true;
11260 }
11261
11262 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11263 let path = name.path();
11264 let keep = !drop_winner
11265 && winner.is_some_and(|w| {
11266 path.file_name()
11267 .is_some_and(|n| n == format!("cand-{w}").as_str())
11268 });
11269 if keep {
11270 continue;
11271 }
11272 remove_preserved_worktree(&repo, &path).await?;
11273 removed.push(path.to_string_lossy().into_owned());
11274 }
11275
11276 remove_if_empty(&root);
11285
11286 if state.enabled_worktree_config && drop_winner {
11287 git::release_worktree_config(&repo).await.ok();
11291 state.enabled_worktree_config = false;
11292 }
11293 state.save_under(home)?;
11294 Ok(removed)
11295}
11296
11297fn remove_if_empty(dir: &Path) {
11308 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11309 std::fs::remove_dir(dir).ok();
11310 }
11311}
11312
11313pub fn worst_open(state: &RunState) -> Option<Severity> {
11315 state
11316 .reviews
11317 .last()?
11318 .reviews
11319 .iter()
11320 .flat_map(|r| r.findings.iter())
11321 .map(|f| f.severity)
11322 .max()
11323}
11324
11325async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11329 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11330 git::acquire_worktree_config(repo).await?;
11331 state.enabled_worktree_config = true;
11332 }
11333 Ok(())
11334}
11335
11336#[cfg(test)]
11337mod tests {
11338 #[tokio::test]
11339 async fn launcher_base_fetch_requires_remote_success() {
11340 let tmp = tempfile::tempdir().unwrap();
11341 let repo = tmp.path().join("repo");
11342 let remote = tmp.path().join("remote");
11343 std::fs::create_dir(&repo).unwrap();
11344 std::fs::create_dir(&remote).unwrap();
11345 git::git(&remote, &["init", "--bare", "-b", "main"])
11346 .await
11347 .unwrap();
11348 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11349 git::git(
11350 &repo,
11351 &[
11352 "-c",
11353 "user.name=Fixture",
11354 "-c",
11355 "user.email=fixture@example.invalid",
11356 "commit",
11357 "--allow-empty",
11358 "-m",
11359 "fixture",
11360 ],
11361 )
11362 .await
11363 .unwrap();
11364 git::git(
11365 &repo,
11366 &["remote", "add", "origin", remote.to_str().unwrap()],
11367 )
11368 .await
11369 .unwrap();
11370 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11371 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11372 assert_eq!(
11373 resolve_base(&repo, "main", "origin", None)
11374 .await
11375 .unwrap()
11376 .commit,
11377 head
11378 );
11379 #[cfg(unix)]
11380 {
11381 let policy: crate::config::Coordination = toml::from_str(
11382 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11383 ).unwrap();
11384 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11385 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11386 let held = crate::repo_lock::SeatLease::operation_with(
11387 &repo,
11388 Some(&policy),
11389 &crate::repo_lock::WaitPolicy::default(),
11390 )
11391 .await
11392 .unwrap();
11393 assert!(held.is_some());
11394 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11395 tokio::pin!(fetch);
11396 assert!(
11397 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11398 .await
11399 .is_err()
11400 );
11401 assert!(
11402 !repo.join(".git/FETCH_HEAD").exists(),
11403 "no fetch before admission"
11404 );
11405 drop(held);
11406 let fetched = fetch.await.unwrap();
11407 assert_eq!(fetched.commit, head);
11408 assert_eq!(fetched.remote, "origin");
11409 assert_eq!(fetched.branch, "main");
11410 }
11411 git::git(
11413 &repo,
11414 &[
11415 "remote",
11416 "set-url",
11417 "origin",
11418 tmp.path().join("missing").to_str().unwrap(),
11419 ],
11420 )
11421 .await
11422 .unwrap();
11423 let error = resolve_base(&repo, "main", "origin", None)
11424 .await
11425 .unwrap_err()
11426 .to_string();
11427 assert!(error.contains("cannot read origin/main"));
11428 assert!(error.contains("refusing to branch off"));
11429 }
11430
11431 #[test]
11432 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11433 let leaked = vec!["chore(deps): update a crate".to_owned()];
11434 let own = "fix(daemon): apply a chosen action";
11435 assert!(should_retitle("", own, &leaked));
11436 assert!(should_retitle(
11437 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11438 own,
11439 &leaked
11440 ));
11441 assert!(should_retitle(
11442 "chore: land candidate A of run 1",
11443 own,
11444 &leaked
11445 ));
11446 assert!(should_retitle(
11447 "magi: candidate A (uncommitted work)",
11448 own,
11449 &leaked
11450 ));
11451 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11452 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11453 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11454 }
11455
11456 #[test]
11457 fn pr_merge_plan_creates_adopts_or_stops() {
11458 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11459 assert_eq!(
11460 pr_merge_plan(Ok(land::OpenPr::One {
11461 url: "u".into(),
11462 title: "t".into()
11463 })),
11464 PrPlan::Adopt {
11465 url: "u".into(),
11466 title: "t".into()
11467 }
11468 );
11469 let PrPlan::Stop(many) =
11470 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11471 else {
11472 panic!("many must stop");
11473 };
11474 assert!(many.contains('a') && many.contains('b'));
11475 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11476 panic!("a failed lookup must stop");
11477 };
11478 assert!(err.contains("bad token"));
11479 }
11480
11481 use super::*;
11482 use crate::run::GateStatus;
11483 use std::collections::BTreeMap;
11484 use std::time::Duration;
11485
11486 fn conductor() -> AgentSpec {
11487 AgentSpec {
11488 id: "conductor".to_owned(),
11489 kind: crate::config::AgentKind::Command,
11490 model: None,
11491 command: vec!["true".to_owned()],
11492 extra_args: Vec::new(),
11493 env: BTreeMap::new(),
11494 prompt_delivery: None,
11495 }
11496 }
11497
11498 fn spec(id: &str) -> AgentSpec {
11499 AgentSpec {
11500 id: id.to_owned(),
11501 kind: crate::config::AgentKind::Command,
11502 model: None,
11503 command: vec!["true".to_owned()],
11504 extra_args: Vec::new(),
11505 env: BTreeMap::new(),
11506 prompt_delivery: None,
11507 }
11508 }
11509
11510 fn ids(xs: &[&str]) -> BTreeSet<String> {
11511 xs.iter().map(|s| (*s).to_owned()).collect()
11512 }
11513
11514 #[test]
11515 fn pick_successor_skips_an_agent_another_seat_holds() {
11516 let roster = [spec("a"), spec("b"), spec("c")];
11518 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11519 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11520 }
11521
11522 #[test]
11523 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11524 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11526 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11527 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11528 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11529 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11530 }
11531
11532 #[test]
11533 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11534 let roster = [spec("a"), spec("b")];
11535 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11536 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11537 let carried = ids(&["b"]);
11538 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11539 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11540 }
11541
11542 #[test]
11543 fn pick_successor_returns_none_without_any_untried_successor() {
11544 let roster = [spec("a"), spec("b")];
11545 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11546 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11547 let carried = ids(&["a"]);
11548 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11549 }
11550
11551 #[test]
11552 fn pick_successor_rescue_avoids_another_seats_occupant() {
11553 let roster = [spec("a"), spec("b"), spec("c")];
11555 let carried = ids(&["b", "c"]);
11556 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11557 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11558 }
11559
11560 #[test]
11561 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11562 let roster = [spec("a"), spec("b"), spec("c")];
11563 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11564 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11565 }
11566
11567 #[test]
11568 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11569 let roster = [spec("a"), spec("b"), spec("c")];
11570 let failed = ids(&["a", "b", "c"]);
11571 let mut tried = ids(&["b"]);
11573 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11574 assert_eq!(first.id, "a");
11575 tried.insert(first.id.clone());
11576 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11577 assert_eq!(second.id, "c");
11578 tried.insert(second.id.clone());
11579 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11580 }
11581
11582 #[test]
11583 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11584 let roster = [spec("a"), spec("b")];
11585 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11586 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11587 }
11588
11589 #[test]
11590 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11591 let roster = [spec("a"), spec("b"), spec("c")];
11592 let h = SeatHistory {
11593 failed: ids(&["a"]),
11594 last_ok: Some("b".to_owned()),
11595 last_fail: None,
11596 };
11597 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11598 let gone = SeatHistory {
11600 last_ok: Some("zzz".to_owned()),
11601 ..h.clone()
11602 };
11603 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11604 let failed = SeatHistory {
11605 failed: ids(&["a", "b"]),
11606 last_ok: Some("b".to_owned()),
11607 last_fail: None,
11608 };
11609 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11610 }
11611
11612 #[test]
11613 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11614 let first = SeatState::new("review-1", "alpha", 7);
11615 let next = handover_seat("review-1", "gamma", 7);
11616 assert_ne!(first.claude_session, next.claude_session);
11617 }
11618
11619 #[test]
11620 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11621 let mut state = state_with_summary("x", "y");
11622 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11623 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11624 assert_ne!(a.claude_session, b.claude_session);
11625 }
11626
11627 #[test]
11628 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11629 let roster = [spec("a"), spec("b")];
11630 let h = SeatHistory {
11631 failed: ids(&["a", "b"]),
11632 ..SeatHistory::default()
11633 };
11634 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11635 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11636 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11637 }
11638
11639 #[test]
11646 fn failure_signature_ignores_numbers_and_paths() {
11647 assert_eq!(
11648 failure_signature("exited with Some(2) and no usable output"),
11649 failure_signature("exited with Some(137) and no usable output")
11650 );
11651 assert_eq!(
11652 failure_signature("process exited with no usable output (exit code 2)"),
11653 failure_signature("process exited with no usable output (exit code 137)")
11654 );
11655 for (old, new) in [
11656 (
11657 "exited with Some(2) and no usable output",
11658 "process exited with no usable output (exit code 2)",
11659 ),
11660 (
11661 "exited with None and no usable output",
11662 "process exited with no usable output (exit code unavailable)",
11663 ),
11664 ] {
11665 let previous = FailClass::Other(failure_signature(old));
11666 let current = FailClass::Other(failure_signature(new));
11667 assert_eq!(previous, current, "resumption retains the failure class");
11668 assert!(!should_hand_over(Some(&previous), ¤t));
11669 }
11670 assert_eq!(
11671 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11672 failure_signature("cannot open /var/x.txt: denied")
11673 );
11674 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11675 }
11676
11677 #[test]
11678 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11679 let other = FailClass::Other("x".into());
11680 assert!(should_hand_over(None, &FailClass::Quota));
11681 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11682 assert!(should_hand_over(
11683 Some(&FailClass::Timeout),
11684 &FailClass::Timeout
11685 ));
11686 assert!(should_hand_over(None, &other));
11687 assert!(!should_hand_over(Some(&other), &other));
11688 assert!(should_hand_over(
11689 Some(&other),
11690 &FailClass::Other("y".into())
11691 ));
11692 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11694 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11695 }
11696
11697 #[test]
11698 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11699 let a = SeatState::new("judge-1", "alpha", 7);
11700 let b = handover_seat("judge-1", "beta", 7);
11701 assert_ne!(a.claude_session, b.claude_session);
11702 assert_eq!(b.turns, 0);
11703 }
11704
11705 #[test]
11706 fn a_timeout_is_classified_apart_from_other_failures() {
11707 assert_eq!(
11708 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11709 Some(FailClass::Timeout)
11710 );
11711 assert!(matches!(
11712 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11713 Some(FailClass::Other(_))
11714 ));
11715 }
11716
11717 #[test]
11718 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11719 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11720 let tried = BTreeSet::from(["beta".to_owned()]);
11721 let next = next_untried_in_roster(&roster, 1, &tried);
11724 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11725 }
11726
11727 #[test]
11728 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11729 let roster = vec![spec("alpha"), spec("beta")];
11730 let tried = BTreeSet::from(["beta".to_owned()]);
11731 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11735 }
11736
11737 #[test]
11738 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11739 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11740 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11741 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11745 }
11746
11747 #[test]
11748 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11749 let roster = vec![spec("a"), spec("a"), spec("b")];
11750 let tried = BTreeSet::from(["a".to_owned()]);
11751 let next = next_untried_in_roster(&roster, 0, &tried);
11752 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11753 }
11754
11755 #[test]
11756 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11757 let roster = vec![spec("a"), spec("b")];
11758 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11759 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11760 }
11761
11762 #[test]
11763 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11764 let dir = tempfile::tempdir().unwrap();
11765 let bay = dir.path().join("ffff");
11766
11767 remove_if_empty(&bay);
11769 assert!(!bay.exists());
11770
11771 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11774 remove_if_empty(&bay);
11775 assert!(bay.exists(), "non-empty directory must survive");
11776
11777 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11779 remove_if_empty(&bay);
11780 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11781 }
11782
11783 #[test]
11792 fn a_full_panel_that_found_nothing_is_clean() {
11793 assert!(round_is_clean(
11794 0,
11795 true,
11796 2,
11797 2,
11798 0,
11799 IncompleteReviewPolicy::Block
11800 ));
11801 }
11802
11803 #[test]
11804 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11805 assert!(!round_is_clean(
11806 0,
11807 true,
11808 1,
11809 2,
11810 0,
11811 IncompleteReviewPolicy::Block
11812 ));
11813 }
11814
11815 #[test]
11816 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11817 assert!(!round_is_clean(
11818 1,
11819 true,
11820 1,
11821 2,
11822 0,
11823 IncompleteReviewPolicy::Warn
11824 ));
11825 }
11826
11827 #[test]
11828 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11829 assert!(round_is_clean(
11830 0,
11831 true,
11832 1,
11833 2,
11834 0,
11835 IncompleteReviewPolicy::Warn
11836 ));
11837 }
11838
11839 #[test]
11840 fn a_full_panel_with_an_open_finding_is_not_clean() {
11841 assert!(!round_is_clean(
11842 1,
11843 true,
11844 2,
11845 2,
11846 0,
11847 IncompleteReviewPolicy::Block
11848 ));
11849 }
11850
11851 #[test]
11852 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11853 assert!(!round_is_clean(
11854 0,
11855 false,
11856 2,
11857 2,
11858 0,
11859 IncompleteReviewPolicy::Block
11860 ));
11861 }
11862
11863 #[test]
11870 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11871 assert!(round_is_clean(
11874 0,
11875 true,
11876 1,
11877 2,
11878 1,
11879 IncompleteReviewPolicy::Block
11880 ));
11881 }
11882
11883 #[test]
11884 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11885 assert!(!round_is_clean(
11888 0,
11889 true,
11890 1,
11891 2,
11892 0,
11893 IncompleteReviewPolicy::Block
11894 ));
11895 }
11896
11897 #[test]
11898 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11899 assert!(!round_is_clean(
11900 1,
11901 true,
11902 1,
11903 2,
11904 1,
11905 IncompleteReviewPolicy::Block
11906 ));
11907 assert!(!round_is_clean(
11908 0,
11909 false,
11910 1,
11911 2,
11912 1,
11913 IncompleteReviewPolicy::Block
11914 ));
11915 }
11916
11917 #[test]
11918 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11919 assert!(!round_is_clean(
11923 0,
11924 true,
11925 0,
11926 2,
11927 2,
11928 IncompleteReviewPolicy::Block
11929 ));
11930 }
11931
11932 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11933 CommandOutcome {
11934 command: "test".to_owned(),
11935 code,
11936 output_tail: String::new(),
11937 duration_ms: 0,
11938 resource_blocked,
11939 }
11940 }
11941
11942 #[test]
11943 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11944 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11945 assert!(
11946 !verify_inconclusive(&[outcome(Some(1), false)]),
11947 "an ordinary failure is still evidence about the patch"
11948 );
11949 assert!(verify_inconclusive(&[outcome(None, true)]));
11950 assert!(
11951 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11952 "one inconclusive outcome taints the whole batch"
11953 );
11954 assert!(!verify_inconclusive(&[]));
11955 }
11956
11957 #[tokio::test]
11958 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11959 let calls = std::sync::atomic::AtomicUsize::new(0);
11963 let started = Instant::now();
11964 wait_for_pids_with(
11965 &[123],
11966 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11967 Duration::from_millis(5),
11968 Duration::from_secs(5),
11969 )
11970 .await;
11971 assert!(
11972 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11973 "must keep checking rather than deciding on the first answer"
11974 );
11975 assert!(
11976 started.elapsed() < Duration::from_secs(1),
11977 "must return the moment it is confirmed dead, not wait out the ceiling"
11978 );
11979 }
11980
11981 #[tokio::test]
11982 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11983 let started = Instant::now();
11984 wait_for_pids_with(
11985 &[123],
11986 |_| true, Duration::from_millis(5),
11988 Duration::from_millis(30),
11989 )
11990 .await;
11991 let elapsed = started.elapsed();
11992 assert!(
11993 elapsed >= Duration::from_millis(30),
11994 "must not give up before its own ceiling: {elapsed:?}"
11995 );
11996 assert!(
11997 elapsed < Duration::from_secs(1),
11998 "must not wait past its own ceiling either: {elapsed:?}"
11999 );
12000 }
12001
12002 #[tokio::test]
12003 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12004 let started = Instant::now();
12005 wait_for_pids_with(
12006 &[],
12007 |_| true,
12008 Duration::from_secs(5),
12009 Duration::from_secs(5),
12010 )
12011 .await;
12012 assert!(
12013 started.elapsed() < Duration::from_millis(200),
12014 "an empty pid list has nothing to confirm"
12015 );
12016 }
12017
12018 fn review_round(
12024 clean: bool,
12025 blocking: usize,
12026 answered: usize,
12027 expected: usize,
12028 progressed: bool,
12029 e2e_ok: bool,
12030 ) -> ReviewRound {
12031 ReviewRound {
12032 round: 1,
12033 head: "h".to_owned(),
12034 verified_head: None,
12035 verified_at: None,
12036 reviews: Vec::new(),
12037 e2e: vec![CommandOutcome {
12038 command: "test".to_owned(),
12039 code: Some(if e2e_ok { 0 } else { 1 }),
12040 output_tail: String::new(),
12041 duration_ms: 0,
12042 resource_blocked: false,
12043 }],
12044 verify_retried: false,
12045 e2e_deferred: false,
12046 e2e_defer_reason: None,
12047 fix: None,
12048 blocking,
12049 answered,
12050 expected,
12051 clean,
12052 progressed,
12053 vote_split: false,
12054 reconsideration: Vec::new(),
12055 verdict: None,
12056 }
12057 }
12058
12059 #[test]
12060 fn review_conclusion_is_none_when_nothing_has_run() {
12061 assert_eq!(review_conclusion(&[], 3), None);
12062 }
12063
12064 #[test]
12065 fn review_conclusion_is_none_while_rounds_remain() {
12066 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12067 assert_eq!(review_conclusion(&rounds, 3), None);
12068 }
12069
12070 #[test]
12071 fn review_conclusion_is_gating_once_a_round_is_clean() {
12072 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12073 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12074 }
12075
12076 #[test]
12077 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12078 let rounds = vec![
12079 review_round(false, 1, 2, 2, true, true),
12080 review_round(false, 1, 2, 2, true, true),
12081 ];
12082 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12083 }
12084
12085 #[test]
12086 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12087 let rounds = vec![
12088 review_round(false, 1, 2, 2, true, true),
12089 review_round(false, 1, 2, 2, true, false),
12090 ];
12091 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12092 }
12093
12094 #[test]
12095 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12096 let mut blocked = review_round(false, 1, 2, 2, true, false);
12103 blocked.e2e[0].resource_blocked = true;
12104 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12105 assert_eq!(review_conclusion(&rounds, 2), None);
12106 }
12107
12108 #[test]
12109 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12110 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12112 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12113 }
12114
12115 #[test]
12116 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12117 let rounds = vec![
12118 review_round(false, 1, 2, 2, false, true),
12119 review_round(false, 1, 2, 2, false, true),
12120 ];
12121 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12122 }
12123
12124 fn secs(n: u64) -> Duration {
12125 Duration::from_secs(n)
12126 }
12127
12128 #[tokio::test]
12131 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12132 ask_test_home();
12133 let tmp = tempfile::tempdir().unwrap();
12134 let repo = tmp.path().join("repo");
12135 std::fs::create_dir_all(&repo).unwrap();
12136 init_repo(&repo);
12137 let g = |args: &[&str]| {
12138 let out = std::process::Command::new("git")
12139 .args(args)
12140 .current_dir(&repo)
12141 .quiet()
12142 .output()
12143 .unwrap();
12144 assert!(
12145 out.status.success(),
12146 "{}",
12147 String::from_utf8_lossy(&out.stderr)
12148 );
12149 };
12150 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12151 g(&["checkout", "-b", "historical"]);
12152 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12153 g(&["add", "."]);
12154 g(&["commit", "-m", "old alternative"]);
12155 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12156 g(&["checkout", "-b", "magi/task-selected", &base]);
12157 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12158 g(&["add", "."]);
12159 g(&["commit", "-m", "measurement infrastructure"]);
12160 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12161 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12162 g(&["add", "."]);
12163 g(&["commit", "-m", "dependent implementation"]);
12164 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12165 g(&["checkout", "main"]);
12166 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12167 g(&["add", "."]);
12168 g(&["commit", "-m", "unrelated base change"]);
12169 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12170 let origin = tmp.path().join("origin.git");
12171 g(&[
12172 "clone",
12173 "--bare",
12174 repo.to_str().unwrap(),
12175 origin.to_str().unwrap(),
12176 ]);
12177 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12178 g(&["fetch", "origin"]);
12179 let selection = crate::pr_recovery::Selection {
12180 url: "https://example.test/project/pull/1".into(),
12181 branch: "magi/task-selected".into(),
12182 head: head.clone(),
12183 source_head: None,
12184 source_run: None,
12185 source_proof: None,
12186 disposition: Some(crate::pr_recovery::Disposition::Review),
12187 instruction: "preserve measured behavior".into(),
12188 approved_at: jiff::Timestamp::now(),
12189 };
12190 let mut template = runner_at(RunStatus::Reviewing);
12191 template.state.repo = repo.clone();
12192 template.state.base_commit = current.clone();
12193 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12194 template.state.origin = Some(crate::run::Origin::queue("selected"));
12195 template.state.pr_head = Some(selection.branch.clone());
12196 template.state.pr_recovery = Some(selection.clone());
12197 let mut runner =
12198 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12199 .await
12200 .unwrap();
12201 let wt = runner.state.winner().unwrap().worktree.clone();
12202 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12203 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12204 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12205 assert!(runner.state.seeds.is_empty());
12206 assert_eq!(runner.state.instruction, "preserve measured behavior");
12207 runner.sync_to_base().await.unwrap();
12208 assert_eq!(
12209 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12210 "prerequisite"
12211 );
12212 assert_eq!(
12213 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12214 "measured implementation"
12215 );
12216 assert_eq!(
12217 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12218 "current main change"
12219 );
12220 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12221 assert_eq!(
12222 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12223 head
12224 );
12225
12226 let mut partial = runner_at(RunStatus::Reviewing);
12228 partial.state.repo = repo.clone();
12229 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12230 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12231 run: Some("exit 1".into()),
12232 ..Default::default()
12233 }];
12234 let selection = runner.state.pr_recovery.clone().unwrap();
12235 partial.state.pr_recovery = Some(selection.clone());
12236 partial.state.pr_head = Some(selection.branch.clone());
12237 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12238 let id = partial.state.id.clone();
12239 let error = match Runner::open_selected(
12240 &repo,
12241 partial.state,
12242 partial.roles,
12243 1,
12244 runner.state.base_commit.clone(),
12245 selection,
12246 )
12247 .await
12248 {
12249 Ok(_) => panic!("setup must fail"),
12250 Err(e) => e,
12251 };
12252 assert_eq!(
12253 error
12254 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12255 .unwrap()
12256 .run,
12257 id
12258 );
12259 let failed = RunState::load(&id).unwrap();
12260 assert_eq!(failed.status, RunStatus::Blocked);
12261 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12262 assert!(Runner::resume(&id).is_err());
12263 runner.state.status = RunStatus::Ready;
12264 runner.state.driver_exited = true;
12265 runner.state.save().unwrap();
12266 let mut retry = runner_at(RunStatus::Reviewing);
12267 retry.state.repo = repo.clone();
12268 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12269 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12270 reselected.approved_at = jiff::Timestamp::now();
12271 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12272 retry.state.pr_head = Some(reselected.branch.clone());
12273 retry.state.pr_recovery = Some(reselected.clone());
12274 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12275 .await
12276 .unwrap();
12277 let restarted = Runner::open_selected(
12278 &repo,
12279 retry.state,
12280 retry.roles,
12281 1,
12282 runner.state.base_commit.clone(),
12283 reselected,
12284 )
12285 .await
12286 .unwrap();
12287 assert_eq!(
12288 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12289 .await
12290 .unwrap(),
12291 head
12292 );
12293 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12294 assert_eq!(
12295 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12296 head
12297 );
12298 }
12299
12300 #[tokio::test]
12301 #[cfg(unix)]
12302 async fn fallback_and_interrupted_restoration_require_coordination() {
12303 use std::os::fd::AsRawFd as _;
12304 ask_test_home();
12305 let tmp = tempfile::tempdir().unwrap();
12306 let mut runner = registered_runner(tmp.path()).await;
12307 runner.state.config.graph.timeout_resource_wait = 1;
12308 let winner = runner.state.candidates[0].clone();
12309 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12310 let leftover = winner.worktree.join("leftover");
12311 std::fs::write(&leftover, "untrusted edits").unwrap();
12312 let next = tmp.path().join("next-fixer");
12313 let mut failed = spec("first");
12314 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12315 let mut successor = spec("second");
12316 successor.command = vec![
12317 "sh".into(),
12318 "-c".into(),
12319 format!("touch '{}'; echo done", next.display()),
12320 ];
12321 let attempts = vec![
12322 (failed.clone(), "first".into()),
12323 (successor.clone(), "second".into()),
12324 ];
12325 let lock_path = runner.state.repo.join(".git/updater.lock");
12326 let external = std::fs::File::create(&lock_path).unwrap();
12327 assert_eq!(
12328 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12329 0
12330 );
12331 let artifacts = tmp.path().join("artifacts");
12332 let build = |spec, seat| SeatJob {
12333 spec,
12334 seat,
12335 cwd: winner.worktree.clone(),
12336 prompt: "fixture".into(),
12337 timeout: Duration::from_secs(10),
12338 allow_write: true,
12339 sessions: false,
12340 artifacts: artifacts.clone(),
12341 stem: "fix".into(),
12342 handover: None,
12343 };
12344 let (_, _, out) = runner
12345 .ask_fixer_among(
12346 attempts.clone(),
12347 "graft",
12348 None,
12349 Some((&winner.worktree, &head)),
12350 build,
12351 )
12352 .await;
12353 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12354 assert!(leftover.exists(), "contention must prevent clean");
12355 assert!(!next.exists());
12356 assert!(runner.state.handovers.is_empty());
12357 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12359 from_run: "earlier".into(),
12360 generation: 1,
12361 mode: crate::recovery::Mode::VerifyRepair,
12362 reasons: Vec::new(),
12363 reviewed_head: None,
12364 rounds: Vec::new(),
12365 open_findings: Vec::new(),
12366 evidence: None,
12367 criteria_inherited: true,
12368 repair: crate::recovery::Repair::Started {
12369 before: head.clone(),
12370 },
12371 });
12372 assert!(
12373 runner
12374 .recovery_repair(&winner)
12375 .await
12376 .unwrap_err()
12377 .is::<crate::repo_lock::ResourceBlocked>()
12378 );
12379 assert!(matches!(
12380 runner.state.recovery.as_ref().unwrap().repair,
12381 crate::recovery::Repair::Started { .. }
12382 ));
12383 assert!(leftover.exists());
12384 drop(external);
12385 runner.recovery_repair(&winner).await.unwrap();
12386 assert!(!leftover.exists());
12387 assert!(matches!(
12388 runner.state.recovery.as_ref().unwrap().repair,
12389 crate::recovery::Repair::Done { .. }
12390 ));
12391 std::fs::remove_file(&lock_path).unwrap();
12393 std::fs::create_dir(&lock_path).unwrap();
12394 std::fs::write(&leftover, "untrusted edits").unwrap();
12395 let (_, _, out) = runner
12396 .ask_fixer_among(
12397 attempts,
12398 "graft",
12399 None,
12400 Some((&winner.worktree, &head)),
12401 build,
12402 )
12403 .await;
12404 assert!(matches!(out, AgentOutcome::Failed(_)));
12405 assert!(leftover.exists());
12406 assert!(!next.exists());
12407 assert_eq!(
12408 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12409 head
12410 );
12411 }
12412
12413 #[cfg(unix)]
12414 async fn registered_runner(root: &Path) -> Runner {
12415 ask_test_home();
12416 let repo = root.join("repo");
12417 std::fs::create_dir_all(&repo).unwrap();
12418 init_repo(&repo);
12419 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12420 let mut runner = runner_at(RunStatus::Prep);
12421 runner.state.repo = repo.clone();
12422 runner.state.config.coordination = crate::config::test_coordination();
12423 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12424 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12425 runner.state.config.graph.advise = false;
12426 runner.roles.implementers = vec![spec("implement")];
12427 runner.roles.judges = vec![spec("judge")];
12428 runner.prep().await.unwrap();
12429 runner
12430 }
12431
12432 #[tokio::test]
12433 #[cfg(unix)]
12434 async fn parked_implementer_persists_completed_work() {
12435 for registered in [false, true] {
12436 let tmp = tempfile::tempdir().unwrap();
12437 let mut runner = registered_runner(tmp.path()).await;
12438 if !registered {
12439 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12440 }
12441 let marker = tmp.path().join("started");
12442 let release = tmp.path().join("release");
12443 let mut implementer = spec("implement");
12444 implementer.command = vec![
12445 "sh".into(),
12446 "-c".into(),
12447 format!(
12448 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12449 marker.display(),
12450 release.display()
12451 ),
12452 ];
12453 runner.roles.implementers = vec![implementer.clone()];
12454 runner.state.config.agents = vec![implementer];
12455 let pause = runner.pause.clone();
12456 let (result, ()) = tokio::join!(runner.implement(), async {
12457 tokio::time::timeout(Duration::from_secs(10), async {
12458 while !marker.exists() {
12459 tokio::time::sleep(Duration::from_millis(10)).await;
12460 }
12461 })
12462 .await
12463 .unwrap();
12464 pause.park();
12465 std::fs::write(&release, "go").unwrap();
12466 });
12467 result.unwrap();
12468 let candidate = &runner.state.candidates[0];
12469 assert!(candidate.commits > 0, "{candidate:?}");
12470 assert!(candidate.summary.contains("saved-result"));
12471 assert_eq!(candidate.files, 1);
12472 let before = candidate.commits;
12473 runner.implement().await.unwrap();
12474 assert_eq!(runner.state.candidates[0].commits, before);
12475 }
12476 }
12477
12478 #[tokio::test]
12479 #[cfg(unix)]
12480 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12481 let tmp = tempfile::tempdir().unwrap();
12482 let mut runner = registered_runner(tmp.path()).await;
12483 let winner = runner.state.candidates[0].clone();
12484 let cache = tmp.path().join("cache");
12485 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12486 runner.state.config.graph.timeout_resource_wait = 0;
12487 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12488 let crate::cache::AcquireOutcome::Acquired(held) =
12489 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12490 else {
12491 panic!("cache must be free")
12492 };
12493 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12494 from_run: "earlier".into(),
12495 generation: 1,
12496 mode: crate::recovery::Mode::VerifyRepair,
12497 reasons: Vec::new(),
12498 reviewed_head: None,
12499 rounds: Vec::new(),
12500 open_findings: Vec::new(),
12501 evidence: None,
12502 criteria_inherited: true,
12503 repair: crate::recovery::Repair::NotStarted,
12504 });
12505 runner.recovery_repair(&winner).await.unwrap();
12506 assert!(matches!(
12507 runner.state.recovery.as_ref().unwrap().repair,
12508 crate::recovery::Repair::WaitingForRestore { .. }
12509 ));
12510 drop(held);
12511 runner.state.resource_block = None;
12512 runner.state.config.graph.timeout_resource_wait = 1;
12513 runner.recovery_repair(&winner).await.unwrap();
12514 assert!(matches!(
12515 runner.state.recovery.as_ref().unwrap().repair,
12516 crate::recovery::Repair::Done { .. }
12517 ));
12518 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12519 }
12520
12521 #[tokio::test]
12522 #[cfg(unix)]
12523 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12524 use std::os::fd::AsRawFd as _;
12525 for graft in [false, true] {
12526 let tmp = tempfile::tempdir().unwrap();
12527 let mut runner = registered_runner(tmp.path()).await;
12528 runner.state.config.graph.timeout_resource_wait = 1;
12529 let winner = runner.state.candidates[0].clone();
12530 let marker = tmp.path().join("fixer-started");
12531 let mut fixer = spec("fixer");
12532 fixer.command = vec![
12533 "sh".into(),
12534 "-c".into(),
12535 format!("echo started >> '{}'; echo done", marker.display()),
12536 ];
12537 runner.roles.fixer = Some(vec![fixer]);
12538 if graft {
12539 let mut loser = winner.clone();
12540 loser.label = 'B';
12541 loser.index = 1;
12542 runner.state.candidates.push(loser);
12543 runner.state.config.graph.graft_runner_up = true;
12544 runner.state.tally = Some(Tally {
12545 first_choice: BTreeMap::new(),
12546 borda: BTreeMap::new(),
12547 winner: winner.label,
12548 rankings: 1,
12549 unanimous_initial: true,
12550 deliberated: false,
12551 changed_votes: 0,
12552 unanimous_final: true,
12553 tie_break: None,
12554 judges: 1,
12555 present: 1,
12556 quorum: 1,
12557 met_quorum: true,
12558 uncontested: None,
12559 });
12560 runner.state.judgements.push(crate::run::Judgement {
12561 judge: 1,
12562 seat: "judge-1".into(),
12563 agent: "judge".into(),
12564 ranking: vec![winner.label, 'B'],
12565 reasons: BTreeMap::new(),
12566 confidence: None,
12567 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12568 order: vec![0, 1],
12569 failed: None,
12570 duration_ms: 0,
12571 });
12572 } else {
12573 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12574 from_run: "earlier".into(),
12575 generation: 1,
12576 mode: crate::recovery::Mode::VerifyRepair,
12577 reasons: Vec::new(),
12578 reviewed_head: None,
12579 rounds: Vec::new(),
12580 open_findings: Vec::new(),
12581 evidence: None,
12582 criteria_inherited: true,
12583 repair: crate::recovery::Repair::NotStarted,
12584 });
12585 }
12586 let reader = crate::repo_lock::SeatLease::for_checkout(
12587 &winner.worktree,
12588 false,
12589 crate::config::test_coordination().as_ref(),
12590 )
12591 .await
12592 .unwrap();
12593 let pause = runner.pause.clone();
12594 let (result, ()) = tokio::join!(
12595 async {
12596 if graft {
12597 runner.graft().await
12598 } else {
12599 runner.recovery_repair(&winner).await
12600 }
12601 },
12602 async {
12603 tokio::time::sleep(Duration::from_millis(100)).await;
12604 pause.park();
12605 }
12606 );
12607 result.unwrap();
12608 assert!(!marker.exists(), "a blocked seat must not spawn");
12609 runner.state = RunState::load(&runner.state.id).unwrap();
12611 if graft {
12612 assert_eq!(
12613 runner.state.graft.as_ref().unwrap().outcome,
12614 GraftOutcome::WaitingForRestore
12615 );
12616 } else {
12617 assert!(matches!(
12618 runner.state.recovery.as_ref().unwrap().repair,
12619 crate::recovery::Repair::WaitingForRestore { .. }
12620 ));
12621 }
12622 drop(reader);
12623 runner.pause = Pause::new();
12624 runner.state.resource_block = None;
12625 let external =
12627 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12628 assert_eq!(
12629 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12630 0
12631 );
12632 let leftover = winner.worktree.join("leftover");
12633 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12634 let result = if graft {
12635 runner.graft().await
12636 } else {
12637 runner.recovery_repair(&winner).await
12638 };
12639 assert!(
12640 result
12641 .unwrap_err()
12642 .is::<crate::repo_lock::ResourceBlocked>()
12643 );
12644 assert!(leftover.exists());
12645 assert!(!marker.exists());
12646 drop(external);
12647 runner.state.resource_block = None;
12648 let result = if graft {
12649 runner.graft().await
12650 } else {
12651 runner.recovery_repair(&winner).await
12652 };
12653 result.unwrap();
12654 assert!(!leftover.exists(), "restore must precede retry");
12655 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12656 if graft {
12657 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12658 } else {
12659 assert!(matches!(
12660 runner.state.recovery.as_ref().unwrap().repair,
12661 crate::recovery::Repair::Done { .. }
12662 ));
12663 }
12664 }
12665 }
12666
12667 #[tokio::test]
12668 #[cfg(unix)]
12669 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12670 use std::os::fd::AsRawFd as _;
12671 let tmp = tempfile::tempdir().unwrap();
12672 let mut runner = registered_runner(tmp.path()).await;
12673 runner.state.config.graph.timeout_resource_wait = 1;
12674 let winner = runner.state.candidates[0].clone();
12675 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12676 runner.state.tally = Some(Tally {
12677 first_choice: BTreeMap::new(),
12678 borda: BTreeMap::new(),
12679 winner: winner.label,
12680 rankings: 0,
12681 unanimous_initial: false,
12682 deliberated: false,
12683 changed_votes: 0,
12684 unanimous_final: false,
12685 tie_break: None,
12686 judges: 0,
12687 present: 0,
12688 quorum: 0,
12689 met_quorum: true,
12690 uncontested: Some("fixture".into()),
12691 });
12692 runner.state.graft = Some(GraftRecord {
12693 pre: pre.clone(),
12694 post: None,
12695 lines: None,
12696 limit: None,
12697 agent: None,
12698 outcome: GraftOutcome::Running,
12699 requested: Vec::new(),
12700 adoption: None,
12701 adoption_error: None,
12702 });
12703 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12704 git::commit_all(&winner.worktree, "unfinished graft")
12705 .await
12706 .unwrap();
12707 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12708 let leftover = winner.worktree.join("leftover");
12709 std::fs::write(&leftover, "untrusted edits").unwrap();
12710 let path = runner.state.repo.join(".git/updater.lock");
12711 let external = std::fs::File::create(&path).unwrap();
12712 assert_eq!(
12713 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12714 0
12715 );
12716 assert!(
12717 runner
12718 .graft()
12719 .await
12720 .unwrap_err()
12721 .is::<crate::repo_lock::ResourceBlocked>()
12722 );
12723 assert!(matches!(
12724 runner.state.graft.as_ref().unwrap().outcome,
12725 GraftOutcome::Running
12726 ));
12727 assert_eq!(
12728 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12729 unfinished
12730 );
12731 assert!(leftover.exists());
12732 drop(external);
12733 std::fs::remove_file(&path).unwrap();
12734 std::fs::create_dir(&path).unwrap();
12735 assert!(runner.graft().await.is_err());
12736 assert!(matches!(
12737 runner.state.graft.as_ref().unwrap().outcome,
12738 GraftOutcome::Running
12739 ));
12740 assert_eq!(
12741 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12742 unfinished
12743 );
12744 assert!(leftover.exists());
12745 std::fs::remove_dir(&path).unwrap();
12746 runner.graft().await.unwrap();
12747 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12748 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12749 assert!(!leftover.exists());
12750 }
12751
12752 #[tokio::test]
12753 #[cfg(unix)]
12754 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12755 use std::os::fd::AsRawFd as _;
12756 let tmp = tempfile::tempdir().unwrap();
12757 let mut runner = registered_runner(tmp.path()).await;
12758 runner.state.prep_complete = Some(false);
12759 runner.state.config.graph.timeout_resource_wait = 1;
12760 runner.state.save().unwrap();
12761 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12762 assert_eq!(
12763 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12764 0
12765 );
12766 runner.execute().await.unwrap();
12767 let disk = RunState::load(&runner.state.id).unwrap();
12768 assert!(disk.driver_exited);
12769 assert_eq!(disk.driver_pid, None);
12770 assert_eq!(runner.state.status, RunStatus::Blocked);
12771 assert_eq!(
12772 runner.state.resource_block.as_ref().unwrap().kind,
12773 "repository coordination"
12774 );
12775 assert!(runner.state.prep_failure.is_none());
12776 assert!(runner.state.seats.is_empty());
12777 runner.pause.park();
12778 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12781 .await
12782 .unwrap()
12783 .unwrap();
12784 assert!(runner.state.parked);
12785 assert!(runner.state.prep_failure.is_none());
12786 let disk = RunState::load(&runner.state.id).unwrap();
12787 assert!(disk.driver_exited);
12788 assert_eq!(disk.driver_pid, None);
12789 }
12790
12791 #[tokio::test]
12792 #[cfg(unix)]
12793 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12794 use std::os::fd::AsRawFd as _;
12795 let tmp = tempfile::tempdir().unwrap();
12796 let mut runner = registered_runner(tmp.path()).await;
12797 runner.state.prep_complete = Some(false);
12798 runner.state.config.graph.timeout_resource_wait = 30;
12799 runner.state.save().unwrap();
12800 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12801 assert_eq!(
12802 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12803 0
12804 );
12805 assert!(
12806 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12807 .await
12808 .is_err()
12809 );
12810 let disk = RunState::load(&runner.state.id).unwrap();
12811 assert!(disk.driver_exited);
12812 assert_eq!(disk.driver_pid, None);
12813 assert!(disk.status.resumable());
12814 assert!(runner.state.driver_exited);
12815 assert_eq!(runner.state.driver_pid, None);
12816 runner.state.save().unwrap();
12817 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12818 }
12819
12820 #[tokio::test]
12821 #[cfg(unix)]
12822 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12823 let tmp = tempfile::tempdir().unwrap();
12824 let mut runner = registered_runner(tmp.path()).await;
12825 let winner = runner.state.candidates[0].clone();
12826 let cache = tmp.path().join("cache");
12827 let started = tmp.path().join("pre-gate-started");
12828 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12829 runner.state.config.verify.gate =
12830 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12831 let home = crate::run::home();
12832 let owner =
12833 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12834 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12835 crate::cache::AcquireOutcome::Acquired(g) => g,
12836 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12837 };
12838 let mut gate = Box::pin(runner.run_gate(&winner));
12839 assert!(
12840 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12841 .await
12842 .is_err()
12843 );
12844 assert!(!started.exists(), "pre-gate must wait for the cache too");
12845 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12849 let editor = crate::repo_lock::SeatLease::checkout_with(
12850 &winner.worktree,
12851 true,
12852 crate::config::test_coordination().as_ref(),
12853 &wait,
12854 )
12855 .await
12856 .unwrap();
12857 drop(editor);
12858 guard.release();
12859 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12860 .await
12861 .unwrap()
12862 .unwrap();
12863 drop(gate);
12864 assert!(outcomes.iter().all(CommandOutcome::ok));
12865 assert!(started.exists());
12866 }
12867
12868 #[tokio::test]
12869 #[cfg(unix)]
12870 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12871 use std::os::fd::AsRawFd as _;
12872
12873 ask_test_home();
12874 let tmp = tempfile::tempdir().unwrap();
12875 let repo = tmp.path().join("repo");
12876 std::fs::create_dir_all(&repo).unwrap();
12877 init_repo(&repo);
12878 let common = repo.join(".git");
12879 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12880 let mut runner = runner_at(RunStatus::Prep);
12881 runner.state.repo = repo.clone();
12882 runner.state.config.coordination = crate::config::test_coordination();
12883 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12884 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12885 runner.state.config.graph.advise = false;
12886 runner.roles.implementers = vec![spec("implement")];
12887 runner.roles.judges = vec![spec("judge")];
12888 runner.prep().await.unwrap();
12889 let winner = runner.state.candidates[0].clone();
12890 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12893 let completed = tmp.path().join("seat-completed");
12894 runner.roles.implementers[0].command = vec![
12895 "sh".into(),
12896 "-c".into(),
12897 format!("touch '{}'; echo completed", completed.display()),
12898 ];
12899 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12900 assert_eq!(
12901 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12902 0
12903 );
12904 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12905 let mut implement = Box::pin(runner.implement());
12906 tokio::select! {
12907 result = &mut implement => panic!("rescue must wait: {result:?}"),
12908 result = tokio::time::timeout(Duration::from_secs(10), async {
12909 while !completed.exists() {
12910 tokio::time::sleep(Duration::from_millis(20)).await;
12911 }
12912 }) => result.unwrap(),
12913 }
12914 assert!(
12915 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12916 .await
12917 .is_err()
12918 );
12919 assert_eq!(
12920 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12921 before
12922 );
12923 assert_eq!(
12924 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12925 0
12926 );
12927 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12928 .await
12929 .unwrap()
12930 .unwrap();
12931 drop(implement);
12932 assert_ne!(
12933 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12934 before
12935 );
12936 assert!(git::is_clean(&winner.worktree).await.unwrap());
12937
12938 let started = tmp.path().join("pre-gate-started");
12939 let proceed = tmp.path().join("pre-gate-proceed");
12940 let checked = tmp.path().join("gate-ran");
12941 runner.state.config.verify.pre_gate = vec![format!(
12942 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12943 started.display(),
12944 proceed.display()
12945 )];
12946 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12947 assert_eq!(
12948 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12949 0
12950 );
12951 let mut gate = Box::pin(runner.run_gate(&winner));
12952 assert!(
12953 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12954 .await
12955 .is_err()
12956 );
12957 assert!(!started.exists());
12958 assert!(!checked.exists());
12959 assert!(git::is_clean(&winner.worktree).await.unwrap());
12960 assert_eq!(
12961 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12962 0
12963 );
12964 tokio::select! {
12965 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12966 result = tokio::time::timeout(Duration::from_secs(10), async {
12967 while !started.exists() {
12968 tokio::time::sleep(Duration::from_millis(20)).await;
12969 }
12970 }) => result.unwrap(),
12971 }
12972 assert_eq!(
12973 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12974 -1,
12975 "mechanical edits must hold the real lock"
12976 );
12977 std::fs::write(proceed, "ready").unwrap();
12978 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12979 .await
12980 .unwrap()
12981 .unwrap();
12982 drop(gate);
12983 assert!(outcomes.iter().all(CommandOutcome::ok));
12984 assert!(checked.exists());
12985 assert!(runner.state.pre_gate_commit.is_some());
12986 assert!(git::is_clean(&winner.worktree).await.unwrap());
12987 assert_eq!(
12988 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12989 0,
12990 "gate completion must release coordination"
12991 );
12992 drop(outsider);
12993 std::fs::remove_file(common.join("updater.lock")).unwrap();
12994 std::fs::create_dir(common.join("updater.lock")).unwrap();
12995 std::fs::remove_file(&started).unwrap();
12996 std::fs::remove_file(&checked).unwrap();
12997 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12998 let error = runner.run_gate(&winner).await.unwrap_err();
12999 assert!(
13000 error
13001 .to_string()
13002 .contains("acquire mandatory repository lock")
13003 );
13004 assert!(
13005 !started.exists(),
13006 "failed acquisition must prevent pre-gate"
13007 );
13008 assert!(!checked.exists(), "failed acquisition must prevent gate");
13009 assert!(git::is_clean(&winner.worktree).await.unwrap());
13010 assert_eq!(
13011 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13012 head
13013 );
13014 }
13015
13016 #[tokio::test]
13017 #[cfg(unix)]
13018 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13019 use std::os::fd::AsRawFd as _;
13020 use std::time::Duration;
13021
13022 ask_test_home();
13023 let tmp = tempfile::tempdir().unwrap();
13024 let repo = tmp.path().join("repo");
13025 std::fs::create_dir_all(&repo).unwrap();
13026 init_repo(&repo);
13027 let common = repo.join(".git");
13028 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13029 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13030 assert_eq!(
13031 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13032 0
13033 );
13034 let mut runner = runner_at(RunStatus::Prep);
13035 runner.state.repo = repo.clone();
13036 runner.state.config.coordination = crate::config::test_coordination();
13037 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13038 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13039 runner.state.config.graph.advise = true;
13040 runner.state.config.graph.advisors = 1;
13041 runner.roles.implementers = vec![spec("implement")];
13042 runner.roles.judges = vec![spec("judge")];
13043 let root = runner.state.worktree_root();
13044 let started = tmp.path().join("setup-started");
13045 let proceed = tmp.path().join("setup-proceed");
13046 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13047 run: Some(format!(
13048 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13049 started.display(),
13050 proceed.display()
13051 )),
13052 ..Default::default()
13053 }];
13054 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13055 let config = git::git(&repo, &["config", "--local", "--list"])
13056 .await
13057 .unwrap();
13058 let mut prep = Box::pin(runner.prep());
13059 assert!(
13060 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13061 .await
13062 .is_err()
13063 );
13064 assert!(
13065 !root.exists(),
13066 "contention must prevent even checkout creation"
13067 );
13068 assert!(!started.exists(), "contention must prevent setup");
13069 assert_eq!(
13070 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13071 branches
13072 );
13073 assert_eq!(
13074 git::git(&repo, &["config", "--local", "--list"])
13075 .await
13076 .unwrap(),
13077 config
13078 );
13079 drop(lock);
13080 tokio::select! {
13081 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13082 result = tokio::time::timeout(Duration::from_secs(10), async {
13083 while !started.exists() {
13084 tokio::time::sleep(Duration::from_millis(20)).await;
13085 }
13086 }) => result.unwrap(),
13087 }
13088 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13089 assert_eq!(
13090 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13091 -1
13092 );
13093 assert!(
13094 tokio::time::timeout(
13095 Duration::from_millis(100),
13096 crate::repo_lock::SeatLease::for_checkout(
13097 &repo,
13098 true,
13099 crate::config::test_coordination().as_ref()
13100 )
13101 )
13102 .await
13103 .is_err(),
13104 "preparation must also exclude local editors"
13105 );
13106 std::fs::write(proceed, "ready").unwrap();
13107 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13108 .await
13109 .unwrap()
13110 .unwrap();
13111 drop(prep);
13112 assert_eq!(runner.state.prep_complete, Some(true));
13113 assert!(root.join("judge-1").exists());
13114 assert!(root.join("advisor-1").exists());
13115 assert_eq!(
13116 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13117 0
13118 );
13119 }
13120
13121 #[tokio::test]
13122 #[cfg(unix)]
13123 async fn preparation_lock_failure_prevents_repository_changes() {
13124 ask_test_home();
13125 let tmp = tempfile::tempdir().unwrap();
13126 let repo = tmp.path().join("repo");
13127 std::fs::create_dir_all(&repo).unwrap();
13128 init_repo(&repo);
13129 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13130 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13133 let mut runner = runner_at(RunStatus::Prep);
13134 runner.state.repo = repo.clone();
13135 runner.state.config.coordination = crate::config::test_coordination();
13136 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13137 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13138 runner.roles.implementers = vec![spec("implement")];
13139 let error = runner.prep().await.unwrap_err();
13140 assert!(
13141 error
13142 .to_string()
13143 .contains("acquire mandatory repository lock")
13144 );
13145 assert!(!runner.state.worktree_root().exists());
13146 assert!(runner.state.candidates.is_empty());
13147 assert!(!repo.join(".git/config.worktree").exists());
13148 assert_eq!(
13149 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13150 "* main"
13151 );
13152 }
13153
13154 #[tokio::test]
13155 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13156 ask_test_home();
13157 let tmp = tempfile::tempdir().unwrap();
13158 let repo = tmp.path().join("repo");
13159 std::fs::create_dir_all(&repo).unwrap();
13160 init_repo(&repo);
13161 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13162 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13163 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13164 git::git(&repo, &["add", "."]).await.unwrap();
13165 git::git(&repo, &["commit", "-m", "required seed"])
13166 .await
13167 .unwrap();
13168 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13169 git::git(&repo, &["checkout", "main"]).await.unwrap();
13170 let mut runner = runner_at(RunStatus::Prep);
13171 runner.state.repo = repo.clone();
13172 runner.state.base_commit = base.clone();
13173 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13174 runner.state.config.graph.advise = false;
13175 runner.state.config.blind.commit_msg_hook = false;
13176 runner.state.seeds = vec![refs::Seed {
13177 token: seed.clone(),
13178 sha: seed.clone(),
13179 kind: refs::SeedKind::Unmerged,
13180 branch: false,
13181 detail: String::new(),
13182 }];
13183 runner.roles.implementers = vec![spec("one"), spec("two")];
13184 runner.roles.judges = vec![spec("judge")];
13185 let label = blind::assign_labels(2, runner.state.seed)[0];
13186 let branch = runner.state.branch_for(label);
13187 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13188 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13189 .await
13190 .unwrap();
13191 runner.state.candidates.push(Candidate {
13192 index: 0,
13193 label,
13194 agent: "one".into(),
13195 branch,
13196 worktree,
13197 summary: String::new(),
13198 stat: String::new(),
13199 files: 0,
13200 commits: 0,
13201 empty: false,
13202 failed: None,
13203 verified_noop: None,
13204 duration_ms: 0,
13205 folded: false,
13206 });
13207 runner.state.save().unwrap();
13208 runner.state = RunState::load(&runner.state.id).unwrap();
13210 assert_eq!(runner.state.prep_complete, Some(false));
13211 runner.prep().await.unwrap();
13212 assert_eq!(runner.state.prep_complete, Some(true));
13213 assert_eq!(runner.state.candidates.len(), 2);
13214 for candidate in &runner.state.candidates {
13215 assert_eq!(
13216 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13217 git::tree_of(&repo, &seed).await.unwrap()
13218 );
13219 assert_eq!(
13220 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13221 "required commit"
13222 );
13223 }
13224 assert!(runner.state.worktree_root().join("judge-1").exists());
13225 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13226 runner.prep().await.unwrap();
13227 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13228
13229 runner.state.prep_complete = Some(false);
13230 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13231 run: Some(format!(
13232 "test -f '{}'",
13233 tmp.path().join("dependency-ready").display()
13234 )),
13235 ..Default::default()
13236 }];
13237 assert!(runner.prep().await.is_err());
13238 assert!(runner.state.prep_failure.is_some());
13239 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13240 runner.state = RunState::load(&runner.state.id).unwrap();
13241 assert!(
13242 runner.prep().await.is_err(),
13243 "correcting the cause alone must not auto-retry"
13244 );
13245 let failed_at = runner.state.events.last().unwrap().at;
13246 assert!(
13247 runner.retry_preparation(failed_at).is_err(),
13248 "old approvals cannot unlock the hold"
13249 );
13250 let mut task = crate::queue::Task::new(
13251 "retry".into(),
13252 "retry setup".into(),
13253 repo.clone(),
13254 crate::queue::Source::Human,
13255 );
13256 task.start(runner.state.id.clone());
13257 task.hold_machine(Some("setup failed".into()));
13258 task.release_operator("cli", None);
13259 runner
13260 .retry_preparation(task.resumption.as_ref().unwrap().at)
13261 .unwrap();
13262 runner.prep().await.unwrap();
13263 assert!(runner.state.prep_failure.is_none());
13264 assert_eq!(runner.state.prep_complete, Some(true));
13265 assert_eq!(runner.state.candidates.len(), 2);
13266 assert!(runner.state.worktree_root().join("judge-1").exists());
13267 for candidate in &runner.state.candidates {
13268 assert_eq!(
13269 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13270 git::tree_of(&repo, &seed).await.unwrap()
13271 );
13272 }
13273 }
13274
13275 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13276 let mut heads = Vec::new();
13277 for candidate in candidates {
13278 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13279 }
13280 heads
13281 }
13282
13283 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13284 ask_test_home();
13285 let tmp = tempfile::tempdir().unwrap();
13286 let repo = tmp.path().join("repo");
13287 std::fs::create_dir(&repo).unwrap();
13288 init_repo(&repo);
13289 let mut runner = runner_at(RunStatus::Reviewing);
13290 runner.state.repo = repo.clone();
13291 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13292 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13293 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13294 let branch = format!("candidate-{label}");
13295 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13296 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13297 .await
13298 .unwrap();
13299 if label == 'B' {
13300 std::fs::write(worktree.join("change"), "winner").unwrap();
13301 git::rescue_commit(&worktree, "candidate change")
13302 .await
13303 .unwrap();
13304 }
13305 runner.state.candidates.push(Candidate {
13306 index,
13307 label,
13308 agent: "alpha".into(),
13309 branch,
13310 worktree,
13311 summary: "completed reply".into(),
13312 stat: String::new(),
13313 files: usize::from(label == 'B'),
13314 commits: usize::from(label == 'B'),
13315 empty: label == 'A',
13316 failed: None,
13317 verified_noop: None,
13318 duration_ms: 1,
13319 folded: false,
13320 });
13321 }
13322 runner.state.judge_skipped = true;
13323 runner.state.tally = Some(Tally {
13324 first_choice: BTreeMap::new(),
13325 borda: BTreeMap::new(),
13326 winner: 'B',
13327 rankings: 0,
13328 unanimous_initial: false,
13329 deliberated: false,
13330 changed_votes: 0,
13331 unanimous_final: false,
13332 tie_break: None,
13333 judges: 0,
13334 present: 0,
13335 quorum: 0,
13336 met_quorum: true,
13337 uncontested: Some("only B produced a change".into()),
13338 });
13339 runner.state.save().unwrap();
13340 (tmp, runner)
13341 }
13342
13343 #[tokio::test]
13344 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13345 let (_tmp, mut runner) = preservation_resume_fixture().await;
13346 runner.implement().await.unwrap();
13348 assert!(runner.state.candidate_preservation.is_empty());
13349 assert!(
13350 !runner
13351 .state
13352 .dir()
13353 .join("artifacts/candidate-A-recovery")
13354 .exists()
13355 );
13356 runner.fold_losers().await.unwrap();
13357 assert!(!runner.state.candidates[0].worktree.exists());
13358 for schema in [14, crate::run::SCHEMA] {
13359 runner.state.schema = schema;
13360 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13361 runner.state.save().unwrap();
13362 runner.state = RunState::load(&runner.state.id).unwrap();
13363 runner.implement().await.unwrap();
13364 assert!(runner.state.candidate_preservation.is_empty());
13365 assert!(runner.state.candidates[0].folded);
13366 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13367 }
13368 }
13369
13370 #[tokio::test]
13371 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13372 let (_tmp, mut runner) = preservation_resume_fixture().await;
13373 let wt = runner.state.candidates[0].worktree.clone();
13374 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13375 runner.state.schema = 14;
13376 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13377 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13378 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13379 for _ in 0..2 {
13380 assert!(runner.implement().await.is_err());
13381 assert_eq!(runner.state.status, RunStatus::Blocked);
13382 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13383 assert!(
13384 runner.state.candidates[0].empty,
13385 "old verdict must remain unusable"
13386 );
13387 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13388 assert!(runner.state.judge_skipped);
13389 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13390 assert!(
13391 crate::run::read_artifact(&runner.state, "cand-A.patch")
13392 .unwrap()
13393 .contains("owned edits")
13394 );
13395 assert!(wt.exists());
13396 assert_eq!(
13397 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13398 .await
13399 .unwrap(),
13400 1
13401 );
13402 runner.state = RunState::load(&runner.state.id).unwrap();
13403 }
13404 }
13405
13406 #[tokio::test]
13407 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13408 let (_tmp, mut runner) = preservation_resume_fixture().await;
13409 let wt = runner.state.candidates[0].worktree.clone();
13410 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13411 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13412 let index_lock = PathBuf::from(
13413 git::git(
13414 &wt,
13415 &[
13416 "rev-parse",
13417 "--path-format=absolute",
13418 "--git-path",
13419 "index.lock",
13420 ],
13421 )
13422 .await
13423 .unwrap(),
13424 );
13425 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13426 runner.state.schema = 14;
13427 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13428 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13429 assert!(runner.implement().await.is_err());
13430 assert!(runner.after_implement().is_err());
13431 assert!(runner.fold_losers().await.is_err());
13432 assert_eq!(
13433 std::fs::read_to_string(wt.join("recovered")).unwrap(),
13434 "failed seat edits"
13435 );
13436 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13437 runner.state = RunState::load(&runner.state.id).unwrap();
13438 std::fs::remove_file(index_lock).unwrap();
13439 assert!(runner.implement().await.is_err());
13441 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13442 assert!(runner.state.candidates[0].empty);
13443 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13444 assert!(
13445 crate::run::read_artifact(&runner.state, "cand-A.patch")
13446 .unwrap()
13447 .contains("failed seat edits")
13448 );
13449 assert!(wt.exists());
13450 }
13451
13452 #[tokio::test]
13453 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13454 let (_tmp, mut runner) = preservation_resume_fixture().await;
13455 let repo = runner.state.repo.clone();
13456 let common = git::git(
13457 &repo,
13458 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13459 )
13460 .await
13461 .unwrap();
13462 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13463 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13464 assert!(runner.fold_losers().await.is_err());
13465 assert!(!runner.state.candidates[0].worktree.exists());
13466 assert!(!runner.state.candidates[0].folded);
13467 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13468 runner.state = RunState::load(&runner.state.id).unwrap();
13469 std::fs::remove_file(branch_lock).unwrap();
13470 let judge = runner.state.worktree_root().join("judge-1");
13471 git::worktree_add_detached(&repo, &judge, "main")
13472 .await
13473 .unwrap();
13474 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13475 runner.roles.judges = vec![spec("judge")];
13476 assert!(runner.fold_losers().await.is_err());
13477 runner.state = RunState::load(&runner.state.id).unwrap();
13478 assert!(
13479 runner.state.candidates[0].folded,
13480 "candidate progress saved before judge failure"
13481 );
13482 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13483 assert!(judge.join("uncommitted").exists());
13484 std::fs::remove_file(judge.join("uncommitted")).unwrap();
13485 runner.fold_losers().await.unwrap();
13486 assert!(!judge.exists());
13487 runner.fold_losers().await.unwrap();
13488 }
13489
13490 #[tokio::test]
13491 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13492 let (_tmp, mut runner) = preservation_resume_fixture().await;
13493 runner.state.candidates.retain(|c| c.label == 'B');
13494 runner.state.tally = None;
13495 runner.state.judge_skipped = false;
13496 runner.state.prep_complete = Some(true);
13497 runner.state.config.graph.advise = false;
13498 runner.state.config.graph.selected_implementer = Some("alpha".into());
13499 let wt = runner.state.candidates[0].worktree.clone();
13500 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13501 runner
13502 .state
13503 .candidate_preservation
13504 .insert('B', "awaiting persistence".into());
13505 let index_lock = git::git(
13506 &wt,
13507 &[
13508 "rev-parse",
13509 "--path-format=absolute",
13510 "--git-path",
13511 "index.lock",
13512 ],
13513 )
13514 .await
13515 .unwrap();
13516 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13517 runner.state.save().unwrap();
13518 assert!(runner.execute_graph().await.is_err());
13519 assert_eq!(runner.state.status, RunStatus::Blocked);
13520 assert!(runner.state.candidate_preservation.contains_key(&'B'));
13521 std::fs::remove_file(index_lock).unwrap();
13522 for _ in 0..2 {
13525 runner.state = RunState::load(&runner.state.id).unwrap();
13526 runner.pause.park();
13527 runner.execute_graph().await.unwrap();
13528 assert_eq!(runner.state.status, RunStatus::Implementing);
13529 assert!(runner.state.candidate_preservation.is_empty());
13530 assert_eq!(runner.state.candidates[0].commits, 2);
13531 assert!(runner.state.tally.is_none());
13532 }
13533 runner.implement().await.unwrap();
13534 assert_eq!(runner.state.status, RunStatus::Judging);
13535 runner.state.status = RunStatus::Blocked;
13537 runner.state.candidates[0].failed = Some("implementation failed".into());
13538 runner.state.save().unwrap();
13539 let error = runner.execute_graph().await.unwrap_err();
13540 assert!(
13541 error
13542 .to_string()
13543 .contains("selected implementation is held")
13544 );
13545 }
13546
13547 #[tokio::test]
13548 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13549 let (_tmp, mut runner) = preservation_resume_fixture().await;
13550 let repo = runner.state.repo.clone();
13551 let candidate = runner.state.candidates[0].clone();
13552 let registered = git::worktree_holding(&repo, &candidate.branch)
13555 .await
13556 .unwrap()
13557 .unwrap();
13558 assert_eq!(
13559 registered.canonicalize().unwrap(),
13560 candidate.worktree.canonicalize().unwrap()
13561 );
13562 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13563 assert_eq!(
13564 git::worktree_holding(&repo, &candidate.branch)
13565 .await
13566 .unwrap(),
13567 Some(registered)
13568 );
13569 assert!(
13570 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13571 "the stale registration initially keeps the branch checked out"
13572 );
13573 runner.fold_losers().await.unwrap();
13574 assert!(runner.state.candidates[0].folded);
13575 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13576 assert_eq!(
13577 git::worktree_holding(&repo, &candidate.branch)
13578 .await
13579 .unwrap(),
13580 None
13581 );
13582 runner.state = RunState::load(&runner.state.id).unwrap();
13583 runner.fold_losers().await.unwrap();
13584 }
13585
13586 #[tokio::test]
13587 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13588 let (_tmp, mut runner) = preservation_resume_fixture().await;
13589 let repo = runner.state.repo.clone();
13590 let candidate = runner.state.candidates[0].clone();
13591 let winner = runner.state.candidates[1].clone();
13592 let registered = git::worktree_holding(&repo, &candidate.branch)
13593 .await
13594 .unwrap()
13595 .unwrap();
13596 assert_eq!(
13597 registered.canonicalize().unwrap(),
13598 candidate.worktree.canonicalize().unwrap()
13599 );
13600 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13601 assert_eq!(
13602 git::worktree_holding(&repo, &candidate.branch)
13603 .await
13604 .unwrap(),
13605 Some(registered)
13606 );
13607 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13608
13609 for _ in 0..2 {
13610 fold_run(&mut runner.state, false, &crate::run::home())
13611 .await
13612 .unwrap();
13613 runner.state = RunState::load(&runner.state.id).unwrap();
13614 assert!(runner.state.candidates[0].folded);
13615 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13616 assert_eq!(
13617 git::worktree_holding(&repo, &candidate.branch)
13618 .await
13619 .unwrap(),
13620 None
13621 );
13622 assert!(!runner.state.candidates[1].folded);
13623 assert!(winner.worktree.exists());
13624 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13625 }
13626 }
13627
13628 #[tokio::test]
13629 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13630 ask_test_home();
13631 let tmp = tempfile::tempdir().unwrap();
13632 let repo = tmp.path().join("repo");
13633 std::fs::create_dir(&repo).unwrap();
13634 init_repo(&repo);
13635 let mut runner = runner_at(RunStatus::Implementing);
13636 runner.state.repo = repo.clone();
13637 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13638 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13639 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13640 let branch = format!("candidate-{label}");
13641 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13642 git::worktree_add_branch(&repo, &wt, &branch, "main")
13643 .await
13644 .unwrap();
13645 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13646 git::git(&wt, &["add", "README.md"]).await.unwrap();
13647 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13648 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13649 runner.state.candidates.push(Candidate {
13650 index,
13651 label,
13652 agent: "alpha".to_owned(),
13653 branch,
13654 worktree: wt,
13655 summary: "completed implementation".to_owned(),
13656 stat: String::new(),
13657 files: 0,
13658 commits: 0,
13659 empty: false,
13660 failed: None,
13661 verified_noop: None,
13662 duration_ms: 42,
13663 folded: false,
13664 });
13665 runner
13666 .state
13667 .candidate_preservation
13668 .insert(label, "awaiting persistence".to_owned());
13669 }
13670 let wt = runner.state.candidates[0].worktree.clone();
13671 let index_lock = git::git(
13672 &wt,
13673 &[
13674 "rev-parse",
13675 "--path-format=absolute",
13676 "--git-path",
13677 "index.lock",
13678 ],
13679 )
13680 .await
13681 .unwrap();
13682 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13683 runner.state.save().unwrap();
13684 assert!(runner.persist_candidates().await.is_err());
13685 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13686 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13687 assert_eq!(
13688 runner.state.candidates[1].commits, 1,
13689 "other results are still collected"
13690 );
13691 assert!(
13692 !runner.state.candidates[0].empty,
13693 "failed rescue is not no change"
13694 );
13695 assert_eq!(
13696 runner.state.candidates[0].summary,
13697 "completed implementation"
13698 );
13699 assert!(runner.after_implement().is_err());
13700 let home = tmp.path().join("fold-home");
13701 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13702 assert!(wt.join("new-file").exists());
13703 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13704 assert!(recovery.join("staged.patch").exists());
13705 assert!(recovery.join("unstaged.patch").exists());
13706 assert_eq!(
13707 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13708 "untracked edits\n"
13709 );
13710 runner.state.begin_pass();
13711 runner.state.end_pass_parked(ParkCause::Upgrade {
13712 from: "1.0".into(),
13713 to: Some("1.1".into()),
13714 });
13715 runner.state.save().unwrap();
13716 runner.state = RunState::load(&runner.state.id).unwrap();
13717 assert!(matches!(
13718 runner.state.passes.last().unwrap().end,
13719 Some(crate::run::PassEnd::Parked {
13720 cause: ParkCause::Upgrade { .. }
13721 })
13722 ));
13723 std::fs::remove_file(index_lock).unwrap();
13724 runner.implement().await.unwrap();
13726 assert!(runner.state.candidate_preservation.is_empty());
13727 assert!(
13728 runner
13729 .state
13730 .candidates
13731 .iter()
13732 .all(|c| c.commits == 1 && !c.empty)
13733 );
13734 assert!(git::is_clean(&wt).await.unwrap());
13735 assert!(
13736 crate::run::read_artifact(&runner.state, "cand-A.patch")
13737 .unwrap()
13738 .contains("later tracked edits")
13739 );
13740 }
13741
13742 #[tokio::test]
13743 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13744 ask_test_home();
13745 let tmp = tempfile::tempdir().unwrap();
13746 let repo = tmp.path().join("repo");
13747 std::fs::create_dir(&repo).unwrap();
13748 init_repo(&repo);
13749 let wt = tmp.path().join("candidate");
13750 git::worktree_add_branch(&repo, &wt, "candidate", "main")
13751 .await
13752 .unwrap();
13753 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13754 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13755 assert!(wt.join("uncommitted").exists());
13756 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13757 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13758 let mut runner = runner_at(RunStatus::Implementing);
13759 runner.state.repo = repo.clone();
13760 runner.state.base_commit = "missing-base".to_owned();
13761 runner.state.candidates.push(Candidate {
13762 index: 0,
13763 label: 'A',
13764 agent: "alpha".into(),
13765 branch: "candidate".into(),
13766 worktree: wt.clone(),
13767 summary: "completed".into(),
13768 stat: String::new(),
13769 files: 0,
13770 commits: 0,
13771 empty: false,
13772 failed: None,
13773 verified_noop: None,
13774 duration_ms: 1,
13775 folded: false,
13776 });
13777 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13778 assert!(
13779 fold_run(&mut runner.state, true, &tmp.path().join("home"))
13780 .await
13781 .is_err()
13782 );
13783 assert!(!runner.state.candidates[0].folded);
13784 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13785 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13786 runner
13787 .state
13788 .candidate_preservation
13789 .insert('A', "pending".into());
13790 assert!(runner.persist_candidates().await.is_err());
13791 assert!(!runner.state.candidates[0].empty);
13792 assert!(!runner.state.candidates[0].folded);
13793 assert!(wt.exists());
13794 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13795 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13796 std::fs::create_dir_all(&patch_path).unwrap();
13797 assert!(
13798 runner.persist_candidates().await.is_err(),
13799 "artifact failure stays pending"
13800 );
13801 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13802 assert!(!runner.state.candidates[0].empty);
13803 std::fs::remove_dir(patch_path).unwrap();
13804 runner.persist_candidates().await.unwrap();
13805 assert!(
13806 runner.state.candidates[0].empty,
13807 "only successful inspection confirms no change"
13808 );
13809 }
13810
13811 fn init_repo(dir: &Path) {
13812 let run = |args: &[&str]| {
13813 let out = std::process::Command::new("git")
13814 .args(args)
13815 .current_dir(dir)
13816 .quiet()
13817 .output()
13818 .expect("spawn git");
13819 assert!(
13820 out.status.success(),
13821 "git {args:?} failed: {}",
13822 String::from_utf8_lossy(&out.stderr)
13823 );
13824 };
13825 run(&["init", "-b", "main"]);
13826 run(&["config", "user.name", "magi test"]);
13827 run(&["config", "user.email", "magi@example.com"]);
13828 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13829 run(&["add", "-A"]);
13830 run(&["commit", "-m", "init"]);
13831 }
13832
13833 fn ask_test_home() {
13841 crate::run::pin_test_home();
13842 }
13843
13844 fn runner_at(status: RunStatus) -> Runner {
13847 let mut state = RunState::new(
13848 PathBuf::from("/nonexistent/repo"),
13849 "main".to_owned(),
13850 "deadbeef".to_owned(),
13851 "task".to_owned(),
13852 Config::default(),
13853 );
13854 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13857 state.id = format!(
13858 "20000101-000000-{:04x}",
13859 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13860 );
13861 state.status = status;
13862 Runner {
13863 state,
13864 roles: ResolvedRoles {
13865 implementers: Vec::new(),
13866 judges: Vec::new(),
13867 reviewers: Vec::new(),
13868 fixer: None,
13869 conductor: conductor(),
13870 implementer_roster: Vec::new(),
13871 judge_roster: Vec::new(),
13872 reviewer_roster: Vec::new(),
13873 },
13874 sem: Arc::new(Semaphore::new(1)),
13875 pause: Pause::new(),
13876 interrupt: Pause::new(),
13877 }
13878 }
13879
13880 #[test]
13881 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13882 crate::run::pin_test_home();
13883 let mut runner = runner_at(RunStatus::Gating);
13884 runner.state.save().unwrap();
13885 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13888 assert!(runner.claim_driver().is_err());
13889 assert!(runner.state.driver_exited);
13890 assert_eq!(runner.state.driver_pid, None);
13891 let disk = RunState::load(&runner.state.id).unwrap();
13892 assert!(disk.driver_exited);
13893 assert_eq!(disk.driver_pid, None);
13894 assert_eq!(disk.status, RunStatus::Gating);
13895 }
13896
13897 #[tokio::test]
13898 async fn an_early_graph_error_releases_its_driver() {
13899 crate::run::pin_test_home();
13900 let mut runner = runner_at(RunStatus::Blocked);
13901 runner.state.config.graph.selected_implementer = Some("selected".into());
13902 runner.state.save().unwrap();
13903 assert!(runner.execute().await.is_err());
13904 let disk = RunState::load(&runner.state.id).unwrap();
13905 assert!(disk.driver_exited);
13906 assert_eq!(disk.driver_pid, None);
13907 assert_eq!(disk.status, RunStatus::Blocked);
13908 }
13909
13910 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13911 crate::run::MergeRepair {
13912 question: "repair-question".into(),
13913 head: "old-head".into(),
13914 pr: "https://github.com/o/r/pull/42".into(),
13915 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13916 gate: vec![outcome(Some(0), false)],
13917 gate_head: Some("old-head".into()),
13918 gate_fixes: Vec::new(),
13919 pre_gate: Vec::new(),
13920 phase,
13921 failure: None,
13922 }
13923 }
13924
13925 #[tokio::test]
13926 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13927 crate::run::pin_test_home();
13928 let mut runner = runner_at(RunStatus::Landing);
13929 runner.state.merge_repairs.push(merge_repair_request(
13930 crate::run::MergeRepairPhase::Requested,
13931 ));
13932 runner.state.save().unwrap();
13933 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13934 assert!(
13935 runner
13936 .execute()
13937 .await
13938 .unwrap_err()
13939 .to_string()
13940 .contains("already running")
13941 );
13942 assert_eq!(
13943 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13944 crate::run::MergeRepairPhase::Requested
13945 );
13946 drop(claim);
13947 let mut disk = runner.state.clone();
13948 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13949 disk.save().unwrap();
13950 assert!(
13951 runner
13952 .execute()
13953 .await
13954 .unwrap_err()
13955 .to_string()
13956 .contains("decision changed")
13957 );
13958 assert_eq!(
13959 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13960 crate::run::MergeRepairPhase::Complete
13961 );
13962 }
13963
13964 #[test]
13965 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13966 crate::run::pin_test_home();
13967 let mut runner = runner_at(RunStatus::Landing);
13968 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13969 runner.state.reviews = request.reviews.clone();
13970 runner.state.gate = request.gate.clone();
13971 runner.state.gate_ran = true;
13972 runner.state.land_approval = Some(crate::run::LandApproval {
13973 question: "old".into(),
13974 head: "old-head".into(),
13975 });
13976 runner.state.merge_repairs.push(request);
13977 runner.begin_merge_repair_review(0).unwrap();
13978 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13979 assert!(!runner.state.gate_status().ok());
13980 assert!(runner.state.land_approval.is_none());
13981 let disk = RunState::load(&runner.state.id).unwrap();
13982 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13983 assert!(disk.merge_repairs[0].gate[0].ok());
13984 assert_eq!(
13985 disk.merge_repairs[0].phase,
13986 crate::run::MergeRepairPhase::Reviewing
13987 );
13988 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13989 assert!(disk.reviews.is_empty());
13990 }
13991
13992 #[tokio::test]
13993 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13994 crate::run::pin_test_home();
13995 for phase in [
13996 crate::run::MergeRepairPhase::Fixing,
13997 crate::run::MergeRepairPhase::Requested,
13998 ] {
13999 let mut runner = runner_at(RunStatus::Landing);
14000 runner.state.config.graph.land_rounds = 0;
14001 runner.state.merge_repairs.push(merge_repair_request(phase));
14002 runner.state.save().unwrap();
14003 assert!(!runner.repair_merge_review().await.unwrap());
14004 assert_eq!(runner.state.status, RunStatus::Blocked);
14005 assert_eq!(
14006 runner.state.merge_repairs[0].phase,
14007 crate::run::MergeRepairPhase::Failed
14008 );
14009 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14010 assert!(
14011 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14012 "interrupted"
14013 } else {
14014 "budget exhausted"
14015 })
14016 );
14017 assert!(runner.state.merge.is_none());
14018 }
14019 }
14020
14021 #[test]
14025 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14026 crate::run::pin_test_home();
14027 let mut runner = runner_at(RunStatus::Implementing);
14028 let interrupt = Pause::new();
14029 runner.watch_interrupt(interrupt.clone());
14030
14031 interrupt.park_because("task a1b2 asked to run first");
14032
14033 assert!(runner.park_here().expect("park_here"));
14034 assert!(runner.state.parked);
14035 let last = runner.state.events.last().expect("a park event");
14036 assert_eq!(last.node, "park");
14037 assert!(
14038 last.message.contains("task a1b2 asked to run first"),
14039 "expected the interrupt reason in {:?}",
14040 last.message
14041 );
14042 }
14043
14044 #[test]
14047 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14048 crate::run::pin_test_home();
14049 let mut runner = runner_at(RunStatus::Implementing);
14050 runner.state.begin_pass();
14051 let shutdown = Pause::new();
14052 runner.on_pause(shutdown.clone());
14053 let upgrade = ParkCause::Upgrade {
14054 from: "0.121.0".to_owned(),
14055 to: Some("0.121.1".to_owned()),
14056 };
14057 shutdown.park_for(upgrade.clone());
14058 shutdown.park_for(ParkCause::Operator);
14059
14060 assert!(runner.park_here().expect("park_here"));
14061 assert_eq!(
14062 runner.state.passes[0].end,
14063 Some(crate::run::PassEnd::Parked { cause: upgrade })
14064 );
14065 let last = runner.state.events.last().expect("a park event");
14066 assert!(last.message.contains("0.121.0"), "{}", last.message);
14067 }
14068
14069 #[test]
14077 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14078 crate::run::pin_test_home();
14079 let mut runner = runner_at(RunStatus::Implementing);
14080 let shutdown = Pause::new();
14081 runner.on_pause(shutdown.clone());
14082 let interrupt = Pause::new();
14083 runner.watch_interrupt(interrupt.clone());
14084
14085 assert!(!runner.park_here().expect("park_here"));
14087 assert!(!runner.state.parked);
14088
14089 interrupt.park_because("test");
14091 assert!(!shutdown.parked());
14092 assert!(runner.park_here().expect("park_here"));
14093 }
14094
14095 #[tokio::test]
14109 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14110 crate::run::pin_test_home();
14111 let mut runner = runner_at(RunStatus::Implementing);
14112 let interrupt = Pause::new();
14113 runner.watch_interrupt(interrupt.clone());
14114
14115 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14116 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14117
14118 let node = async move {
14122 started_tx.send(()).expect("send started");
14123 finish_rx.await.expect("recv finish");
14124 "node finished"
14125 };
14126
14127 let interrupter = async move {
14128 started_rx.await.expect("recv started");
14129 interrupt.park_because("higher-priority task waiting");
14131 tokio::task::yield_now().await;
14135 finish_tx.send(()).expect("send finish");
14136 };
14137
14138 let (node_result, ()) = tokio::join!(node, interrupter);
14139 assert_eq!(
14140 node_result, "node finished",
14141 "the in-flight call ran to completion"
14142 );
14143
14144 assert!(runner.park_here().expect("park_here"));
14147 assert!(runner.state.parked);
14148 }
14149
14150 #[test]
14156 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14157 crate::run::pin_test_home();
14158 let mut runner = runner_at(RunStatus::Judging);
14159 runner.state.config.agents = vec![conductor()];
14163 runner.state.candidates = vec![Candidate {
14164 index: 0,
14165 label: 'A',
14166 agent: "alpha".to_owned(),
14167 branch: "magi/x/A".to_owned(),
14168 worktree: PathBuf::from("/nonexistent/worktree"),
14169 summary: "did the thing".to_owned(),
14170 stat: "1 file changed".to_owned(),
14171 files: 1,
14172 commits: 1,
14173 empty: false,
14174 failed: None,
14175 verified_noop: None,
14176 duration_ms: 1234,
14177 folded: false,
14178 }];
14179 let run_id = runner.state.id.clone();
14180
14181 let interrupt = Pause::new();
14182 runner.watch_interrupt(interrupt.clone());
14183 interrupt.park_because("task c3d4 asked to run first");
14184 assert!(runner.park_here().expect("park_here"));
14185
14186 let resumed = Runner::resume(&run_id).expect("resume");
14187 assert_eq!(resumed.state.candidates.len(), 1);
14188 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14189 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14190 assert_eq!(resumed.state.status, runner.state.status);
14191 assert!(
14192 resumed.state.parked,
14193 "still parked until `execute` actually walks the graph again"
14194 );
14195 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14196 }
14197
14198 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14200 let mut q = ask::Question::new(
14201 run.to_owned(),
14202 "implement".to_owned(),
14203 "impl-A".to_owned(),
14204 "Which storage backend should the cache use?".to_owned(),
14205 String::new(),
14206 vec!["SQLite".to_owned(), "Redis".to_owned()],
14207 );
14208 store.put(&mut q).unwrap();
14209 q
14210 }
14211
14212 #[test]
14213 fn a_failed_runs_open_question_is_abandoned() {
14214 ask_test_home();
14215 let store = ask::Questions::open();
14216 let mut runner = runner_at(RunStatus::Failed);
14217 let run = runner.state.id.clone();
14218 let q = ask_open_question(&store, &run);
14219
14220 runner.settle_questions();
14221
14222 let back = store.get(&q.id).unwrap();
14223 assert!(
14224 !back.status.open(),
14225 "the seat that asked died with the run; nobody is left to read an answer"
14226 );
14227 assert!(
14228 back.detail.contains(&run) && back.detail.contains("failed"),
14229 "the reason names what the run became, not just that it is gone: {}",
14230 back.detail
14231 );
14232 }
14233
14234 #[test]
14235 fn a_merged_runs_open_question_is_abandoned_too() {
14236 ask_test_home();
14237 let store = ask::Questions::open();
14238 for status in [RunStatus::Merged, RunStatus::Ready] {
14241 let mut runner = runner_at(status);
14242 let run = runner.state.id.clone();
14243 let q = ask_open_question(&store, &run);
14244
14245 runner.settle_questions();
14246
14247 let back = store.get(&q.id).unwrap();
14248 assert!(
14249 !back.status.open(),
14250 "{status:?} run's question must not outlive the run"
14251 );
14252 }
14253 }
14254
14255 #[test]
14256 fn a_still_resumable_runs_open_question_is_left_alone() {
14257 ask_test_home();
14258 let store = ask::Questions::open();
14259 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14265 let mut runner = runner_at(status);
14266 let run = runner.state.id.clone();
14267 let q = ask_open_question(&store, &run);
14268
14269 runner.settle_questions();
14270
14271 let back = store.get(&q.id).unwrap();
14272 assert!(
14273 back.status.open(),
14274 "{status:?} is still alive; the question must still be waiting"
14275 );
14276 }
14277 }
14278
14279 #[test]
14280 fn settle_questions_never_touches_an_already_answered_question() {
14281 ask_test_home();
14282 let store = ask::Questions::open();
14283 let mut runner = runner_at(RunStatus::Failed);
14284 let run = runner.state.id.clone();
14285 let mut q = ask_open_question(&store, &run);
14286 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14287 .unwrap();
14288 store.put(&mut q).unwrap();
14289
14290 runner.settle_questions();
14295 runner.settle_questions();
14296
14297 let back = store.get(&q.id).unwrap();
14298 assert_eq!(
14299 back.status,
14300 ask::QuestionStatus::Answered,
14301 "a real answer is a decision on record, never overwritten by a sweep"
14302 );
14303 }
14304
14305 #[tokio::test]
14316 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14317 crate::run::pin_test_home();
14318 let tmp = tempfile::tempdir().expect("tempdir");
14319 let repo = tmp.path().join("repo");
14320 std::fs::create_dir_all(&repo).unwrap();
14321 init_repo(&repo);
14322
14323 let mut config = Config::default();
14324 config.graph.worktree_root = Some(tmp.path().join("wt"));
14325
14326 let mut state = RunState::new(
14327 repo.clone(),
14328 "main".to_owned(),
14329 "deadbeef".to_owned(),
14330 "task".to_owned(),
14331 config,
14332 );
14333 let root = state.worktree_root();
14334 let wt_a = root.join("cand-A");
14335 let wt_b = root.join("cand-B");
14336 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14337 .await
14338 .expect("worktree A");
14339 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14340 .await
14341 .expect("worktree B");
14342
14343 state.candidates = vec![
14344 Candidate {
14345 index: 0,
14346 label: 'A',
14347 agent: "alpha".to_owned(),
14348 branch: "magi/x/A".to_owned(),
14349 worktree: wt_a.clone(),
14350 summary: String::new(),
14351 stat: String::new(),
14352 files: 0,
14353 commits: 0,
14354 empty: false,
14355 failed: None,
14356 verified_noop: None,
14357 duration_ms: 0,
14358 folded: false,
14359 },
14360 Candidate {
14361 index: 1,
14362 label: 'B',
14363 agent: "beta".to_owned(),
14364 branch: "magi/x/B".to_owned(),
14365 worktree: wt_b.clone(),
14366 summary: String::new(),
14367 stat: String::new(),
14368 files: 0,
14369 commits: 0,
14370 empty: false,
14371 failed: None,
14372 verified_noop: None,
14373 duration_ms: 0,
14374 folded: false,
14375 },
14376 ];
14377 state.tally = Some(Tally {
14378 first_choice: BTreeMap::from([('A', 1)]),
14379 borda: BTreeMap::new(),
14380 winner: 'A',
14381 rankings: 1,
14382 unanimous_initial: true,
14383 deliberated: false,
14384 changed_votes: 0,
14385 unanimous_final: true,
14386 tie_break: None,
14387 judges: 1,
14388 present: 1,
14389 quorum: 1,
14390 met_quorum: true,
14391 uncontested: None,
14392 });
14393 state.status = RunStatus::Ready;
14394
14395 fold_run(&mut state, false, &crate::run::home())
14396 .await
14397 .expect("fold_run");
14398
14399 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14400 assert!(
14401 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14402 "the unmerged winner's branch survives"
14403 );
14404 assert!(
14405 !state.candidates[0].folded,
14406 "the winner is not marked folded"
14407 );
14408
14409 assert!(!wt_b.exists(), "the loser's worktree is removed");
14410 assert!(
14411 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14412 "the loser's branch is removed"
14413 );
14414 assert!(state.candidates[1].folded, "the loser is marked folded");
14415 }
14416
14417 #[tokio::test]
14418 #[cfg(unix)]
14419 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14420 use std::os::fd::AsRawFd as _;
14421 use std::time::Duration;
14422
14423 let tmp = tempfile::tempdir().unwrap();
14424 let repo = tmp.path().join("repo");
14425 std::fs::create_dir_all(&repo).unwrap();
14426 init_repo(&repo);
14427 let common = repo.join(".git");
14428 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14429 let home = tmp.path().join("home");
14430 let mut config = Config::default();
14431 config.coordination = crate::config::test_coordination();
14432 config.graph.worktree_root = Some(tmp.path().join("wt"));
14433 config.graph.timeout_resource_wait = 1;
14434 let mut state = RunState::new(
14435 repo.clone(),
14436 "main".into(),
14437 "deadbeef".into(),
14438 "task".into(),
14439 config,
14440 );
14441 let root = state.worktree_root();
14442 let candidate = root.join("cand-A");
14443 let reviewer = root.join("review-1");
14444 let branch = state.branch_for('A');
14445 git::worktree_add_branch(&repo, &candidate, &branch, "main")
14446 .await
14447 .unwrap();
14448 git::worktree_add_detached(&repo, &reviewer, "main")
14449 .await
14450 .unwrap();
14451 git::acquire_worktree_config(&repo).await.unwrap();
14452 state.enabled_worktree_config = true;
14453 state.status = RunStatus::Ready;
14454 state.candidates.push(Candidate {
14455 index: 0,
14456 label: 'A',
14457 agent: "implement".into(),
14458 branch: branch.clone(),
14459 worktree: candidate.clone(),
14460 summary: String::new(),
14461 stat: String::new(),
14462 files: 0,
14463 commits: 0,
14464 empty: false,
14465 failed: None,
14466 verified_noop: None,
14467 duration_ms: 0,
14468 folded: false,
14469 });
14470 state.save_under(&home).unwrap();
14471 let config_before = std::fs::read(common.join("config")).unwrap();
14472 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14473 .await
14474 .unwrap();
14475 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14478 assert_eq!(
14479 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14480 0
14481 );
14482 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14483 assert!(
14484 tokio::time::timeout(Duration::from_millis(200), &mut fold)
14485 .await
14486 .is_err()
14487 );
14488 assert!(candidate.exists());
14489 assert!(reviewer.exists());
14490 assert!(git::branch_exists(&repo, &branch).await.unwrap());
14491 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14492 assert_eq!(
14493 git::git(&repo, &["worktree", "list", "--porcelain"])
14494 .await
14495 .unwrap(),
14496 worktrees_before
14497 );
14498 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14499 .await
14500 .unwrap()
14501 .unwrap_err();
14502 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14503 drop(fold);
14504 assert!(candidate.exists() && reviewer.exists());
14505 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14506 assert_eq!(
14507 state.resource_block.as_ref().unwrap().kind,
14508 "repository coordination"
14509 );
14510 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14511 drop(external);
14512 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14513 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14514 .await
14515 .unwrap()
14516 .unwrap();
14517 drop(fold);
14518 assert!(removed.contains(&branch));
14519 assert!(!root.exists());
14520 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14521 assert!(state.candidates[0].folded);
14522 assert!(!state.enabled_worktree_config);
14523 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14524 let enabled = git::git_raw(
14525 &repo,
14526 &["config", "--local", "--get", "extensions.worktreeConfig"],
14527 )
14528 .await
14529 .unwrap();
14530 assert!(!enabled.ok(), "the last config reference was released");
14531 }
14532
14533 #[tokio::test]
14536 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14537 let tmp = tempfile::tempdir().expect("tempdir");
14538 let repo = tmp.path().join("repo");
14539 std::fs::create_dir_all(&repo).unwrap();
14540 init_repo(&repo);
14541 let home = tmp.path().join("home");
14542
14543 let mut config = Config::default();
14544 config.graph.worktree_root = Some(tmp.path().join("wt"));
14545 let mut state = RunState::new(
14546 repo.clone(),
14547 "main".to_owned(),
14548 "deadbeef".to_owned(),
14549 "task".to_owned(),
14550 config,
14551 );
14552 git::git(&repo, &["branch", "magi/x/A", "main"])
14554 .await
14555 .expect("branch");
14556 state.candidates = vec![Candidate {
14557 index: 0,
14558 label: 'A',
14559 agent: "alpha".to_owned(),
14560 branch: "magi/x/A".to_owned(),
14561 worktree: state.worktree_root().join("cand-A"),
14562 summary: String::new(),
14563 stat: String::new(),
14564 files: 0,
14565 commits: 0,
14566 empty: false,
14567 failed: None,
14568 verified_noop: None,
14569 duration_ms: 0,
14570 folded: true,
14571 }];
14572 state.released_to = Some("20260901-000000-new1".to_owned());
14573 state.released_branches = vec!["magi/x/A".to_owned()];
14574
14575 fold_run(&mut state, true, &home).await.expect("fold_run");
14576
14577 assert!(
14578 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14579 "the handed-over branch survives a fold"
14580 );
14581 }
14582
14583 #[tokio::test]
14587 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14588 let tmp = tempfile::tempdir().expect("tempdir");
14589 let repo = tmp.path().join("repo");
14590 std::fs::create_dir_all(&repo).unwrap();
14591 init_repo(&repo);
14592 let run = |args: &[&str]| {
14593 let out = std::process::Command::new("git")
14594 .quiet()
14595 .args(args)
14596 .current_dir(&repo)
14597 .output()
14598 .expect("spawn git");
14599 assert!(out.status.success(), "git {args:?}");
14600 };
14601 run(&["branch", "magi/x/A"]);
14602 run(&["checkout", "-q", "-b", "magi/x/B"]);
14603 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14604 run(&["add", "-A"]);
14605 run(&["commit", "-q", "-m", "work"]);
14606 run(&["checkout", "-q", "main"]);
14607
14608 let probe = RunState::new(
14612 repo.clone(),
14613 "main".to_owned(),
14614 "deadbeef".to_owned(),
14615 "task".to_owned(),
14616 Config::default(),
14617 );
14618 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14619 let bare = tmp.path().join("origin.git");
14620 let out = std::process::Command::new("git")
14621 .quiet()
14622 .args(["init", "-q", "--bare"])
14623 .arg(&bare)
14624 .output()
14625 .expect("spawn git");
14626 assert!(out.status.success(), "git init --bare");
14627 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14628 run(&["push", "-q", "origin", "main"]);
14629
14630 let mut state = RunState::new(
14631 repo.clone(),
14632 "main".to_owned(),
14633 "deadbeef".to_owned(),
14634 "task".to_owned(),
14635 Config::default(),
14636 );
14637 state.seeds = vec![refs::Seed {
14638 token: "magi/27b2/A".to_owned(),
14639 kind: refs::SeedKind::Unresolved,
14640 sha: String::new(),
14641 branch: true,
14642 detail: "no branch or commit named magi/27b2/A".to_owned(),
14643 }];
14644
14645 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14646 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14647 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14648 let detail = empty_candidate_detail(&state, "main");
14649 assert!(detail.starts_with("empty candidate"), "{detail}");
14650 assert!(detail.contains("magi/27b2/A"), "{detail}");
14651 }
14652
14653 #[tokio::test]
14662 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14663 let tmp = tempfile::tempdir().expect("tempdir");
14664 let repo = tmp.path().join("repo");
14665 std::fs::create_dir_all(&repo).unwrap();
14666 init_repo(&repo);
14667
14668 let mut config = Config::default();
14669 config.merge.mode = MergeMode::Local;
14670
14671 let mut state = RunState::new(
14672 repo.clone(),
14673 "main".to_owned(),
14674 "deadbeef".to_owned(),
14675 "task".to_owned(),
14676 config,
14677 );
14678 state.candidates = vec![Candidate {
14679 index: 0,
14680 label: 'A',
14681 agent: "alpha".to_owned(),
14682 branch: "does-not-exist".to_owned(),
14683 worktree: repo.clone(),
14684 summary: String::new(),
14685 stat: String::new(),
14686 files: 0,
14687 commits: 0,
14688 empty: false,
14689 failed: None,
14690 verified_noop: None,
14691 duration_ms: 0,
14692 folded: false,
14693 }];
14694 state.tally = Some(Tally {
14695 first_choice: BTreeMap::from([('A', 1)]),
14696 borda: BTreeMap::new(),
14697 winner: 'A',
14698 rankings: 1,
14699 unanimous_initial: true,
14700 deliberated: false,
14701 changed_votes: 0,
14702 unanimous_final: true,
14703 tie_break: None,
14704 judges: 0,
14705 present: 0,
14706 quorum: 0,
14707 met_quorum: true,
14708 uncontested: Some("only candidate A produced a change".to_owned()),
14709 });
14710 state.reviews = vec![ReviewRound {
14711 round: 1,
14712 head: "deadbeef".to_owned(),
14713 verified_head: None,
14714 verified_at: None,
14715 reviews: Vec::new(),
14716 e2e: Vec::new(),
14717 fix: None,
14718 blocking: 0,
14719 answered: 0,
14720 expected: 0,
14721 clean: true,
14722 verify_retried: false,
14723 e2e_deferred: false,
14724 e2e_defer_reason: None,
14725 progressed: false,
14726 vote_split: false,
14727 reconsideration: Vec::new(),
14728 verdict: None,
14729 }];
14730 state.gate = vec![CommandOutcome {
14731 command: "test".to_owned(),
14732 code: Some(0),
14733 output_tail: String::new(),
14734 duration_ms: 0,
14735 resource_blocked: false,
14736 }];
14737 state.gate_ran = true;
14738 state.status = RunStatus::Ready;
14743 state.merge = Some(MergeOutcome {
14744 mode: MergeMode::Local,
14745 ok: false,
14746 detail: "already concluded".to_owned(),
14747 empty: false,
14748 });
14749
14750 let mut runner = Runner {
14751 state,
14752 roles: ResolvedRoles {
14753 implementers: Vec::new(),
14754 judges: Vec::new(),
14755 reviewers: Vec::new(),
14756 fixer: None,
14757 conductor: conductor(),
14758 implementer_roster: Vec::new(),
14759 judge_roster: Vec::new(),
14760 reviewer_roster: Vec::new(),
14761 },
14762 sem: Arc::new(Semaphore::new(1)),
14763 pause: Pause::new(),
14764 interrupt: Pause::new(),
14765 };
14766
14767 runner.merge().await.expect("merge");
14768
14769 assert_eq!(
14770 runner.state.status,
14771 RunStatus::Ready,
14772 "a concluded run's status must not change on reentry"
14773 );
14774 assert_eq!(
14775 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14776 Some("already concluded"),
14777 "merge must not run again once the node already recorded an outcome"
14778 );
14779 }
14780
14781 #[tokio::test]
14790 async fn merge_refuses_a_gate_that_has_not_actually_run() {
14791 let tmp = tempfile::tempdir().expect("tempdir");
14792 let repo = tmp.path().join("repo");
14793 std::fs::create_dir_all(&repo).unwrap();
14794 init_repo(&repo);
14795
14796 let mut config = Config::default();
14797 config.merge.mode = MergeMode::Local;
14798
14799 let mut state = RunState::new(
14800 repo.clone(),
14801 "main".to_owned(),
14802 "deadbeef".to_owned(),
14803 "task".to_owned(),
14804 config,
14805 );
14806 state.candidates = vec![Candidate {
14807 index: 0,
14808 label: 'A',
14809 agent: "alpha".to_owned(),
14810 branch: "does-not-exist".to_owned(),
14811 worktree: repo.clone(),
14812 summary: String::new(),
14813 stat: String::new(),
14814 files: 0,
14815 commits: 0,
14816 empty: false,
14817 failed: None,
14818 verified_noop: None,
14819 duration_ms: 0,
14820 folded: false,
14821 }];
14822 state.tally = Some(Tally {
14823 first_choice: BTreeMap::from([('A', 1)]),
14824 borda: BTreeMap::new(),
14825 winner: 'A',
14826 rankings: 1,
14827 unanimous_initial: true,
14828 deliberated: false,
14829 changed_votes: 0,
14830 unanimous_final: true,
14831 tie_break: None,
14832 judges: 0,
14833 present: 0,
14834 quorum: 0,
14835 met_quorum: true,
14836 uncontested: Some("only candidate A produced a change".to_owned()),
14837 });
14838 state.reviews = vec![ReviewRound {
14839 round: 1,
14840 head: "deadbeef".to_owned(),
14841 verified_head: None,
14842 verified_at: None,
14843 reviews: Vec::new(),
14844 e2e: Vec::new(),
14845 fix: None,
14846 blocking: 0,
14847 answered: 0,
14848 expected: 0,
14849 clean: true,
14850 verify_retried: false,
14851 e2e_deferred: false,
14852 e2e_defer_reason: None,
14853 progressed: false,
14854 vote_split: false,
14855 reconsideration: Vec::new(),
14856 verdict: None,
14857 }];
14858 state.gate = Vec::new();
14860 state.gate_ran = false;
14861 state.status = RunStatus::Gating;
14862
14863 let mut runner = Runner {
14864 state,
14865 roles: ResolvedRoles {
14866 implementers: Vec::new(),
14867 judges: Vec::new(),
14868 reviewers: Vec::new(),
14869 fixer: None,
14870 conductor: conductor(),
14871 implementer_roster: Vec::new(),
14872 judge_roster: Vec::new(),
14873 reviewer_roster: Vec::new(),
14874 },
14875 sem: Arc::new(Semaphore::new(1)),
14876 pause: Pause::new(),
14877 interrupt: Pause::new(),
14878 };
14879
14880 runner.merge().await.expect("merge");
14881
14882 assert!(
14883 runner.state.merge.is_none(),
14884 "an empty gate must never be read as a passing one: {:?}",
14885 runner.state.merge
14886 );
14887 }
14888
14889 #[tokio::test]
14896 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14897 let tmp = tempfile::tempdir().expect("tempdir");
14898 let repo = tmp.path().join("repo");
14899 std::fs::create_dir_all(&repo).unwrap();
14900 init_repo(&repo);
14901
14902 let config = Config::default();
14904
14905 let mut state = RunState::new(
14906 repo.clone(),
14907 "main".to_owned(),
14908 "deadbeef".to_owned(),
14909 "task".to_owned(),
14910 config,
14911 );
14912 state.candidates = vec![Candidate {
14913 index: 0,
14914 label: 'A',
14915 agent: "alpha".to_owned(),
14916 branch: "does-not-exist".to_owned(),
14917 worktree: repo.clone(),
14918 summary: String::new(),
14919 stat: String::new(),
14920 files: 0,
14921 commits: 0,
14922 empty: false,
14923 failed: None,
14924 verified_noop: None,
14925 duration_ms: 0,
14926 folded: false,
14927 }];
14928 state.tally = Some(Tally {
14929 first_choice: BTreeMap::from([('A', 1)]),
14930 borda: BTreeMap::new(),
14931 winner: 'A',
14932 rankings: 1,
14933 unanimous_initial: true,
14934 deliberated: false,
14935 changed_votes: 0,
14936 unanimous_final: true,
14937 tie_break: None,
14938 judges: 0,
14939 present: 0,
14940 quorum: 0,
14941 met_quorum: true,
14942 uncontested: Some("only candidate A produced a change".to_owned()),
14943 });
14944 state.reviews = vec![ReviewRound {
14945 round: 1,
14946 head: "deadbeef".to_owned(),
14947 verified_head: None,
14948 verified_at: None,
14949 reviews: Vec::new(),
14950 e2e: Vec::new(),
14951 fix: None,
14952 blocking: 0,
14953 answered: 0,
14954 expected: 0,
14955 clean: true,
14956 verify_retried: false,
14957 e2e_deferred: false,
14958 e2e_defer_reason: None,
14959 progressed: false,
14960 vote_split: false,
14961 reconsideration: Vec::new(),
14962 verdict: None,
14963 }];
14964
14965 let mut runner = Runner {
14966 state,
14967 roles: ResolvedRoles {
14968 implementers: Vec::new(),
14969 judges: Vec::new(),
14970 reviewers: Vec::new(),
14971 fixer: None,
14972 conductor: conductor(),
14973 implementer_roster: Vec::new(),
14974 judge_roster: Vec::new(),
14975 reviewer_roster: Vec::new(),
14976 },
14977 sem: Arc::new(Semaphore::new(1)),
14978 pause: Pause::new(),
14979 interrupt: Pause::new(),
14980 };
14981
14982 runner.gate().await.expect("gate");
14983 assert!(
14984 runner.state.gate_ran,
14985 "zero configured commands is still a real attempt, not an unrun gate"
14986 );
14987 assert!(runner.state.gate.is_empty());
14988 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
14989 assert_ne!(
14990 runner.state.status,
14991 RunStatus::Blocked,
14992 "a gate with nothing to check must not read as failed"
14993 );
14994
14995 runner.merge().await.expect("merge");
14996 assert_eq!(
14997 runner.state.status,
14998 RunStatus::Ready,
14999 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15000 );
15001 }
15002
15003 #[tokio::test]
15014 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15015 crate::run::pin_test_home();
15016 let home = crate::run::home();
15017
15018 let tmp = tempfile::tempdir().expect("tempdir");
15019 let repo = tmp.path().join("repo");
15020 std::fs::create_dir_all(&repo).unwrap();
15021 init_repo(&repo);
15022 let cache_dir = tmp.path().join("target");
15025
15026 let mut config = Config::default();
15027 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15028 config.graph.timeout_verify = Some(2);
15031 config.graph.timeout_resource_wait = 2;
15032
15033 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15034 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15035 .expect("no io error acquiring directly")
15036 {
15037 crate::cache::AcquireOutcome::Acquired(g) => g,
15038 crate::cache::AcquireOutcome::Busy(b) => {
15039 panic!("expected the direct acquire to win the lease first: {b:?}")
15040 }
15041 };
15042
15043 let mut state = RunState::new(
15044 repo.clone(),
15045 "main".to_owned(),
15046 "deadbeef".to_owned(),
15047 "task".to_owned(),
15048 config,
15049 );
15050 state.candidates = vec![Candidate {
15051 index: 0,
15052 label: 'A',
15053 agent: "alpha".to_owned(),
15054 branch: "does-not-exist".to_owned(),
15055 worktree: repo.clone(),
15056 summary: String::new(),
15057 stat: String::new(),
15058 files: 0,
15059 commits: 0,
15060 empty: false,
15061 failed: None,
15062 verified_noop: None,
15063 duration_ms: 0,
15064 folded: false,
15065 }];
15066 state.tally = Some(Tally {
15067 first_choice: BTreeMap::from([('A', 1)]),
15068 borda: BTreeMap::new(),
15069 winner: 'A',
15070 rankings: 1,
15071 unanimous_initial: true,
15072 deliberated: false,
15073 changed_votes: 0,
15074 unanimous_final: true,
15075 tie_break: None,
15076 judges: 0,
15077 present: 0,
15078 quorum: 0,
15079 met_quorum: true,
15080 uncontested: Some("only candidate A produced a change".to_owned()),
15081 });
15082 state.reviews = vec![ReviewRound {
15083 round: 1,
15084 head: "deadbeef".to_owned(),
15085 verified_head: None,
15086 verified_at: None,
15087 reviews: Vec::new(),
15088 e2e: Vec::new(),
15089 fix: None,
15090 blocking: 0,
15091 answered: 0,
15092 expected: 0,
15093 clean: true,
15094 verify_retried: false,
15095 e2e_deferred: false,
15096 e2e_defer_reason: None,
15097 progressed: false,
15098 vote_split: false,
15099 reconsideration: Vec::new(),
15100 verdict: None,
15101 }];
15102
15103 let mut runner = Runner {
15104 state,
15105 roles: ResolvedRoles {
15106 implementers: Vec::new(),
15107 judges: Vec::new(),
15108 reviewers: Vec::new(),
15109 fixer: None,
15110 conductor: conductor(),
15111 implementer_roster: Vec::new(),
15112 judge_roster: Vec::new(),
15113 reviewer_roster: Vec::new(),
15114 },
15115 sem: Arc::new(Semaphore::new(1)),
15116 pause: Pause::new(),
15117 interrupt: Pause::new(),
15118 };
15119
15120 let started = std::time::Instant::now();
15121 runner.gate().await.expect("gate");
15122 assert!(
15123 started.elapsed() < Duration::from_secs(1),
15124 "a gate with nothing to run must never wait on a lease it never needed"
15125 );
15126 assert!(
15127 runner.state.gate_ran,
15128 "zero commands is still a real, immediate attempt"
15129 );
15130 assert!(runner.state.gate.is_empty());
15131 assert_ne!(
15132 runner.state.status,
15133 RunStatus::Blocked,
15134 "must not read as resource-blocked on a lease it never asked for"
15135 );
15136 }
15137
15138 #[tokio::test]
15148 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15149 crate::run::pin_test_home();
15150
15151 let tmp = tempfile::tempdir().expect("tempdir");
15152 let repo = tmp.path().join("repo");
15153 std::fs::create_dir_all(&repo).unwrap();
15154 init_repo(&repo);
15155
15156 let mut config = Config::default();
15157 config.verify.gate = vec![
15158 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15159 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15160 .to_owned(),
15161 ];
15162
15163 let mut state = RunState::new(
15164 repo.clone(),
15165 "main".to_owned(),
15166 "deadbeef".to_owned(),
15167 "task".to_owned(),
15168 config,
15169 );
15170 let run_id = state.id.clone();
15171 state.candidates = vec![Candidate {
15172 index: 0,
15173 label: 'A',
15174 agent: "alpha".to_owned(),
15175 branch: "does-not-exist".to_owned(),
15176 worktree: repo.clone(),
15177 summary: String::new(),
15178 stat: String::new(),
15179 files: 0,
15180 commits: 0,
15181 empty: false,
15182 failed: None,
15183 verified_noop: None,
15184 duration_ms: 0,
15185 folded: false,
15186 }];
15187 state.tally = Some(Tally {
15188 first_choice: BTreeMap::from([('A', 1)]),
15189 borda: BTreeMap::new(),
15190 winner: 'A',
15191 rankings: 1,
15192 unanimous_initial: true,
15193 deliberated: false,
15194 changed_votes: 0,
15195 unanimous_final: true,
15196 tie_break: None,
15197 judges: 0,
15198 present: 0,
15199 quorum: 0,
15200 met_quorum: true,
15201 uncontested: Some("only candidate A produced a change".to_owned()),
15202 });
15203 state.reviews = vec![ReviewRound {
15204 round: 1,
15205 head: "deadbeef".to_owned(),
15206 verified_head: None,
15207 verified_at: None,
15208 reviews: Vec::new(),
15209 e2e: Vec::new(),
15210 fix: None,
15211 blocking: 0,
15212 answered: 0,
15213 expected: 0,
15214 clean: true,
15215 verify_retried: false,
15216 e2e_deferred: false,
15217 e2e_defer_reason: None,
15218 progressed: false,
15219 vote_split: false,
15220 reconsideration: Vec::new(),
15221 verdict: None,
15222 }];
15223
15224 let mut runner = Runner {
15225 state,
15226 roles: ResolvedRoles {
15227 implementers: Vec::new(),
15228 judges: Vec::new(),
15229 reviewers: Vec::new(),
15230 fixer: None,
15231 conductor: conductor(),
15232 implementer_roster: Vec::new(),
15233 judge_roster: Vec::new(),
15234 reviewer_roster: Vec::new(),
15235 },
15236 sem: Arc::new(Semaphore::new(1)),
15237 pause: Pause::new(),
15238 interrupt: Pause::new(),
15239 };
15240
15241 let started_marker = repo.join("started.marker");
15242 let release_marker = repo.join("release.marker");
15243 let poller = tokio::spawn(async move {
15244 for _ in 0..100 {
15249 if started_marker.exists()
15250 && let Ok(s) = crate::run::RunState::load(&run_id)
15251 && let Some(a) = s.active.get("gate")
15252 {
15253 std::fs::write(&release_marker, b"go").expect("release marker");
15254 return Some(a.clone());
15255 }
15256 tokio::time::sleep(Duration::from_millis(50)).await;
15257 }
15258 None
15259 });
15260
15261 runner.gate().await.expect("gate");
15262 let captured = poller.await.expect("poller task");
15263 let captured = captured.expect(
15264 "the poller never saw a `gate` task entry in run.json while the command was \
15265 still blocked on its own release marker",
15266 );
15267
15268 assert_eq!(captured.task.as_deref(), Some("gate"));
15269 assert_eq!(captured.node, "gate");
15270 assert_eq!(captured.index, Some(1));
15271 assert_eq!(captured.total, Some(1));
15272 assert!(
15273 captured
15274 .command
15275 .as_deref()
15276 .is_some_and(|c| c.contains("started.marker")),
15277 "{captured:?}"
15278 );
15279
15280 assert!(
15281 runner.state.active.is_empty(),
15282 "the entry must be cleared once the command actually finished: {:?}",
15283 runner.state.active
15284 );
15285 assert!(runner.state.gate_ran);
15286 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15287 }
15288
15289 #[tokio::test]
15292 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15293 use crate::verdict::{Finding, ReviewVote, Severity};
15294 crate::run::pin_test_home();
15295 let tmp = tempfile::tempdir().expect("tempdir");
15296 let repo = tmp.path().join("repo");
15297 std::fs::create_dir_all(&repo).unwrap();
15298 init_repo(&repo);
15299
15300 for (severity, vote, expect) in [
15301 (Severity::Major, ReviewVote::Reject, true),
15302 (Severity::Minor, ReviewVote::Reject, false),
15303 (Severity::Major, ReviewVote::Approve, false),
15304 ] {
15305 let mut round = review_round(false, 1, 1, 1, false, true);
15306 round.reviews = vec![ReviewRecord {
15307 reviewer: 1,
15308 agent: "alpha".to_owned(),
15309 summary: String::new(),
15310 findings: vec![Finding {
15311 id: "R1-1-1".to_owned(),
15312 severity,
15313 file: None,
15314 line: None,
15315 title: "t".to_owned(),
15316 detail: String::new(),
15317 }],
15318 vote: Some(vote),
15319 failed: None,
15320 duration_ms: 0,
15321 attempts: 0,
15322 }];
15323 let mut state = RunState::new(
15324 repo.clone(),
15325 "main".to_owned(),
15326 "deadbeef".to_owned(),
15327 "task".to_owned(),
15328 Config::default(),
15329 );
15330 state.reviews = vec![round];
15331 let mut runner = Runner {
15332 state,
15333 roles: ResolvedRoles {
15334 implementers: Vec::new(),
15335 judges: Vec::new(),
15336 reviewers: Vec::new(),
15337 fixer: None,
15338 conductor: conductor(),
15339 implementer_roster: Vec::new(),
15340 judge_roster: Vec::new(),
15341 reviewer_roster: Vec::new(),
15342 },
15343 sem: Arc::new(Semaphore::new(1)),
15344 pause: Pause::new(),
15345 interrupt: Pause::new(),
15346 };
15347 let shell = runner.state.config.shell();
15348 runner
15349 .stop_reviewing("round budget spent", &shell, &repo)
15350 .await
15351 .expect("stop_reviewing");
15352 assert_eq!(runner.state.status, RunStatus::Gating);
15353 assert_eq!(
15354 runner.state.contested_handoff.is_some(),
15355 expect,
15356 "{severity:?} + {vote:?}"
15357 );
15358 }
15359 }
15360
15361 #[tokio::test]
15374 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15375 crate::run::pin_test_home();
15376 let home = crate::run::home();
15377
15378 let tmp = tempfile::tempdir().expect("tempdir");
15379 let repo = tmp.path().join("repo");
15380 std::fs::create_dir_all(&repo).unwrap();
15381 init_repo(&repo);
15382 let head = crate::git::rev_parse(&repo, "HEAD")
15383 .await
15384 .expect("rev-parse");
15385 let cache_dir = tmp.path().join("target");
15388
15389 let mut config = Config::default();
15390 config.verify.e2e = vec![format!(
15391 "CARGO_TARGET_DIR='{}' test -f README.md",
15392 cache_dir.display()
15393 )];
15394 config.graph.review_rounds = 1;
15395 config.graph.timeout_verify = Some(2);
15398 config.graph.timeout_resource_wait = 2;
15399
15400 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15401 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15402 .expect("no io error acquiring directly")
15403 {
15404 crate::cache::AcquireOutcome::Acquired(g) => g,
15405 crate::cache::AcquireOutcome::Busy(b) => {
15406 panic!("expected the direct acquire to win the lease first: {b:?}")
15407 }
15408 };
15409
15410 let mut state = RunState::new(
15411 repo.clone(),
15412 "main".to_owned(),
15413 head.clone(),
15414 "task".to_owned(),
15415 config,
15416 );
15417 state.candidates = vec![Candidate {
15418 index: 0,
15419 label: 'A',
15420 agent: "alpha".to_owned(),
15421 branch: "does-not-exist".to_owned(),
15422 worktree: repo.clone(),
15423 summary: String::new(),
15424 stat: String::new(),
15425 files: 0,
15426 commits: 0,
15427 empty: false,
15428 failed: None,
15429 verified_noop: None,
15430 duration_ms: 0,
15431 folded: false,
15432 }];
15433 state.tally = Some(Tally {
15434 first_choice: BTreeMap::from([('A', 1)]),
15435 borda: BTreeMap::new(),
15436 winner: 'A',
15437 rankings: 1,
15438 unanimous_initial: true,
15439 deliberated: false,
15440 changed_votes: 0,
15441 unanimous_final: true,
15442 tie_break: None,
15443 judges: 0,
15444 present: 0,
15445 quorum: 0,
15446 met_quorum: true,
15447 uncontested: Some("only candidate A produced a change".to_owned()),
15448 });
15449 state.reviews = vec![ReviewRound {
15453 round: 1,
15454 head: head.clone(),
15455 verified_head: None,
15456 verified_at: None,
15457 reviews: Vec::new(),
15458 e2e: Vec::new(),
15459 fix: None,
15460 blocking: 1,
15461 answered: 1,
15462 expected: 1,
15463 clean: false,
15464 verify_retried: false,
15465 e2e_deferred: true,
15466 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15467 progressed: false,
15468 vote_split: false,
15469 reconsideration: Vec::new(),
15470 verdict: None,
15471 }];
15472
15473 let mut runner = Runner {
15474 state,
15475 roles: ResolvedRoles {
15476 implementers: Vec::new(),
15477 judges: Vec::new(),
15478 reviewers: Vec::new(),
15479 fixer: None,
15480 conductor: conductor(),
15481 implementer_roster: Vec::new(),
15482 judge_roster: Vec::new(),
15483 reviewer_roster: Vec::new(),
15484 },
15485 sem: Arc::new(Semaphore::new(1)),
15486 pause: Pause::new(),
15487 interrupt: Pause::new(),
15488 };
15489
15490 let shell = runner.state.config.shell();
15491 runner
15492 .stop_reviewing("round budget spent", &shell, &repo)
15493 .await
15494 .expect("stop_reviewing");
15495
15496 let last = runner.state.reviews.last().expect("round record");
15497 assert_eq!(
15498 last.e2e_status(),
15499 E2eStatus::ResourceBlocked,
15500 "the shared cache is still held; the attempt must read as blocked, not deferred or \
15501 failed: {last:?}"
15502 );
15503 assert_eq!(
15504 last.verified_head.as_deref(),
15505 Some(head.as_str()),
15506 "which commit this attempt targeted is known even though nothing finished checking \
15507 it"
15508 );
15509 let first_attempt_at = last
15510 .verified_at
15511 .expect("when this attempt ran is known too");
15512 assert_ne!(
15513 runner.state.status,
15514 RunStatus::Blocked,
15515 "contention is evidence about the machine, not the patch — it must not settle the \
15516 run as blocked: {:?}",
15517 runner.state.status
15518 );
15519 assert!(
15520 !runner
15521 .state
15522 .events
15523 .iter()
15524 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15525 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15526 runner.state.events
15527 );
15528
15529 runner
15534 .stop_reviewing("round budget spent", &shell, &repo)
15535 .await
15536 .expect("stop_reviewing retry");
15537 assert_eq!(
15538 runner.state.reviews.len(),
15539 1,
15540 "no new round was started: {:?}",
15541 runner.state.reviews
15542 );
15543 let last = runner.state.reviews.last().expect("round record");
15544 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15545 assert!(
15546 last.verified_at.expect("still known") > first_attempt_at,
15547 "a second reentry must be a fresh attempt, not a stale copy of the first"
15548 );
15549 assert_ne!(runner.state.status, RunStatus::Blocked);
15550
15551 held.release();
15552 }
15553
15554 #[tokio::test]
15566 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15567 crate::run::pin_test_home();
15568 let home = crate::run::home();
15569
15570 let tmp = tempfile::tempdir().expect("tempdir");
15571 let repo = tmp.path().join("repo");
15572 std::fs::create_dir_all(&repo).unwrap();
15573 init_repo(&repo);
15574 let head = crate::git::rev_parse(&repo, "HEAD")
15575 .await
15576 .expect("rev-parse");
15577 let cache_dir = tmp.path().join("target");
15578
15579 let mut config = Config::default();
15580 config.verify.e2e = vec![format!(
15581 "CARGO_TARGET_DIR='{}' test -f README.md",
15582 cache_dir.display()
15583 )];
15584 config.graph.review_rounds = 1;
15585 config.graph.timeout_verify = Some(2);
15586 config.graph.timeout_resource_wait = 2;
15587
15588 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15589 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15590 .expect("no io error acquiring directly")
15591 {
15592 crate::cache::AcquireOutcome::Acquired(g) => g,
15593 crate::cache::AcquireOutcome::Busy(b) => {
15594 panic!("expected the direct acquire to win the lease first: {b:?}")
15595 }
15596 };
15597
15598 let mut state = RunState::new(
15599 repo.clone(),
15600 "main".to_owned(),
15601 head.clone(),
15602 "task".to_owned(),
15603 config,
15604 );
15605 state.candidates = vec![Candidate {
15606 index: 0,
15607 label: 'A',
15608 agent: "alpha".to_owned(),
15609 branch: "does-not-exist".to_owned(),
15610 worktree: repo.clone(),
15611 summary: String::new(),
15612 stat: String::new(),
15613 files: 0,
15614 commits: 0,
15615 empty: false,
15616 failed: None,
15617 verified_noop: None,
15618 duration_ms: 0,
15619 folded: false,
15620 }];
15621 state.tally = Some(Tally {
15622 first_choice: BTreeMap::from([('A', 1)]),
15623 borda: BTreeMap::new(),
15624 winner: 'A',
15625 rankings: 1,
15626 unanimous_initial: true,
15627 deliberated: false,
15628 changed_votes: 0,
15629 unanimous_final: true,
15630 tie_break: None,
15631 judges: 0,
15632 present: 0,
15633 quorum: 0,
15634 met_quorum: true,
15635 uncontested: Some("only candidate A produced a change".to_owned()),
15636 });
15637 state.reviews = vec![ReviewRound {
15641 round: 1,
15642 head: head.clone(),
15643 verified_head: Some(head.clone()),
15644 verified_at: Some(jiff::Timestamp::now()),
15645 reviews: Vec::new(),
15646 e2e: vec![CommandOutcome {
15647 command: format!(
15648 "CARGO_TARGET_DIR='{}' test -f README.md",
15649 cache_dir.display()
15650 ),
15651 code: None,
15652 output_tail: "waiting for the shared build cache".to_owned(),
15653 duration_ms: 0,
15654 resource_blocked: true,
15655 }],
15656 fix: None,
15657 blocking: 1,
15658 answered: 1,
15659 expected: 1,
15660 clean: false,
15661 verify_retried: false,
15662 e2e_deferred: false,
15663 e2e_defer_reason: None,
15664 progressed: false,
15665 vote_split: false,
15666 reconsideration: Vec::new(),
15667 verdict: None,
15668 }];
15669
15670 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15671 let mut runner = Runner {
15672 state,
15673 roles: ResolvedRoles {
15674 implementers: Vec::new(),
15675 judges: Vec::new(),
15676 reviewers: Vec::new(),
15677 fixer: None,
15678 conductor: conductor(),
15679 implementer_roster: Vec::new(),
15680 judge_roster: Vec::new(),
15681 reviewer_roster: Vec::new(),
15682 },
15683 sem: Arc::new(Semaphore::new(1)),
15684 pause: Pause::new(),
15685 interrupt: Pause::new(),
15686 };
15687
15688 runner.review_loop().await.expect("review_loop");
15693
15694 assert_eq!(
15695 runner.state.reviews.len(),
15696 1,
15697 "no new round was started on top of the unresolved one: {:?}",
15698 runner.state.reviews
15699 );
15700 let last = &runner.state.reviews[0];
15701 assert_eq!(
15702 last.e2e_status(),
15703 E2eStatus::ResourceBlocked,
15704 "still contended: {last:?}"
15705 );
15706 assert!(
15707 last.verified_at.expect("still known") > first_attempt_at,
15708 "review_loop must have actually retried the check, not left it exactly as found"
15709 );
15710 assert_ne!(
15711 runner.state.status,
15712 RunStatus::Blocked,
15713 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15714 runner.state.status
15715 );
15716
15717 held.release();
15718 }
15719
15720 #[tokio::test]
15721 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15722 crate::run::pin_test_home();
15723 let tmp = tempfile::tempdir().expect("tempdir");
15724 let repo = tmp.path().join("repo");
15725 std::fs::create_dir_all(&repo).unwrap();
15726 init_repo(&repo);
15727
15728 let mut config = Config::default();
15729 config.merge.mode = MergeMode::Pr;
15730 config.graph.land = true;
15731 config.graph.land_approval = false;
15732
15733 let mut state = RunState::new(
15734 repo.clone(),
15735 "main".to_owned(),
15736 "deadbeef".to_owned(),
15737 "task".to_owned(),
15738 config,
15739 );
15740 state.candidates = vec![Candidate {
15741 index: 0,
15742 label: 'A',
15743 agent: "alpha".to_owned(),
15744 branch: "does-not-exist".to_owned(),
15745 worktree: repo.clone(),
15746 summary: String::new(),
15747 stat: String::new(),
15748 files: 0,
15749 commits: 0,
15750 empty: false,
15751 failed: None,
15752 verified_noop: None,
15753 duration_ms: 0,
15754 folded: false,
15755 }];
15756 state.tally = Some(Tally {
15757 first_choice: BTreeMap::from([('A', 1)]),
15758 borda: BTreeMap::new(),
15759 winner: 'A',
15760 rankings: 1,
15761 unanimous_initial: true,
15762 deliberated: false,
15763 changed_votes: 0,
15764 unanimous_final: true,
15765 tie_break: None,
15766 judges: 0,
15767 present: 0,
15768 quorum: 0,
15769 met_quorum: true,
15770 uncontested: Some("only candidate A produced a change".to_owned()),
15771 });
15772 state.reviews = vec![ReviewRound {
15773 round: 1,
15774 head: "deadbeef".to_owned(),
15775 verified_head: None,
15776 verified_at: None,
15777 reviews: Vec::new(),
15778 e2e: Vec::new(),
15779 fix: None,
15780 blocking: 0,
15781 answered: 0,
15782 expected: 0,
15783 clean: true,
15784 verify_retried: false,
15785 e2e_deferred: false,
15786 e2e_defer_reason: None,
15787 progressed: false,
15788 vote_split: false,
15789 reconsideration: Vec::new(),
15790 verdict: None,
15791 }];
15792 state.gate = vec![CommandOutcome {
15793 command: "test".to_owned(),
15794 code: Some(0),
15795 output_tail: String::new(),
15796 duration_ms: 0,
15797 resource_blocked: false,
15798 }];
15799 state.gate_ran = true;
15800 state.status = RunStatus::Landing;
15804 state.merge = Some(MergeOutcome {
15805 mode: MergeMode::Pr,
15806 ok: true,
15807 detail: "https://example.invalid/x/y/pull/1".to_owned(),
15808 empty: false,
15809 });
15810
15811 ask_test_home();
15815 state.save().expect("save landing run");
15818 let store = ask::Questions::open();
15819 let q = ask_open_question(&store, &state.id);
15820
15821 let mut runner = Runner {
15822 state,
15823 roles: ResolvedRoles {
15824 implementers: Vec::new(),
15825 judges: Vec::new(),
15826 reviewers: Vec::new(),
15827 fixer: None,
15828 conductor: conductor(),
15829 implementer_roster: Vec::new(),
15830 judge_roster: Vec::new(),
15831 reviewer_roster: Vec::new(),
15832 },
15833 sem: Arc::new(Semaphore::new(1)),
15834 pause: Pause::new(),
15835 interrupt: Pause::new(),
15836 };
15837
15838 runner.execute().await.expect("execute");
15843
15844 assert_eq!(
15845 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15846 Some("https://example.invalid/x/y/pull/1"),
15847 "reentry must not push again or open a second pull request over the \
15848 one `land` is already watching"
15849 );
15850 assert_ne!(
15851 runner.state.status,
15852 RunStatus::Landing,
15853 "land could not actually reach the fake pull request, so it must \
15854 have given up rather than left the run silently parked forever"
15855 );
15856 assert_eq!(runner.state.status, RunStatus::Blocked);
15860 assert!(
15861 store.get(&q.id).unwrap().status.open(),
15862 "Blocked is still alive; settle_questions must have been a no-op here"
15863 );
15864 }
15865
15866 fn state_with_round(round: ReviewRound) -> RunState {
15867 let mut s = RunState::new(
15868 PathBuf::from("/repo"),
15869 "main".to_owned(),
15870 "abc1234".to_owned(),
15871 "add retries".to_owned(),
15872 Config::default(),
15873 );
15874 s.reviews = vec![round];
15875 s
15876 }
15877
15878 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15879 crate::verdict::Finding {
15880 id: id.to_owned(),
15881 severity,
15882 file: None,
15883 line: None,
15884 title: title.to_owned(),
15885 detail: String::new(),
15886 }
15887 }
15888
15889 #[test]
15890 fn pr_body_names_open_findings_and_declined_ones() {
15891 let round = ReviewRound {
15892 round: 2,
15893 head: "deadbee".to_owned(),
15894 verified_head: None,
15895 verified_at: None,
15896 reviews: vec![ReviewRecord {
15897 attempts: 0,
15898 reviewer: 1,
15899 agent: "alpha".to_owned(),
15900 summary: String::new(),
15901 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15902 vote: None,
15903 failed: None,
15904 duration_ms: 0,
15905 }],
15906 e2e: vec![CommandOutcome {
15907 command: "cargo test".to_owned(),
15908 code: Some(0),
15909 output_tail: String::new(),
15910 duration_ms: 0,
15911 resource_blocked: false,
15912 }],
15913 verify_retried: false,
15914 e2e_deferred: false,
15915 e2e_defer_reason: None,
15916 fix: Some(FixRecord {
15917 agent: "alpha".to_owned(),
15918 addressed: Vec::new(),
15919 rejected: vec![crate::verdict::Rejection {
15920 id: "R1-1-1".to_owned(),
15921 why: "not reachable from any caller".to_owned(),
15922 }],
15923 notes: String::new(),
15924 committed: true,
15925 failed: None,
15926 duration_ms: 0,
15927 continuation: None,
15928 }),
15929 blocking: 0,
15930 answered: 1,
15931 expected: 1,
15932 clean: false,
15933 progressed: true,
15934 vote_split: false,
15935 reconsideration: Vec::new(),
15936 verdict: None,
15937 };
15938 let state = state_with_round(round);
15939 let body = pr_message(&state, 'A').body;
15940
15941 assert!(body.contains("add retries"), "the task must still be there");
15942 assert!(body.contains("R2-1-1"), "{body}");
15943 assert!(!body.contains("unused import"), "{body}");
15944 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15945 assert!(
15946 !body.contains("not reachable from any caller"),
15947 "the reason it was declined: {body}"
15948 );
15949 }
15950
15951 #[test]
15952 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15953 let round = ReviewRound {
15954 round: 1,
15955 head: "deadbee".to_owned(),
15956 verified_head: None,
15957 verified_at: None,
15958 reviews: vec![ReviewRecord {
15959 attempts: 0,
15960 reviewer: 1,
15961 agent: "alpha".to_owned(),
15962 summary: String::new(),
15963 findings: Vec::new(),
15964 vote: None,
15965 failed: None,
15966 duration_ms: 0,
15967 }],
15968 e2e: Vec::new(),
15969 verify_retried: false,
15970 e2e_deferred: false,
15971 e2e_defer_reason: None,
15972 fix: None,
15973 blocking: 0,
15974 answered: 1,
15975 expected: 1,
15976 clean: true,
15977 progressed: false,
15978 vote_split: false,
15979 reconsideration: Vec::new(),
15980 verdict: None,
15981 };
15982 let state = state_with_round(round);
15983 let body = pr_message(&state, 'A').body;
15984 assert!(!body.contains("Open review findings"), "{body}");
15985 assert!(!body.contains("Declined"), "{body}");
15986 }
15987
15988 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15989 let mut state = RunState::new(
15990 PathBuf::from("/repo"),
15991 "main".to_owned(),
15992 "abc1234".to_owned(),
15993 instruction.to_owned(),
15994 Config::default(),
15995 );
15996 state.candidates.push(Candidate {
15997 index: 0,
15998 label: 'A',
15999 agent: "alpha".to_owned(),
16000 branch: "magi/x/A".to_owned(),
16001 worktree: PathBuf::from("/wt"),
16002 summary: summary.to_owned(),
16003 stat: String::new(),
16004 files: 1,
16005 commits: 1,
16006 empty: false,
16007 failed: None,
16008 verified_noop: None,
16009 folded: false,
16010 duration_ms: 0,
16011 });
16012 state
16013 }
16014
16015 fn review_state(subjects: &[&str]) -> RunState {
16016 let mut state = state_with_summary(
16017 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16018 "",
16019 );
16020 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16021 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16022 state
16023 }
16024
16025 #[tokio::test]
16028 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16029 ask_test_home();
16030 let tmp = tempfile::tempdir().unwrap();
16031 let repo = tmp.path().join("repo");
16032 std::fs::create_dir_all(&repo).unwrap();
16033 init_repo(&repo);
16034 let origin = tmp.path().join("origin.git");
16035 let g = |dir: &Path, args: &[&str]| {
16036 let out = std::process::Command::new("git")
16037 .args(args)
16038 .current_dir(dir)
16039 .quiet()
16040 .output()
16041 .expect("spawn git");
16042 assert!(
16043 out.status.success(),
16044 "git {args:?}: {}",
16045 String::from_utf8_lossy(&out.stderr)
16046 );
16047 };
16048 g(
16049 tmp.path(),
16050 &[
16051 "clone",
16052 "--bare",
16053 "-q",
16054 repo.to_str().unwrap(),
16055 origin.to_str().unwrap(),
16056 ],
16057 );
16058 g(
16059 &repo,
16060 &["remote", "add", "origin", origin.to_str().unwrap()],
16061 );
16062 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16063
16064 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16066 g(&repo, &["add", "-A"]);
16067 g(
16068 &repo,
16069 &["commit", "-q", "-m", "chore(deps): update a crate"],
16070 );
16071 g(&repo, &["push", "-q", "origin", "main"]);
16072 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16073 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16074 g(&repo, &["add", "-A"]);
16075 g(
16076 &repo,
16077 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16078 );
16079 g(&repo, &["checkout", "-q", "main"]);
16080
16081 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16082 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16083 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16085 assert_eq!(fallback, c1);
16086
16087 let mut state = review_state(&[
16088 "chore(deps): update a crate",
16089 "fix(daemon): apply a chosen action",
16090 ]);
16091 state.repo = repo.clone();
16092 state.base_branch = "main".to_owned();
16093 state.base_commit = c1;
16094 refresh_reviewed_commits(&mut state, "feat/own").await;
16095 assert_eq!(
16096 state.reviewed_commits,
16097 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16098 );
16099 assert_eq!(
16100 review_title(&state).as_deref(),
16101 Some("fix(daemon): apply a chosen action")
16102 );
16103 assert_eq!(
16104 leaked_subjects(&state, "feat/own").await,
16105 Some(vec!["chore(deps): update a crate".to_owned()])
16106 );
16107 state.config.merge.remote = "nowhere".to_owned();
16110 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16111 }
16112
16113 struct SummaryFixture {
16117 _tmp: tempfile::TempDir,
16118 repo: PathBuf,
16119 reply: PathBuf,
16120 runner: Runner,
16121 }
16122
16123 fn summary_fixture() -> SummaryFixture {
16124 ask_test_home();
16125 let tmp = tempfile::tempdir().unwrap();
16126 let repo = tmp.path().join("repo");
16127 std::fs::create_dir_all(&repo).unwrap();
16128 init_repo(&repo);
16129 let origin = tmp.path().join("origin.git");
16130 let g = |args: &[&str]| {
16131 let out = std::process::Command::new("git")
16132 .args(args)
16133 .current_dir(&repo)
16134 .quiet()
16135 .output()
16136 .expect("spawn git");
16137 assert!(
16138 out.status.success(),
16139 "git {args:?}: {}",
16140 String::from_utf8_lossy(&out.stderr)
16141 );
16142 };
16143 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16144 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16145 g(&["push", "-q", "origin", "main"]);
16146 g(&["checkout", "-q", "-b", "feat/own"]);
16147 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16148 g(&["add", "-A"]);
16149 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16150 let reply = tmp.path().join("reply.txt");
16151 std::fs::write(
16152 &reply,
16153 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16154 )
16155 .unwrap();
16156 let mut runner = runner_at(RunStatus::Gating);
16157 let mut state = review_state(&["feat(own): add the own file"]);
16158 state.repo = repo.clone();
16159 state.base_branch = "main".to_owned();
16160 state.candidates[0].branch = "feat/own".to_owned();
16161 state.candidates[0].worktree = repo.clone();
16162 let mut author = spec("alpha");
16163 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16164 author
16165 .env
16166 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16167 state.config.agents = vec![author];
16168 runner.state = state;
16169 SummaryFixture {
16170 _tmp: tmp,
16171 repo,
16172 reply,
16173 runner,
16174 }
16175 }
16176
16177 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16178 let winner = f.runner.state.candidates[0].clone();
16179 f.runner
16180 .guarded_pr_message(&winner, None, true)
16181 .await
16182 .unwrap()
16183 .expect("an English description never parks")
16184 }
16185
16186 fn clean_round_with_minor_findings() -> ReviewRound {
16187 let mut round = passing_round(2, None);
16188 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16189 round.reviews = vec![ReviewRecord {
16190 attempts: 0,
16191 reviewer: 1,
16192 agent: "alpha".into(),
16193 summary: "内部レビュー".into(),
16194 findings: vec![
16195 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16196 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16197 ],
16198 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16199 failed: None,
16200 duration_ms: 0,
16201 }];
16202 round
16203 }
16204
16205 #[tokio::test]
16206 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16207 for review_only in [false, true] {
16208 let mut f = summary_fixture();
16209 if !review_only {
16210 f.runner.state.candidates[0].agent = "alpha".into();
16211 }
16212 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16213 assert!(f.runner.state.reviews[0].clean);
16214 assert!(f.runner.state.reviews[0].fix.is_none());
16215 assert!(f.runner.state.open_findings().is_empty());
16216 assert_eq!(
16217 crate::pr_outcome::required_finding_ids(&f.runner.state),
16218 ["R2-1-1", "R2-1-2"]
16219 );
16220 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16221 let evidence = crate::pr_outcome::summary_evidence(
16222 &f.runner.state,
16223 crate::pr_outcome::FinalHead::At(&head),
16224 );
16225 assert!(
16226 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16227 );
16228 let omitted = posted(&mut f).await;
16230 assert!(omitted.body.contains("No description"));
16231 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16232 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16233 assert!(f.runner.state.change_summary.is_none());
16234 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16235 let described = posted(&mut f).await;
16236 assert!(described.body.contains("improve documentation for callers"));
16237 assert!(described.body.contains("remaining edge case"));
16238 assert!(described.body.contains("## Open review findings"));
16239 assert!(!described.body.contains("未修正"));
16240 }
16241 }
16242
16243 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16244 let mut f = summary_fixture();
16245 if !review_only {
16246 f.runner.state.candidates[0].agent = "alpha".into();
16247 }
16248 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16249 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16250 fixed
16251 .fix
16252 .as_mut()
16253 .unwrap()
16254 .rejected
16255 .push(crate::verdict::Rejection {
16256 id: "R1-1-1".into(),
16257 why: "呼び出されないコードのため辞退".into(),
16258 });
16259 let mut clean = clean_round_with_minor_findings();
16260 clean.verified_head = Some(head.clone());
16261 f.runner.state.reviews = vec![fixed, clean];
16262 ok_gate(&mut f.runner.state, Some(&head));
16263 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16264 f.runner.state.config.agents.push(scripted("beta", reply));
16265 f.runner.state.config.roles.language_judge =
16266 Some(crate::config::AgentChoice::One("beta".into()));
16267 f
16268 }
16269
16270 #[tokio::test]
16271 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16272 for review_only in [false, true] {
16273 let mut f = rewrite_fixture_with_findings(review_only,
16274 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16275 let m = posted(&mut f).await;
16276 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16277 let expected =
16278 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16279 assert!(m.body.contains(&expected));
16280 assert!(m.body.contains("## Review rounds"));
16281 assert!(
16282 m.body
16283 .contains("Gate (`verify.gate`): passed on the final commit")
16284 );
16285 assert!(
16286 m.body
16287 .contains("Tests (`verify.e2e`): passed on the final commit")
16288 );
16289 let c = f.runner.state.change_summary.as_ref().unwrap();
16290 assert!(
16291 m.body
16292 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16293 );
16294 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16295 assert!(crate::pr_outcome::covers_required_findings(
16296 &f.runner.state,
16297 &m.body
16298 ));
16299 assert!(m.body.contains("documentation remains incomplete"));
16300 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16301 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16302 assert!(f.runner.state.github_text.is_none());
16303 }
16304 }
16305
16306 #[tokio::test]
16307 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16308 for review_only in [false, true] {
16309 for reply in [
16310 "TITLE: fix: public description\nA short English description.",
16311 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16312 ] {
16313 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16314 let winner = f.runner.state.candidates[0].clone();
16315 let posted = f
16316 .runner
16317 .guarded_pr_message(&winner, None, true)
16318 .await
16319 .unwrap();
16320 assert!(posted.is_none());
16321 assert!(f.runner.state.parked);
16322 let withheld = f.runner.state.github_text.as_ref().unwrap();
16323 assert!(
16324 withheld.rewrite.is_none(),
16325 "an incomplete rewrite must not be selectable"
16326 );
16327 let q = ask::Questions::open()
16328 .get(withheld.question.as_ref().unwrap())
16329 .unwrap();
16330 assert!(
16331 !q.choices
16332 .contains(&crate::github_text::USE_REWRITE.to_owned())
16333 );
16334 assert!(
16335 q.detail.contains("R1-1-1")
16336 && q.detail.contains("R2-1-1")
16337 && q.detail.contains("R2-1-2")
16338 );
16339 }
16340 }
16341 }
16342
16343 #[tokio::test]
16344 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16345 for review_only in [false, true] {
16346 let mut f = summary_fixture();
16347 if !review_only {
16348 f.runner.state.candidates[0].agent = "alpha".into();
16349 }
16350 f.runner.state.instruction = "内部タスクの説明".into();
16351 f.runner.state.candidates[0].summary = "内部の候補報告".into();
16352 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16353 f.runner.state.reviews = vec![
16354 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16355 round_with_notes(2, Some("二回目の内部修正報告です。")),
16356 passing_round(3, Some(&head)),
16357 ];
16358 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16359 ok_gate(&mut f.runner.state, Some(&head));
16360 let captured = f.repo.join("public-prompt.txt");
16361 let author = &mut f.runner.state.config.agents[0];
16362 author.command = vec![
16363 "sh".into(),
16364 "-c".into(),
16365 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16366 ];
16367 author.env.insert(
16368 "PROMPT_FILE".into(),
16369 captured.to_string_lossy().into_owned(),
16370 );
16371 let m = posted(&mut f).await;
16372 assert!(m.body.contains("Adds `own.txt`"));
16373 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16374 assert!(m.body.contains("R1-1-1"));
16375 assert!(
16376 m.body
16377 .contains("Gate (`verify.gate`): passed on the final commit")
16378 );
16379 assert!(
16380 m.body
16381 .contains("Tests (`verify.e2e`): passed on the final commit")
16382 );
16383 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16384 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16385 let prompt = std::fs::read_to_string(captured).unwrap();
16386 assert!(prompt.contains(&format!(
16387 "git diff {} {head}",
16388 f.runner.state.change_summary.as_ref().unwrap().base
16389 )));
16390 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16391 assert_eq!(
16392 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16393 "二回目の内部修正報告です。"
16394 );
16395 }
16396 }
16397
16398 #[tokio::test]
16399 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16400 for review_only in [false, true] {
16401 let mut f = summary_fixture();
16402 if !review_only {
16403 f.runner.state.candidates[0].agent = "alpha".into();
16404 }
16405 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16406 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16407 std::fs::write(&f.reply, "no usable description").unwrap();
16408 let failed = posted(&mut f).await;
16409 assert!(
16410 failed
16411 .body
16412 .contains("No description of what this change does could be generated")
16413 );
16414 assert!(failed.body.is_ascii());
16415 assert!(f.runner.state.github_text.is_none());
16416 assert!(f.runner.state.change_summary.is_none());
16417 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16418 let evidence = crate::pr_outcome::summary_evidence(
16419 &f.runner.state,
16420 crate::pr_outcome::FinalHead::At(&head),
16421 );
16422 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16423 assert!(
16424 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16425 .exists()
16426 );
16427 f.runner.state =
16428 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16429 std::fs::write(
16430 &f.reply,
16431 "TITLE: feat(own): add the own file\nAdds the committed own file.",
16432 )
16433 .unwrap();
16434 let retried = posted(&mut f).await;
16435 assert!(retried.body.contains("Adds the committed own file."));
16436 assert_eq!(
16437 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16438 "日本語の修正報告"
16439 );
16440 }
16441 }
16442
16443 #[tokio::test]
16444 async fn same_head_verification_and_findings_invalidate_public_prose() {
16445 let mut f = summary_fixture();
16446 posted(&mut f).await;
16447 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16448 let original = f
16449 .runner
16450 .state
16451 .change_summary
16452 .as_ref()
16453 .unwrap()
16454 .evidence
16455 .clone();
16456 ok_gate(&mut f.runner.state, Some(&head));
16457 std::fs::write(
16458 &f.reply,
16459 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16460 )
16461 .unwrap();
16462 let verified = posted(&mut f).await;
16463 assert!(verified.body.contains("now verified"));
16464 assert_ne!(
16465 f.runner.state.change_summary.as_ref().unwrap().evidence,
16466 original
16467 );
16468 let mut round = round_with_notes(1, Some("内部報告"));
16469 round.reviews.push(ReviewRecord {
16470 attempts: 0,
16471 reviewer: 1,
16472 agent: "alpha".into(),
16473 summary: "内部レビュー".into(),
16474 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16475 vote: None,
16476 failed: None,
16477 duration_ms: 0,
16478 });
16479 round
16480 .fix
16481 .as_mut()
16482 .unwrap()
16483 .rejected
16484 .push(crate::verdict::Rejection {
16485 id: "R1-1-1".into(),
16486 why: "呼び出し元から到達できない".into(),
16487 });
16488 f.runner.state.reviews = vec![round];
16489 let omitted = posted(&mut f).await;
16491 assert!(omitted.body.contains("No description"));
16492 assert!(omitted.body.contains("R1-1-1"));
16493 assert!(!omitted.body.contains("到達"));
16494 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16495 let described = posted(&mut f).await;
16496 assert!(described.body.contains("unreachable from callers"));
16497 assert!(described.body.contains("R1-1-1"));
16498 assert!(!described.body.contains("到達"));
16499 let evidence = f
16500 .runner
16501 .state
16502 .change_summary
16503 .as_ref()
16504 .unwrap()
16505 .evidence
16506 .clone();
16507 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16508 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16509 assert!(posted(&mut f).await.body.contains("rationale updated"));
16510 assert_ne!(
16511 f.runner.state.change_summary.as_ref().unwrap().evidence,
16512 evidence
16513 );
16514 f.runner
16516 .state
16517 .change_summary
16518 .as_mut()
16519 .unwrap()
16520 .evidence
16521 .clear();
16522 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16523 assert!(posted(&mut f).await.body.contains("current specification"));
16524 }
16525
16526 #[tokio::test]
16527 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16528 let mut f = summary_fixture();
16529 let m = posted(&mut f).await;
16530 assert_eq!(m.title, "feat(own): add the own file");
16531 assert!(
16532 m.body
16533 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16534 "{}",
16535 m.body
16536 );
16537 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16538 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16539 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16540 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16541 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16542 let kept = f.runner.state.change_summary.as_ref().unwrap();
16543 assert_eq!(
16544 (kept.head.as_str(), kept.base.as_str()),
16545 (head.as_str(), base.as_str())
16546 );
16547 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16548 }
16549
16550 #[tokio::test]
16551 async fn review_only_summary_receives_the_configured_github_language() {
16552 let mut f = summary_fixture();
16553 f.runner.state.config.graph.github_language = "ja".into();
16554 let captured = f.repo.join("summary-prompt.txt");
16555 let author = &mut f.runner.state.config.agents[0];
16556 author.command = vec![
16557 "sh".into(),
16558 "-c".into(),
16559 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16560 ];
16561 author.env.insert(
16562 "PROMPT_FILE".into(),
16563 captured.to_string_lossy().into_owned(),
16564 );
16565 std::fs::write(
16566 &f.reply,
16567 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16568 )
16569 .unwrap();
16570 let m = posted(&mut f).await;
16571 let prompt = std::fs::read_to_string(captured).unwrap();
16572 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16573 assert!(!prompt.contains("in English"));
16574 assert_eq!(m.title, "feat(own): ファイルを追加");
16575 assert!(m.body.contains("own.txt を追加します。"));
16576 }
16577
16578 #[tokio::test]
16579 async fn review_fixes_come_after_the_whole_change_summary() {
16580 let mut f = summary_fixture();
16581 f.runner.state.reviews = vec![
16582 round_with_notes(1, Some("handled the PATH case")),
16583 round_with_notes(2, None),
16584 ];
16585 let m = posted(&mut f).await;
16586 let summary = m.body.find("## Summary").expect("summary");
16587 let fixes = m.body.find("## Review rounds").expect("review rounds");
16588 assert!(summary < fixes, "{}", m.body);
16589 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16590 assert!(m.body.contains("## Final verification"), "{}", m.body);
16591 }
16592
16593 #[tokio::test]
16594 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16595 let mut f = summary_fixture();
16596 let first = posted(&mut f).await;
16597 let first_head = f.runner.state.change_summary.clone().unwrap().head;
16598 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16600 let again = posted(&mut f).await;
16601 assert_eq!((again.title, again.body), (first.title, first.body));
16602
16603 let g = |args: &[&str]| {
16604 let out = std::process::Command::new("git")
16605 .args(args)
16606 .current_dir(&f.repo)
16607 .quiet()
16608 .output()
16609 .expect("spawn git");
16610 assert!(out.status.success(), "git {args:?}");
16611 };
16612 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16613 g(&["add", "-A"]);
16614 g(&["commit", "-q", "-m", "fix(own): handle more"]);
16615 let second = posted(&mut f).await;
16616 let kept = f.runner.state.change_summary.clone().unwrap();
16617 assert_ne!(kept.head, first_head);
16618 assert!(second.body.contains("Not read yet."), "{}", second.body);
16619 assert!(!second.body.contains("so the first summary is read"));
16620 assert_eq!(second.title, "feat(own): other");
16621 }
16622
16623 #[tokio::test]
16624 async fn an_unavailable_summary_is_stated_and_never_invented() {
16625 let mut f = summary_fixture();
16626 std::fs::write(&f.reply, "no title line here").unwrap();
16627 let m = posted(&mut f).await;
16628 assert!(
16629 m.body
16630 .contains("No description of what this change does could be generated"),
16631 "{}",
16632 m.body
16633 );
16634 assert!(m.body.contains("internal reports remain in local records"));
16635 assert!(f.runner.state.change_summary.is_none());
16636 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16637 let winner = f.runner.state.candidates[0].clone();
16639 let preview = f
16640 .runner
16641 .guarded_pr_message(&winner, None, false)
16642 .await
16643 .unwrap()
16644 .unwrap();
16645 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16646 }
16647
16648 #[test]
16649 fn pr_message_review_single_commit_uses_its_subject() {
16650 let state = review_state(&["feat(nats): per-role user"]);
16651 let m = pr_message(&state, 'A');
16652 assert_eq!(m.title, "feat(nats): per-role user");
16653 assert!(!m.body.contains("Review the work already"), "{}", m.body);
16654 assert!(m.body.contains("## Commits under review"), "{}", m.body);
16655 }
16656
16657 #[test]
16658 fn pr_message_review_multi_commit_takes_the_oldest() {
16659 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16660 let m = pr_message(&state, 'A');
16661 assert_eq!(m.title, "feat: the change");
16662 for s in ["feat: the change", "fix: typo", "fix: again"] {
16663 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16664 }
16665 }
16666
16667 #[test]
16668 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16669 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16670 let mut state = review_state(&[first, "fix: later fixup"]);
16671 state.candidates[0].branch = "機能/ログイン".to_owned();
16672 let m = pr_message(&state, 'A');
16673 assert!(
16674 m.title.starts_with("chore: land candidate A of run"),
16675 "{}",
16676 m.title
16677 );
16678 }
16679 }
16680
16681 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16682 BranchFacts {
16683 commits: commits
16684 .iter()
16685 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16686 .collect(),
16687 stat: stat.to_owned(),
16688 }
16689 }
16690
16691 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16692 let mut r = review_round(true, 0, 1, 1, true, true);
16693 r.round = round;
16694 r.fix = notes.map(|n| FixRecord {
16695 agent: "fixer".to_owned(),
16696 addressed: Vec::new(),
16697 rejected: Vec::new(),
16698 notes: n.to_owned(),
16699 committed: true,
16700 failed: None,
16701 duration_ms: 0,
16702 continuation: None,
16703 });
16704 r
16705 }
16706
16707 #[test]
16708 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16709 let state = review_state(&["ignored"]);
16710 let f = facts(
16711 &[
16712 (
16713 "fix(login): resolve PATH on macOS",
16714 "Login shells skip rc files.",
16715 ),
16716 ("fix: address review", ""),
16717 ],
16718 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16719 );
16720 let m = pr_message_with(&state, 'A', Some(&f));
16721 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16722 assert!(
16723 m.body
16724 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
16725 "{}",
16726 m.body
16727 );
16728 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16729 assert!(m.body.contains("## Diff stat"), "{}", m.body);
16730 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16731 for banned in [
16732 "Review the work already",
16733 "no task statement",
16734 "Original task",
16735 ] {
16736 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16737 }
16738 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16739 }
16740
16741 #[test]
16742 fn pr_message_review_truncates_a_huge_first_commit_body() {
16743 let state = review_state(&["ignored"]);
16744 let f = facts(
16745 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16746 "s",
16747 );
16748 let m = pr_message_with(&state, 'A', Some(&f));
16749 assert!(m.body.len() < 30_000, "{}", m.body.len());
16750 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16751 assert!(
16752 m.body.contains("- ... 1 more commit(s)"),
16753 "{}",
16754 m.body.len()
16755 );
16756 assert!(m.body.ends_with("magi:candidate-a\n"));
16757 }
16758
16759 #[test]
16760 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16761 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16762 assert_eq!(m.title, "feat: x");
16763 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16764 assert!(!m.body.contains("Diff stat"), "{}", m.body);
16765 assert!(!m.body.contains("no task statement"), "{}", m.body);
16766 }
16767
16768 #[test]
16769 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16770 let mut state = review_state(&["日本語の件名"]);
16771 state.candidates[0].branch = "fix/macos-login-path".to_owned();
16772 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16773 state.candidates[0].branch = "機能/ログイン".to_owned();
16774 assert!(
16775 pr_message(&state, 'A')
16776 .title
16777 .starts_with("chore: land candidate A")
16778 );
16779 }
16780
16781 #[test]
16782 fn pr_message_review_fixes_survive_a_clean_final_round() {
16783 let mut state = review_state(&["feat: x"]);
16784 state.reviews = vec![
16785 round_with_notes(1, Some("handled the PATH case")),
16786 round_with_notes(2, None),
16787 ];
16788 let body = pr_message(&state, 'A').body;
16789 assert!(
16790 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16791 "{body}"
16792 );
16793 assert!(body.contains("### Round 1"), "{body}");
16794
16795 state.reviews = vec![
16796 round_with_notes(1, Some("first fix")),
16797 round_with_notes(2, Some("")),
16798 round_with_notes(3, Some("second fix")),
16799 round_with_notes(4, None),
16800 ];
16801 let body = pr_message(&state, 'A').body;
16802 assert!(
16803 body.contains("### Round 1") && !body.contains("first fix"),
16804 "{body}"
16805 );
16806 assert!(
16807 body.contains("### Round 3") && !body.contains("second fix"),
16808 "{body}"
16809 );
16810 assert!(body.contains("### Round 2"), "{body}");
16811 }
16812
16813 const FINAL: &str = "abcdef1234567890";
16814
16815 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16816 let mut r = round_with_notes(round, None);
16817 r.verified_head = verified.map(str::to_owned);
16818 r
16819 }
16820
16821 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16822 state.gate = vec![CommandOutcome {
16823 command: "gate".to_owned(),
16824 code: Some(0),
16825 output_tail: String::new(),
16826 duration_ms: 0,
16827 resource_blocked: false,
16828 }];
16829 state.gate_ran = true;
16830 state.gate_head = head.map(str::to_owned);
16831 }
16832
16833 fn body_at(state: &RunState) -> String {
16834 pr_message_raw(
16835 state,
16836 'A',
16837 None,
16838 crate::pr_outcome::FinalHead::At(FINAL),
16839 ChangeNote::NotAsked,
16840 )
16841 .body
16842 }
16843
16844 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16845
16846 #[test]
16847 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16848 let mut state = review_state(&["feat: x"]);
16849 state.reviews = vec![
16850 round_with_notes(1, Some(SANDBOXED)),
16851 passing_round(2, Some(FINAL)),
16852 ];
16853 ok_gate(&mut state, Some(FINAL));
16854 let body = body_at(&state);
16855 assert!(!body.contains("Handled the PATH case."), "{body}");
16856 assert!(!body.contains("could not commit"), "{body}");
16857 assert!(!body.contains("could not run"), "{body}");
16858 assert!(
16859 body.contains("The fix is committed on the branch"),
16860 "{body}"
16861 );
16862 assert!(!body.contains("sandbox"), "{body}");
16863 assert!(
16864 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16865 "{body}"
16866 );
16867 assert!(
16868 body.contains("Gate (`verify.gate`): passed on the final commit"),
16869 "{body}"
16870 );
16871 assert!(
16873 state.reviews[0]
16874 .fix
16875 .as_ref()
16876 .unwrap()
16877 .notes
16878 .contains("could not commit")
16879 );
16880 }
16881
16882 #[test]
16883 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16884 let mut state = review_state(&["feat: x"]);
16885 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16886 r1.fix.as_mut().unwrap().committed = false;
16887 let mut r2 = passing_round(2, Some(FINAL));
16888 r2.e2e[0].code = Some(1);
16889 state.reviews = vec![r1, r2];
16890 let body = body_at(&state);
16891 assert!(!body.contains("could not commit"), "{body}");
16892 assert!(!body.contains("could not run"), "{body}");
16893 assert!(body.contains("This round produced no commit"), "{body}");
16894 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16895 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16896 assert!(!body.contains("passed"), "{body}");
16897 }
16898
16899 #[test]
16900 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16901 let mut state = review_state(&["feat: x"]);
16902 state.reviews = vec![
16903 round_with_notes(1, Some(SANDBOXED)),
16904 passing_round(2, Some("0123456789abcdef")),
16905 ];
16906 ok_gate(&mut state, Some("0123456789abcdef"));
16907 let body = body_at(&state);
16908 assert!(!body.contains("HTTP tests could not run"), "{body}");
16909 assert!(!body.contains("could not commit"), "{body}");
16911 assert!(
16912 body.contains("passed on an earlier commit (`0123456`)"),
16913 "{body}"
16914 );
16915 assert!(!body.contains("passed on the final commit"), "{body}");
16916
16917 ok_gate(&mut state, None);
16919 let body = body_at(&state);
16920 assert!(
16921 body.contains("passed on a commit that was not recorded"),
16922 "{body}"
16923 );
16924 }
16925
16926 #[test]
16927 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16928 let state = state_with_summary(
16929 "add retries to the client",
16930 "TITLE: feat: retries\n\nDid it.",
16931 );
16932 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16933 assert_eq!(m.title, "feat: retries");
16934 assert!(
16935 m.body.contains("<summary>Original task</summary>"),
16936 "{}",
16937 m.body
16938 );
16939 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16940 assert!(
16941 m.body
16942 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16943 "{}",
16944 m.body
16945 );
16946 }
16947
16948 #[test]
16949 fn pr_message_review_bounds_a_long_english_subject() {
16950 let long = format!("feat: {}", "word ".repeat(100));
16951 let m = pr_message(&review_state(&[&long]), 'A');
16952 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16953 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16954 }
16955
16956 #[test]
16957 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16958 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16959 let m = pr_message(&state, 'A');
16960 assert_eq!(m.title, "add retries");
16961 assert!(m.body.contains("<summary>Original task</summary>"));
16962 assert!(!m.body.contains("Commits under review"));
16963 assert_eq!(landing_subject_source(&state), state.instruction);
16964 }
16965
16966 #[test]
16967 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16968 let state = review_state(&["feat: the change", "fix: typo"]);
16969 let source = landing_subject_source(&state);
16970 assert_eq!(land::merge_subject("", &source), "feat: the change");
16971 assert_eq!(
16972 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16973 "feat: the change"
16974 );
16975 assert_eq!(
16977 land::merge_subject("feat: renamed by hand", &source),
16978 "feat: renamed by hand"
16979 );
16980 let blank = review_state(&["日本語"]);
16981 assert!(
16982 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16983 );
16984 }
16985
16986 #[test]
16987 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16988 let state = review_state(&["feat: the change"]);
16989 let source = landing_subject_source(&state);
16990 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16991 assert_eq!(
16992 land::merge_subject(landing_title(&state, old), &source),
16993 "feat: the change"
16994 );
16995 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16996 let task = state_with_summary("add retries", "");
16997 assert_eq!(landing_title(&task, old), old);
16998 }
16999
17000 #[test]
17001 fn pr_message_describes_the_change_not_the_task() {
17002 let state = state_with_summary(
17003 "今回やってほしいこと: results projector を直す",
17004 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17005 );
17006 let m = pr_message(&state, 'A');
17007 assert_eq!(m.title, "fix(web): batch the runs list reads");
17008 assert!(
17009 m.body.starts_with("## Summary\n\n- reads run.json once"),
17010 "{}",
17011 m.body
17012 );
17013 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17014 let task_at = m.body.find("今回やってほしいこと").unwrap();
17015 let details_at = m.body.find("<details>").unwrap();
17016 assert!(
17017 details_at < task_at,
17018 "the task lives inside <details>: {}",
17019 m.body
17020 );
17021 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17022 assert!(m.body.contains("magi:candidate-a"));
17023 }
17024
17025 #[test]
17026 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17027 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17028 let m = pr_message(&state, 'A');
17029 assert_eq!(m.title, "add retries");
17030 assert!(
17031 m.body.contains("## Summary\n\n- did some things"),
17032 "{}",
17033 m.body
17034 );
17035
17036 let none = RunState::new(
17037 PathBuf::from("/repo"),
17038 "main".to_owned(),
17039 "abc1234".to_owned(),
17040 "add retries".to_owned(),
17041 Config::default(),
17042 );
17043 let m = pr_message(&none, 'A');
17044 assert_eq!(m.title, "add retries");
17045 assert!(!m.body.contains("## Summary"), "{}", m.body);
17046 }
17047
17048 #[test]
17049 fn pr_message_refuses_the_candidate_commit_subject() {
17050 for bad in [
17051 "TITLE: magi: candidate A (uncommitted work)",
17052 "TITLE: chore: stuff (uncommitted work)",
17053 "TITLE: ",
17054 ] {
17055 let state = state_with_summary("add retries", bad);
17056 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17057 }
17058 }
17059
17060 #[test]
17061 fn pr_message_bounds_a_very_long_task_and_title() {
17062 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17063 let state = state_with_summary(&long, "- nothing");
17064 let m = pr_message(&state, 'A');
17065 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17066 assert!(!m.title.contains('\n'));
17067
17068 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17069 let m = pr_message(&state, 'A');
17070 assert!(m.title.starts_with("feat: "));
17071 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17072 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17073 }
17074
17075 fn long_title_of(instruction: &str) -> String {
17076 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17077 }
17078
17079 #[test]
17080 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17081 let first = "Make the landing path keep a readable title for long tasks";
17082 let line = format!(
17083 "{first}. {}",
17084 "And then keep going with more words ".repeat(20)
17085 );
17086 let t = long_title_of(&line);
17087 assert_eq!(t, first);
17088 assert!(!t.starts_with("chore: land"));
17089 }
17090
17091 #[test]
17092 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17093 let line = "word ".repeat(200);
17094 let t = long_title_of(&line);
17095 assert!(t.ends_with("word..."), "{t}");
17096 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17097 }
17098
17099 #[test]
17100 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17101 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17102 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17103 }
17104 }
17105
17106 #[test]
17107 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17108 let mut state = state_with_summary("再試行を追加する", "- nothing");
17109 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17110 state.config.graph.github_language = "ja".into();
17111 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17112 }
17113
17114 #[test]
17115 fn pr_message_title_limit_is_exact() {
17116 let at = "a".repeat(PR_TITLE_MAX);
17117 assert_eq!(long_title_of(&at), at);
17118 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17119 assert!(over.ends_with("..."), "{over}");
17120 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17121 }
17122
17123 #[test]
17124 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17125 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17126 let t = long_title_of(&line);
17127 assert!(t.ends_with("..."), "{t}");
17128 assert!(t.is_ascii(), "{t}");
17129 }
17130
17131 #[test]
17132 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17133 let line = format!(
17134 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17135 "plus more filler words ".repeat(20)
17136 );
17137 let t = long_title_of(&line);
17138 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17139 }
17140
17141 #[test]
17142 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17143 let line = format!("\n\n# {}", "title words ".repeat(40));
17144 let state = state_with_summary(&line, "- nothing");
17145 let m = pr_message(&state, 'A');
17146 assert!(m.title.starts_with("title words"), "{}", m.title);
17147 assert_eq!(
17148 land::merge_subject(&m.title, &landing_subject_source(&state)),
17149 m.title
17150 );
17151 assert_eq!(
17153 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17154 "feat: renamed by hand"
17155 );
17156 }
17157
17158 #[tokio::test]
17159 async fn github_text_rewrite_is_bounded_and_falls_back() {
17160 crate::run::pin_test_home();
17161 for (reply, accepted) in [
17162 (
17163 "TITLE: fix: retries\nAdd retries for failed requests.",
17164 true,
17165 ),
17166 (
17167 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17168 false,
17169 ),
17170 (
17171 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17172 true,
17173 ),
17174 (
17175 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17176 true,
17177 ),
17178 (
17179 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17180 true,
17181 ),
17182 (
17183 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17184 false,
17185 ),
17186 ("TITLE: fix: retries\n", false),
17187 ] {
17188 let dir = tempfile::tempdir().unwrap();
17189 let mut runner = runner_at(RunStatus::Gating);
17190 runner.state = state_with_summary(
17191 "add retries",
17192 "TITLE: fix: retries\n日本語の説明を書きます。",
17193 );
17194 runner.state.repo = dir.path().to_owned();
17195 runner.state.candidates[0].worktree = dir.path().to_owned();
17196 let mut author = spec("alpha");
17197 author.command = vec![
17198 "sh".into(),
17199 "-c".into(),
17200 "printf '%s' \"$REWRITE_REPLY\"".into(),
17201 ];
17202 author.env.insert("REWRITE_REPLY".into(), reply.into());
17203 runner.state.config.agents = vec![author];
17204 seed_public_summary(&mut runner);
17205 let winner = runner.state.candidates[0].clone();
17206 let first = runner
17207 .guarded_pr_message(&winner, None, true)
17208 .await
17209 .unwrap();
17210 let message = if accepted {
17211 first.expect("a passing rewrite never asks")
17212 } else {
17213 assert!(first.is_none());
17215 assert!(runner.state.parked);
17216 let store = ask::Questions::open();
17217 let asked: Vec<_> = store
17218 .list()
17219 .into_iter()
17220 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17221 .collect();
17222 assert_eq!(asked.len(), 1);
17223 assert!(!asked[0].detail.contains("secret"));
17224 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17225 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17226 assert_eq!(
17227 asked[0]
17228 .choices
17229 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17230 rewrite.is_some()
17231 );
17232 store
17235 .update(&asked[0].id, |q| {
17236 q.abandon("test");
17237 Ok(())
17238 })
17239 .unwrap();
17240 let again = runner
17241 .guarded_pr_message(&winner, None, true)
17242 .await
17243 .unwrap()
17244 .expect("an abandoned question falls back");
17245 let n = store
17246 .list()
17247 .into_iter()
17248 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17249 .count();
17250 assert_eq!(n, 1, "asked once per rejected text");
17251 again
17252 };
17253 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17254 assert!(!message.body.contains("secret"));
17255 assert!(!message.body.contains("prefix"));
17256 assert!(!message.title.contains("secret"));
17257 assert_eq!(
17258 message.body.contains("Add retries for failed requests."),
17259 accepted
17260 );
17261 assert_eq!(
17262 runner.state.seats["github-text-rewrite"].turns, 1,
17263 "only one rewrite invocation"
17264 );
17265 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17266 if accepted {
17267 assert!(runner.state.github_text.is_none());
17268 assert!(!runner.state.parked);
17269 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17270 assert!(message.body.contains("## Final verification"));
17271 assert!(message.body.contains("Evidence: final diff"));
17272 assert!(
17273 message
17274 .body
17275 .contains(&format!("magi:run/{}", runner.state.id))
17276 );
17277 }
17278 if !accepted {
17279 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17280 assert!(
17281 message
17282 .body
17283 .contains(&format!("magi:run/{}", runner.state.id))
17284 );
17285 }
17286 }
17287 }
17288
17289 fn scripted(id: &str, reply: &str) -> AgentSpec {
17290 let mut a = spec(id);
17291 a.command = vec![
17292 "sh".into(),
17293 "-c".into(),
17294 "printf '%s' \"$REWRITE_REPLY\"".into(),
17295 ];
17296 a.env.insert("REWRITE_REPLY".into(), reply.into());
17297 a
17298 }
17299
17300 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17301 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17302
17303 fn seed_public_summary(runner: &mut Runner) {
17305 let repo = &runner.state.repo;
17306 let g = |args: &[&str]| {
17307 let out = std::process::Command::new("git")
17308 .args(args)
17309 .current_dir(repo)
17310 .quiet()
17311 .output()
17312 .unwrap();
17313 assert!(out.status.success(), "git {args:?}");
17314 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17315 };
17316 g(&["init", "-q", "-b", "main"]);
17317 g(&[
17318 "-c",
17319 "user.name=Test",
17320 "-c",
17321 "user.email=test@example.invalid",
17322 "commit",
17323 "-q",
17324 "--allow-empty",
17325 "-m",
17326 "feat: fixture",
17327 ]);
17328 let head = g(&["rev-parse", "HEAD"]);
17329 g(&["update-ref", "refs/remotes/origin/main", &head]);
17330 let evidence = crate::pr_outcome::summary_evidence(
17331 &runner.state,
17332 crate::pr_outcome::FinalHead::At(&head),
17333 );
17334 let generated = &runner.state.candidates[0].summary;
17335 runner.state.change_summary = Some(crate::run::ChangeSummary {
17336 base: head.clone(),
17337 head,
17338 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17339 title: summary_title(generated).unwrap(),
17340 body: summary_without_title(generated),
17341 });
17342 }
17343
17344 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17345 let mut runner = runner_at(RunStatus::Gating);
17346 runner.state = state_with_summary("add retries", JA_SUMMARY);
17347 if review_only {
17348 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17349 }
17350 runner.state.repo = dir.to_owned();
17351 runner.state.candidates[0].worktree = dir.to_owned();
17352 runner.state.config.agents = agents;
17353 seed_public_summary(&mut runner);
17354 runner
17355 }
17356
17357 fn gate_questions(runner: &Runner) -> usize {
17358 ask::Questions::open()
17359 .list()
17360 .into_iter()
17361 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17362 .count()
17363 }
17364
17365 #[tokio::test]
17366 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17367 crate::run::pin_test_home();
17368 let dir = tempfile::tempdir().unwrap();
17369 let mut runner = rewrite_runner(
17370 dir.path(),
17371 true,
17372 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17373 );
17374 runner.state.config.roles.language_judge =
17375 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17376 let winner = runner.state.candidates[0].clone();
17377 let msg = runner
17378 .guarded_pr_message(&winner, None, true)
17379 .await
17380 .unwrap()
17381 .expect("an English rewrite posts without asking");
17382 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17383 assert!(msg.body.contains("<details>kept</details>"));
17384 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17385 assert_eq!(gate_questions(&runner), 0);
17386 assert!(runner.state.github_text.is_none());
17387 assert!(!runner.state.seats.contains_key("impl-A"));
17388 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17389 }
17390
17391 #[tokio::test]
17392 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17393 crate::run::pin_test_home();
17394 let dir = tempfile::tempdir().unwrap();
17395 let mut runner = rewrite_runner(
17396 dir.path(),
17397 false,
17398 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17399 );
17400 runner.state.config.roles.language_judge =
17401 Some(crate::config::AgentChoice::One("beta".into()));
17402 runner.state.seats.insert(
17403 "impl-A".into(),
17404 SeatState::new("impl-A", "alpha", runner.state.seed),
17405 );
17406 let winner = runner.state.candidates[0].clone();
17407 let msg = runner
17408 .guarded_pr_message(&winner, None, true)
17409 .await
17410 .unwrap()
17411 .expect("rewritten");
17412 assert!(msg.body.contains("<details>kept</details>"));
17413 assert_eq!(runner.state.seats["impl-A"].turns, 0);
17414 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17415 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17416 }
17417
17418 #[tokio::test]
17419 async fn rewrite_reuses_a_resumable_implementer_session() {
17420 crate::run::pin_test_home();
17421 let dir = tempfile::tempdir().unwrap();
17422 let mut runner = rewrite_runner(
17423 dir.path(),
17424 false,
17425 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17426 );
17427 runner.state.config.roles.language_judge =
17428 Some(crate::config::AgentChoice::One("beta".into()));
17429 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17430 seat.turns = 1;
17431 runner.state.seats.insert("impl-A".into(), seat);
17432 let winner = runner.state.candidates[0].clone();
17433 runner
17434 .guarded_pr_message(&winner, None, true)
17435 .await
17436 .unwrap()
17437 .expect("rewritten");
17438 assert_eq!(runner.state.seats["impl-A"].turns, 2);
17439 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17440 }
17441
17442 #[tokio::test]
17443 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17444 crate::run::pin_test_home();
17445 let dir = tempfile::tempdir().unwrap();
17446 let mut broken = spec("bad");
17447 broken.command = vec!["false".into()];
17448 let mut runner = rewrite_runner(
17449 dir.path(),
17450 true,
17451 vec![
17452 broken,
17453 scripted("noformat", "Add retries."),
17454 scripted("good", EN_REWRITE),
17455 ],
17456 );
17457 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17458 "bad".into(),
17459 "noformat".into(),
17460 "good".into(),
17461 ]));
17462 let winner = runner.state.candidates[0].clone();
17463 let msg = runner
17464 .guarded_pr_message(&winner, None, true)
17465 .await
17466 .unwrap()
17467 .expect("the third agent rewrites");
17468 assert!(msg.body.contains("`src/net.rs`"));
17469 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17470 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17471 }
17472
17473 #[tokio::test]
17474 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17475 crate::run::pin_test_home();
17476 for (agents, role, saved) in [
17478 (
17479 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17480 Some("beta"),
17481 true,
17482 ),
17483 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17484 (Vec::new(), None, false),
17485 ] {
17486 let dir = tempfile::tempdir().unwrap();
17487 let mut runner = rewrite_runner(dir.path(), true, agents);
17488 runner.state.config.roles.language_judge =
17489 role.map(|r| crate::config::AgentChoice::One(r.into()));
17490 let winner = runner.state.candidates[0].clone();
17491 let first = runner
17492 .guarded_pr_message(&winner, None, true)
17493 .await
17494 .unwrap();
17495 assert!(first.is_none(), "withheld text parks the run");
17496 assert!(runner.state.parked);
17497 assert_eq!(gate_questions(&runner), 1);
17498 let g = runner.state.github_text.as_ref().unwrap();
17499 assert_eq!(g.rewrite.is_some(), saved);
17500 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17501 }
17502 }
17503
17504 #[tokio::test]
17505 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17506 crate::run::pin_test_home();
17507 let dir = tempfile::tempdir().unwrap();
17508 let mut runner = runner_at(RunStatus::Gating);
17509 runner.state =
17510 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17511 runner.state.repo = dir.path().to_owned();
17512 runner.state.candidates[0].worktree = dir.path().to_owned();
17513 let mut author = spec("alpha");
17514 author.command = vec![
17515 "sh".into(),
17516 "-c".into(),
17517 "printf '%s' \"$REWRITE_REPLY\"".into(),
17518 ];
17519 author.env.insert(
17520 "REWRITE_REPLY".into(),
17521 format!(
17522 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17523 "English sentence. ".repeat(600)
17524 ),
17525 );
17526 runner.state.config.agents = vec![author];
17527 seed_public_summary(&mut runner);
17528 let winner = runner.state.candidates[0].clone();
17529 assert!(
17530 runner
17531 .guarded_pr_message(&winner, None, true)
17532 .await
17533 .unwrap()
17534 .is_none()
17535 );
17536 let record = runner.state.github_text.clone().unwrap();
17537 let rewrite = record.rewrite.clone().unwrap();
17538 assert_eq!(rewrite.failed, ["body-language"]);
17539 assert!(
17540 rewrite
17541 .body
17542 .ends_with(&format!("magi:run/{}", runner.state.id))
17543 );
17544 let store = ask::Questions::open();
17545 let q = store.get(record.question.as_ref().unwrap()).unwrap();
17546 assert!(
17547 q.choices
17548 .contains(&crate::github_text::USE_REWRITE.to_owned())
17549 );
17550 for label in [
17551 "ORIGINAL",
17552 "FALLBACK",
17553 "REWRITE",
17554 "Rules still failed: body-language",
17555 "truncated",
17556 ] {
17557 assert!(q.detail.contains(label), "{label}");
17558 }
17559 let artifact = std::fs::read_to_string(crate::run::artifact_path(
17560 &runner.state,
17561 &crate::github_text::artifact_name(&record.fingerprint),
17562 ))
17563 .unwrap();
17564 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17565 runner.state =
17567 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17568 store
17569 .update(&q.id, |q| {
17570 q.say("まだ日本語")?;
17571 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17572 })
17573 .unwrap();
17574 assert!(
17575 runner
17576 .guarded_pr_message(&winner, None, true)
17577 .await
17578 .unwrap()
17579 .is_none()
17580 );
17581 let retry = runner.state.github_text.as_ref().unwrap();
17582 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17583 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17584 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17585 assert!(
17586 q.posting_comparison
17587 .as_ref()
17588 .unwrap()
17589 .matches_detail(&q.detail)
17590 );
17591 store
17592 .update(&q.id, |q| {
17593 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17594 })
17595 .unwrap();
17596 let message = runner
17597 .guarded_pr_message(&winner, None, true)
17598 .await
17599 .unwrap()
17600 .unwrap();
17601 assert_eq!(message.title, rewrite.title);
17602 assert_eq!(message.body, rewrite.body);
17603 assert_eq!(
17604 runner.state.github_text.as_ref().unwrap().choice,
17605 Some(GithubTextChoice::Rewrite)
17606 );
17607 let again = runner
17608 .guarded_pr_message(&winner, None, true)
17609 .await
17610 .unwrap()
17611 .unwrap();
17612 assert_eq!(again.title, message.title);
17613 assert_eq!(again.body, message.body);
17614 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17615 assert!(
17616 !runner
17617 .state
17618 .events
17619 .iter()
17620 .any(|e| e.message.contains(&rewrite.title)
17621 || e.message.contains("English sentence."))
17622 );
17623 }
17624
17625 #[tokio::test]
17626 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17627 crate::run::pin_test_home();
17628 let dir = tempfile::tempdir().unwrap();
17629 let mut runner = runner_at(RunStatus::Gating);
17630 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17631 runner.state.repo = dir.path().to_owned();
17632 runner.state.candidates[0].worktree = dir.path().to_owned();
17633 seed_public_summary(&mut runner);
17634 let winner = runner.state.candidates[0].clone();
17635 assert!(
17636 runner
17637 .guarded_pr_message(&winner, None, true)
17638 .await
17639 .unwrap()
17640 .is_none()
17641 );
17642 let record = runner.state.github_text.as_ref().unwrap();
17643 assert!(record.rewrite.is_none());
17644 let mut old = serde_json::to_value(record).unwrap();
17646 old.as_object_mut().unwrap().remove("rewrite");
17647 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17648 assert!(old.rewrite.is_none());
17649 let id = record.question.clone().unwrap();
17650 ask::Questions::open()
17651 .update(&id, |q| {
17652 q.choices.push(crate::github_text::USE_REWRITE.into());
17653 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17654 })
17655 .unwrap();
17656 let message = runner
17657 .guarded_pr_message(&winner, None, true)
17658 .await
17659 .unwrap()
17660 .unwrap();
17661 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17662 assert_eq!(
17663 runner.state.github_text.as_ref().unwrap().choice,
17664 Some(GithubTextChoice::Fallback)
17665 );
17666 }
17667
17668 #[tokio::test]
17669 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17670 crate::run::pin_test_home();
17671 let dir = tempfile::tempdir().unwrap();
17672 let mut runner = runner_at(RunStatus::Gating);
17673 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17674 runner.state.repo = dir.path().to_owned();
17675 runner.state.candidates[0].worktree = dir.path().to_owned();
17676 seed_public_summary(&mut runner);
17677 let winner = runner.state.candidates[0].clone();
17678 assert!(
17679 runner
17680 .guarded_pr_message(&winner, None, true)
17681 .await
17682 .unwrap()
17683 .is_none()
17684 );
17685 let store = ask::Questions::open();
17686 let find = |run: &str| {
17687 store
17688 .list()
17689 .into_iter()
17690 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17691 .unwrap()
17692 };
17693 let run = runner.state.id.clone();
17694 let q = find(&run);
17695 let fp = runner
17696 .state
17697 .github_text
17698 .as_ref()
17699 .unwrap()
17700 .fingerprint
17701 .clone();
17702 let name = crate::github_text::artifact_name(&fp);
17703 let path = crate::run::artifact_path(&runner.state, &name);
17704 assert!(q.detail.contains(&path.display().to_string()));
17705 assert!(q.detail.contains("title-language: title"));
17706 let saved = std::fs::read_to_string(&path).expect("artifact written");
17707 assert!(saved.contains("日本語のタイトル"));
17708 assert!(q.cwd.is_none());
17709 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17710 let say = |id: &str, text: &str, choice: &str| {
17712 store
17713 .update(id, |q| {
17714 q.thread.push(ask::Turn {
17715 who: ask::Who::Operator,
17716 body: text.to_owned(),
17717 at: jiff::Timestamp::now(),
17718 note: None,
17719 });
17720 q.answer(ask::Answer::Choice(choice.to_owned()))
17721 })
17722 .unwrap();
17723 };
17724 say(&q.id, "まだ日本語", "use my text");
17725 assert!(
17726 runner
17727 .guarded_pr_message(&winner, None, true)
17728 .await
17729 .unwrap()
17730 .is_none()
17731 );
17732 let second = find(&run);
17733 assert_ne!(second.id, q.id);
17734 assert!(second.detail.contains("did not pass"));
17735 assert!(second.detail.contains("日本語のタイトル"));
17738 assert!(!second.detail.contains("まだ日本語"));
17739 assert!(second.detail.contains(&path.display().to_string()));
17740 say(&second.id, "fix: retry failed requests", "use my text");
17741 let message = runner
17742 .guarded_pr_message(&winner, None, true)
17743 .await
17744 .unwrap()
17745 .expect("a vetted replacement is posted");
17746 assert_eq!(message.title, "fix: retry failed requests");
17747 assert_eq!(
17748 runner
17749 .state
17750 .github_text
17751 .as_ref()
17752 .unwrap()
17753 .chosen_title
17754 .as_deref(),
17755 Some("fix: retry failed requests")
17756 );
17757 assert!(runner.state.github_text.as_ref().unwrap().resolved);
17758 let again = runner
17760 .guarded_pr_message(&winner, None, true)
17761 .await
17762 .unwrap()
17763 .unwrap();
17764 assert_eq!(again.title, message.title);
17765 assert_eq!(
17766 store
17767 .list()
17768 .into_iter()
17769 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17770 .count(),
17771 2
17772 );
17773 }
17774
17775 #[test]
17776 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17777 let mut state = state_with_summary(
17781 "add retries",
17782 "TITLE: fix(web): batch reads\n- reads run.json once",
17783 );
17784 state.config.graph.language = "ja".to_owned();
17785 let m = pr_message(&state, 'A');
17786 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17787 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17788
17789 let task = "今回やってほしいこと: results projector を直す";
17792 let mut state = state_with_summary(task, "- no title line");
17793 state.config.graph.language = "ja".to_owned();
17794 let m = pr_message(&state, 'A');
17795 assert_eq!(
17796 m.title,
17797 format!("chore: land candidate A of run {}", state.id)
17798 );
17799 assert!(
17800 m.body.contains(&format!(
17801 "<summary>Original task</summary>\n\n{task}\n\n</details>"
17802 )),
17803 "{}",
17804 m.body
17805 );
17806 }
17807
17808 #[test]
17809 fn pr_message_scrubs_home_paths_and_addresses() {
17810 let state = state_with_summary(
17811 "fix it in /Users/someone/src/x",
17812 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17813 );
17814 let m = pr_message(&state, 'A');
17815 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17816 assert!(!m.body.contains(leak), "{}", m.body);
17817 }
17818 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17819 }
17820
17821 #[test]
17822 fn pr_message_survives_a_task_that_closes_details() {
17823 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17824 let m = pr_message(&state, 'A');
17825 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17826 }
17827
17828 #[test]
17829 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17830 let cmd = manual_merge_command(
17831 MergeStyle::Squash,
17832 Path::new("/repo"),
17833 "b",
17834 "fix: \"quoted\" $(x) `y`\n\nbody",
17835 );
17836 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17837 }
17838
17839 #[test]
17840 fn manual_merge_command_matches_the_configured_style() {
17841 let repo = Path::new("/repo");
17842 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17843
17844 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17845 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17846
17847 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17848 assert_eq!(
17849 squash,
17850 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17851 \"Merge magi run 0832 (candidate A)\""
17852 );
17853
17854 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17855 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17856 }
17857
17858 #[test]
17859 fn a_nudge_gets_a_quarter_of_the_budget() {
17860 assert_eq!(retry_budget(secs(1200), true), secs(300));
17862 assert_eq!(retry_budget(secs(3600), true), secs(900));
17863 }
17864
17865 #[test]
17866 fn a_resent_prompt_keeps_the_whole_budget() {
17867 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17870 assert_eq!(retry_budget(secs(60), false), secs(60));
17871 }
17872
17873 #[test]
17874 fn the_floor_never_exceeds_the_original_budget() {
17875 assert_eq!(retry_budget(secs(60), true), secs(60));
17879 assert_eq!(retry_budget(secs(480), true), secs(120));
17880 assert_eq!(retry_budget(secs(0), true), secs(0));
17881 }
17882
17883 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17884 agent::CommandEvidence {
17885 id: "item1".to_owned(),
17886 description: "cargo test".to_owned(),
17887 exit_code,
17888 result_summary: String::new(),
17889 source: "codex".to_owned(),
17890 }
17891 }
17892
17893 #[test]
17894 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17895 assert!(!has_unconfirmed_command(&[]));
17899 }
17900
17901 #[test]
17902 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17903 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17907 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17908 assert!(!has_unconfirmed_command(&[
17909 evidence(Some(0)),
17910 evidence(Some(101))
17911 ]));
17912 }
17913
17914 #[test]
17915 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17916 assert!(has_unconfirmed_command(&[
17917 evidence(Some(0)),
17918 evidence(None)
17919 ]));
17920 }
17921
17922 #[test]
17923 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17924 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17925 assert_eq!(
17926 verified_noop_claim(true, &[], text).as_deref(),
17927 Some("already fixed by b32cfc4, on main.")
17928 );
17929 }
17930
17931 #[test]
17932 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17933 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17936 assert!(verified_noop_claim(false, &[], text).is_none());
17937 }
17938
17939 #[test]
17940 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17941 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17942 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17943 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17945 }
17946
17947 #[test]
17948 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17949 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17950 }
17951
17952 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17955 runner.state.candidates = shape
17956 .iter()
17957 .enumerate()
17958 .map(|(i, &(empty, verified))| Candidate {
17959 index: i,
17960 label: (b'A' + i as u8) as char,
17961 agent: "sonnet".to_owned(),
17962 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17963 worktree: PathBuf::from(format!("/wt/{i}")),
17964 summary: String::new(),
17965 stat: String::new(),
17966 files: 0,
17967 commits: 0,
17968 empty,
17969 failed: None,
17970 verified_noop: verified.map(str::to_owned),
17971 duration_ms: 0,
17972 folded: false,
17973 })
17974 .collect();
17975 }
17976
17977 #[test]
17978 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17979 ask_test_home();
17980 let mut runner = runner_at(RunStatus::Implementing);
17981 set_candidates(
17982 &mut runner,
17983 &[
17984 (true, Some("already on main at b32cfc4")),
17985 (true, Some("same fix, see the existing test")),
17986 ],
17987 );
17988
17989 runner
17990 .after_implement()
17991 .expect("a verified no-op is not an error");
17992
17993 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17994 }
17995
17996 #[test]
17997 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17998 ask_test_home();
17999 let mut runner = runner_at(RunStatus::Implementing);
18000 set_candidates(
18004 &mut runner,
18005 &[(true, Some("already on main at b32cfc4")), (true, None)],
18006 );
18007
18008 let err = runner
18009 .after_implement()
18010 .expect_err("an unverified empty candidate must still fail the run");
18011
18012 assert!(
18013 err.to_string().contains("no candidate produced a change"),
18014 "{err}"
18015 );
18016 assert_eq!(runner.state.status, RunStatus::Failed);
18017 }
18018
18019 #[test]
18020 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18021 ask_test_home();
18022 let mut runner = runner_at(RunStatus::Implementing);
18023 set_candidates(&mut runner, &[(true, None), (true, None)]);
18024
18025 let err = runner
18026 .after_implement()
18027 .expect_err("no candidate declared anything; this is an ordinary failure");
18028
18029 assert!(
18030 err.to_string().contains("no candidate produced a change"),
18031 "{err}"
18032 );
18033 assert_eq!(runner.state.status, RunStatus::Failed);
18034 }
18035}
18036
18037#[derive(Debug)]
18040struct GraftRestoreFailed(String);
18041
18042impl std::fmt::Display for GraftRestoreFailed {
18043 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18044 write!(
18045 f,
18046 "could not restore the winner to its pre-graft commit: {}",
18047 self.0
18048 )
18049 }
18050}
18051
18052impl std::error::Error for GraftRestoreFailed {}
18053
18054#[cfg(test)]
18058mod resource_wait_tests {
18059 use super::*;
18060 use std::collections::BTreeMap;
18061
18062 fn sh(id: &str, script: &str) -> AgentSpec {
18063 AgentSpec {
18064 id: id.to_owned(),
18065 kind: crate::config::AgentKind::Command,
18066 model: None,
18067 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18068 extra_args: Vec::new(),
18069 env: BTreeMap::new(),
18070 prompt_delivery: None,
18071 }
18072 }
18073
18074 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18075 SeatJob {
18076 seat: SeatState::new("impl-A", &spec.id, 1),
18077 spec,
18078 cwd: dir.to_path_buf(),
18079 prompt: "do it".to_owned(),
18080 timeout,
18081 allow_write: true,
18082 sessions: false,
18083 artifacts: dir.join("art"),
18084 stem: "impl-A".to_owned(),
18085 handover: None,
18086 }
18087 }
18088
18089 fn state(resource_wait_secs: u64) -> RunState {
18090 let mut config = Config::default();
18091 config.graph.timeout_resource_wait = resource_wait_secs;
18092 RunState::new(
18093 PathBuf::from("/repo"),
18094 "main".to_owned(),
18095 "abc1234".to_owned(),
18096 "task".to_owned(),
18097 config,
18098 )
18099 }
18100
18101 fn hold(cache: &Path) -> crate::cache::Guard {
18102 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18103 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18104 crate::cache::AcquireOutcome::Acquired(g) => g,
18105 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18106 }
18107 }
18108
18109 async fn run(
18110 jobs: Vec<SeatJob>,
18111 sem: Arc<Semaphore>,
18112 cache: Option<&Path>,
18113 st: &mut RunState,
18114 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18115 let prompts = Prompts::default();
18116 let ctx = WaveCtx {
18117 run: "r",
18118 node: "implement",
18119 prompts: &prompts,
18120 cache,
18121 round: None,
18122 carry_seats: false,
18123 pauses: None,
18124 };
18125 wave(jobs, sem, &ctx, st, 0).await
18126 }
18127
18128 #[tokio::test]
18129 async fn verification_cache_wait_preserves_the_command_budget() {
18130 crate::run::pin_test_home();
18131 let tmp = tempfile::tempdir().unwrap();
18132 let cache = tmp.path().join("target");
18133 let guard = hold(&cache);
18134 let release = tokio::spawn(async move {
18135 tokio::time::sleep(Duration::from_millis(100)).await;
18136 guard.release();
18137 });
18138 let mut st = state(10);
18139 let budget = Duration::from_millis(50);
18140 let wait = repository_wait(&st, None);
18141 let (outcomes, _) = with_cache_lease(
18142 &mut st,
18143 Some(&cache),
18144 "gate",
18145 "gate",
18146 tmp.path(),
18147 "abc1234",
18148 budget,
18149 "test gate",
18150 &wait,
18151 |_, available| async move {
18152 assert_eq!(
18153 available, budget,
18154 "contention must not shorten verification"
18155 );
18156 (Vec::new(), false, Vec::new())
18157 },
18158 )
18159 .await
18160 .unwrap();
18161 release.await.unwrap();
18162 assert!(outcomes.is_empty(), "{outcomes:?}");
18163 assert!(
18164 st.events
18165 .iter()
18166 .any(|event| event.message.contains("waiting for"))
18167 );
18168 assert!(st.resource_wait.is_none());
18169 hold(&cache).release();
18171 }
18172
18173 #[tokio::test]
18174 async fn verification_cache_wait_expires_without_starting_commands() {
18175 crate::run::pin_test_home();
18176 let tmp = tempfile::tempdir().unwrap();
18177 let cache = tmp.path().join("target");
18178 let _held = hold(&cache);
18179 let mut st = state(0);
18180 let wait = repository_wait(&st, None);
18181 let (outcomes, _) = with_cache_lease(
18182 &mut st,
18183 Some(&cache),
18184 "gate",
18185 "gate",
18186 tmp.path(),
18187 "abc1234",
18188 Duration::from_secs(60),
18189 "test gate",
18190 &wait,
18191 |_, _| async { panic!("commands must not start without the cache lease") },
18192 )
18193 .await
18194 .unwrap();
18195 assert_eq!(outcomes.len(), 1);
18196 assert!(outcomes[0].resource_blocked);
18197 assert!(st.resource_wait.is_none());
18198 }
18199
18200 #[tokio::test]
18201 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18202 crate::run::pin_test_home();
18203 let tmp = tempfile::tempdir().unwrap();
18204 let cache = tmp.path().join("target-a");
18205 let guard = hold(&cache);
18206 tokio::spawn(async move {
18208 tokio::time::sleep(Duration::from_millis(1500)).await;
18209 guard.release();
18210 });
18211 let mut st = state(60);
18212 let results = run(
18213 vec![job(
18214 sh("alpha", "echo ok"),
18215 tmp.path(),
18216 Duration::from_secs(1),
18217 )],
18218 Arc::new(Semaphore::new(1)),
18219 Some(&cache),
18220 &mut st,
18221 )
18222 .await;
18223 assert!(
18224 matches!(results[0].2, AgentOutcome::Ok(_)),
18225 "the wait must not be charged to the agent"
18226 );
18227 assert!(st.resource_block.is_none());
18228 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18229 assert!(st.resource_wait.is_none() && st.active.is_empty());
18230 }
18231
18232 #[tokio::test]
18233 #[cfg(unix)]
18234 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18235 use std::os::fd::AsRawFd as _;
18236 crate::run::pin_test_home();
18237 let tmp = tempfile::tempdir().unwrap();
18238 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18239 let common = tmp.path().join(".git");
18240 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18241 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18242 assert_eq!(
18243 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18244 0
18245 );
18246 let mut st = state(1);
18247 st.config.coordination = crate::config::test_coordination();
18248 let mut reader = job(
18249 sh("reader", "echo must-not-run"),
18250 tmp.path(),
18251 Duration::from_secs(30),
18252 );
18253 reader.allow_write = false;
18254 let results = run(
18255 vec![reader.clone()],
18256 Arc::new(Semaphore::new(1)),
18257 None,
18258 &mut st,
18259 )
18260 .await;
18261 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18262 assert_eq!(results[0].1.turns, 0);
18263 assert!(FailClass::of(&results[0].2).is_none());
18264 assert_eq!(
18265 st.resource_block.as_ref().unwrap().kind,
18266 "repository coordination"
18267 );
18268 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18269 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18270 drop(external);
18271 let editor = crate::repo_lock::SeatLease::for_checkout(
18272 tmp.path(),
18273 true,
18274 crate::config::test_coordination().as_ref(),
18275 )
18276 .await
18277 .unwrap();
18278 let pause = Pause::new();
18279 let interrupt = Pause::new();
18280 let prompts = Prompts::default();
18281 let ctx = WaveCtx {
18282 run: "r",
18283 node: "review",
18284 prompts: &prompts,
18285 cache: None,
18286 round: None,
18287 carry_seats: false,
18288 pauses: Some((&pause, &interrupt)),
18289 };
18290 let (results, ()) = tokio::join!(
18291 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18292 async {
18293 tokio::time::sleep(Duration::from_millis(50)).await;
18294 pause.park();
18295 }
18296 );
18297 assert!(
18298 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18299 );
18300 assert_eq!(results[0].1.turns, 0);
18301 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18302 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18303 drop(editor);
18304 }
18305
18306 #[tokio::test]
18307 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18308 crate::run::pin_test_home();
18309 let tmp = tempfile::tempdir().unwrap();
18310 let cache = tmp.path().join("target-b");
18311 let _held = hold(&cache);
18312 let mut st = state(1);
18313 let results = run(
18314 vec![job(
18315 sh("alpha", "echo ok"),
18316 tmp.path(),
18317 Duration::from_secs(1),
18318 )],
18319 Arc::new(Semaphore::new(1)),
18320 Some(&cache),
18321 &mut st,
18322 )
18323 .await;
18324 let (_, seat, out) = &results[0];
18325 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18326 assert!(wave_blocked(&results));
18327 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18328 assert_eq!(seat.turns, 0, "nothing was asked");
18329 assert!(
18330 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18331 "no agent was spawned"
18332 );
18333 assert!(FailClass::of(out).is_none(), "not a failure class");
18334 let block = st.resource_block.as_ref().expect("recorded for the resume");
18335 assert_eq!(block.kind, "shared build cache");
18336 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18337 assert!(st.active.is_empty());
18338 }
18339
18340 #[tokio::test]
18341 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18342 crate::run::pin_test_home();
18343 let tmp = tempfile::tempdir().unwrap();
18344 let mut st = state(1);
18345 let results = run(
18346 vec![job(
18347 sh("alpha", "echo ok"),
18348 tmp.path(),
18349 Duration::from_secs(1),
18350 )],
18351 Arc::new(Semaphore::new(0)),
18352 None,
18353 &mut st,
18354 )
18355 .await;
18356 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18357 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18358 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18359 }
18360
18361 #[tokio::test]
18362 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18363 crate::run::pin_test_home();
18364 let tmp = tempfile::tempdir().unwrap();
18365 let mut st = state(60);
18366 let results = run(
18367 vec![job(
18368 sh("alpha", "sleep 30"),
18369 tmp.path(),
18370 Duration::from_secs(1),
18371 )],
18372 Arc::new(Semaphore::new(1)),
18373 None,
18374 &mut st,
18375 )
18376 .await;
18377 let out = &results[0].2;
18378 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18379 let class = FailClass::of(out).expect("a failure");
18380 assert_eq!(class, FailClass::Timeout);
18381 assert!(should_hand_over(None, &class), "existing handover policy");
18382 assert!(st.resource_block.is_none());
18383 }
18384}