1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
488 let tracking = format!("{remote}/{base_branch}");
489 let fetched = git::fetch(repo, remote, base_branch).await;
490 if let Ok(out) = &fetched
491 && out.ok()
492 && git::rev_exists(repo, &tracking).await
493 {
494 return git::rev_parse(repo, &tracking).await;
495 }
496 let why = match &fetched {
497 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
498 Ok(_) => format!("{remote} has no {base_branch}"),
499 Err(e) => e.to_string(),
500 };
501 bail!(
504 "cannot read {tracking} ({why}); refusing to branch off the local \
505 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
506 base / remote in magi.toml"
507 )
508}
509
510struct FixClaim {
527 path: PathBuf,
528}
529
530impl FixClaim {
531 fn acquire(dir: &Path) -> Result<Self> {
532 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
533 let path = dir.join("fix.lock");
534 match Self::create(&path) {
535 Ok(claim) => Ok(claim),
536 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
537 if Self::reclaim_if_dead(&path) {
538 Self::create(&path).with_context(|| format!("lock {}", path.display()))
539 } else {
540 bail!(
541 "another repair or `magi fix` is already running for this run ({} exists)",
542 path.display()
543 )
544 }
545 }
546 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
547 }
548 }
549
550 fn create(path: &Path) -> std::io::Result<Self> {
551 let mut f = std::fs::OpenOptions::new()
552 .write(true)
553 .create_new(true)
554 .open(path)?;
555 use std::io::Write as _;
556 writeln!(f, "{}", std::process::id())?;
558 Ok(Self {
559 path: path.to_owned(),
560 })
561 }
562
563 fn reclaim_if_dead(path: &Path) -> bool {
567 let dead = std::fs::read_to_string(path)
568 .ok()
569 .and_then(|body| body.trim().parse::<u32>().ok())
570 .is_some_and(|pid| !crate::proc::pid_alive(pid));
571 dead && std::fs::remove_file(path).is_ok()
572 }
573}
574
575impl Drop for FixClaim {
576 fn drop(&mut self) {
577 let _ = std::fs::remove_file(&self.path);
578 }
579}
580
581struct DriverExecution<'a> {
584 runner: &'a mut Runner,
585 driver: crate::driver::DriverGuard,
586}
587
588impl Drop for DriverExecution<'_> {
589 fn drop(&mut self) {
590 self.driver.release_local(&mut self.runner.state);
591 }
592}
593
594impl Runner {
595 pub async fn start(
597 repo: &Path,
598 instruction: String,
599 config: Config,
600 origin: Origin,
601 ) -> Result<Self> {
602 Self::start_naming(repo, instruction, "", config, origin).await
603 }
604
605 pub async fn start_naming(
609 repo: &Path,
610 instruction: String,
611 also_scan: &str,
612 config: Config,
613 origin: Origin,
614 ) -> Result<Self> {
615 let repo = git::toplevel(repo).await?;
616 let missing = agent::missing_programs(&config.agents);
617 if !missing.is_empty() {
618 bail!(
619 "these agent programs are not on PATH: {}. Fix the roster in \
620 magi.toml or install them.",
621 missing.join(", ")
622 );
623 }
624 let base_branch =
625 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
626 .await?;
627 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
628 let roles = config.resolve_roles()?;
629 let max_parallel = config.graph.max_parallel.max(1);
630 let seeds = refs::resolve(
633 &repo,
634 &base_commit,
635 &config.merge.remote,
636 &format!("{also_scan}\n{instruction}"),
637 )
638 .await;
639 refs::plan(&repo, &seeds).await?;
640 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
641 state.origin = Some(origin);
645 for seed in &seeds {
646 state.event(
647 "seed",
648 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
649 );
650 }
651 state.seeds = seeds;
652 state.event("start", format!("run {} created", state.id));
653 state.save()?;
654 Ok(Self {
655 state,
656 roles,
657 sem: Arc::new(Semaphore::new(max_parallel)),
658 pause: Pause::new(),
659 interrupt: Pause::new(),
660 })
661 }
662
663 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
677 Self::review_taking_over(repo, branch, config, None, origin).await
678 }
679
680 pub async fn review_taking_over(
687 repo: &Path,
688 branch: &str,
689 config: Config,
690 takeover: Option<crate::handover::Takeover>,
691 origin: Origin,
692 ) -> Result<Self> {
693 let repo = git::toplevel(repo).await?;
694 let missing = agent::missing_programs(&config.agents);
695 if !missing.is_empty() {
696 bail!(
697 "these agent programs are not on PATH: {}. Fix the roster in \
698 magi.toml or install them.",
699 missing.join(", ")
700 );
701 }
702 let base_branch =
703 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
704 .await?;
705 if base_branch == branch {
706 bail!("`{branch}` is the base branch; there is nothing to review against");
707 }
708 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
709
710 let roles = config.resolve_roles()?;
711 let max_parallel = config.graph.max_parallel.max(1);
712 let mut state = RunState::new(
713 repo.clone(),
714 base_branch,
715 base_commit.clone(),
716 String::new(),
717 config,
718 );
719 state.origin = Some(origin);
720
721 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
728 earlier: Vec::new(),
729 home: crate::run::home(),
730 choice: None,
731 task_instruction: None,
732 });
733 let wait = repository_wait(&state, None);
734 let coordination_config = state.config.coordination.clone();
735 let coordination =
736 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
737 .await?;
738 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
739 if let Some(released) = &released {
740 state.event(
741 "release",
742 format!(
743 "took `{branch}` over from run {}: its worktree was released: {}",
744 crate::run::short_of(&released.old_id),
745 released.audit
746 ),
747 );
748 }
749 if let Some(choice) = takeover.choice.as_ref()
753 && let Err(e) =
754 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
755 .await
756 {
757 if let Some(released) = &released {
758 released.restore(&repo, branch).await;
759 }
760 return Err(e.context("applying the owner's answer about the diverged branch"));
761 }
762 for id in takeover.earlier.iter().rev() {
768 if let Ok(old) = RunState::load_under(id, &takeover.home)
769 && let Some(head) = old.pr_head.clone()
770 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
771 {
772 state.pr_head = Some(head);
773 break;
774 }
775 }
776 drop(coordination);
777 let opened = Self::open_review(
778 &repo,
779 branch,
780 state,
781 roles,
782 max_parallel,
783 base_commit,
784 &takeover,
785 )
786 .await;
787 if opened.is_err()
788 && let Some(released) = &released
789 {
790 let _coordination = crate::repo_lock::SeatLease::operation_with(
791 &repo,
792 coordination_config.as_ref(),
793 &wait,
794 )
795 .await?;
796 released.restore(&repo, branch).await;
797 }
798 opened
799 }
800
801 pub async fn recover_pr(
803 repo: &Path,
804 task: &str,
805 selection: crate::pr_recovery::Selection,
806 config: Config,
807 ) -> Result<Self> {
808 selection.check_task(task, &selection.instruction)?;
809 let base_branch =
810 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
811 .await?;
812 let base_commit = resolve_base(repo, &base_branch, &config.merge.remote).await?;
813 selection
814 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
815 .await?;
816 let roles = config.resolve_roles()?;
817 let max_parallel = config.graph.max_parallel.max(1);
818 let mut state = RunState::new(
819 repo.to_path_buf(),
820 base_branch,
821 base_commit.clone(),
822 selection.instruction.clone(),
823 config,
824 );
825 state.origin = Some(crate::run::Origin::queue(task));
826 state.pr_head = Some(selection.branch.clone());
827 state.pr_recovery = Some(selection.clone());
828 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
829 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
830 }
831
832 async fn open_selected(
833 repo: &Path,
834 state: RunState,
835 roles: ResolvedRoles,
836 max_parallel: usize,
837 base_commit: String,
838 selection: crate::pr_recovery::Selection,
839 ) -> Result<Self> {
840 let mut state = state;
841 let branch = state.branch_for('A');
842 let wait = repository_wait(&state, None);
843 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
844 git::git(repo, &["branch", &branch, &selection.head]).await?;
845 drop(coordination);
846 state.event(
847 "recovery",
848 format!(
849 "selected public implementation {} from {}",
850 selection.head, selection.url
851 ),
852 );
853 let takeover = crate::handover::Takeover {
854 earlier: Vec::new(),
855 home: crate::run::home(),
856 choice: None,
857 task_instruction: Some(selection.instruction.clone()),
858 };
859 state.save()?;
860 let id = state.id.clone();
861 let opened = Self::open_review(
862 repo,
863 &branch,
864 state,
865 roles,
866 max_parallel,
867 base_commit,
868 &takeover,
869 )
870 .await;
871 match opened {
872 Ok(runner) => Ok(runner),
873 Err(e) => {
874 if let Ok(mut partial) = RunState::load(&id) {
875 partial.status = RunStatus::Blocked;
876 partial.driver_exited = true;
877 partial.event(
878 "recovery",
879 format!("startup failed; retained source and candidate {branch}: {e:#}"),
880 );
881 partial.save()?;
882 }
883 Err(crate::pr_recovery::StartupFailed {
884 run: id,
885 cause: format!("{e:#}"),
886 }
887 .into())
888 }
889 }
890 }
891
892 async fn inherit(
897 repo: &Path,
898 branch: &str,
899 base_commit: &str,
900 takeover: &crate::handover::Takeover,
901 ) -> Option<crate::recovery::RecoveryRecord> {
902 use crate::recovery::{Relation, assess};
903 let newest = takeover.earlier.last()?;
904 let mut chosen = None;
905 for id in takeover.earlier.iter().rev() {
906 if let Ok(old) = RunState::load_under(id, &takeover.home)
907 && !old.reviews.is_empty()
908 && (old.winner().is_some_and(|w| w.branch == branch)
909 || old.pr_head.as_deref() == Some(branch))
910 {
911 chosen = Some(old);
912 break;
913 }
914 }
915 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
916 .await
917 .unwrap_or_default();
918 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
919 Some(last) if last.head == tip => Relation::Same,
920 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
921 _ => Relation::Diverged,
922 };
923 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
924 Some(assess(from, chosen.as_ref(), base_commit, relation))
925 }
926
927 async fn open_review(
930 repo: &Path,
931 branch: &str,
932 mut state: RunState,
933 roles: ResolvedRoles,
934 max_parallel: usize,
935 base_commit: String,
936 takeover: &crate::handover::Takeover,
937 ) -> Result<Self> {
938 let wait = repository_wait(&state, None);
939 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
940 if state.pr_recovery.is_none() {
941 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
942 }
943 let start = review_base(
946 repo,
947 &state.config.merge.remote,
948 &state.base_branch,
949 &base_commit,
950 branch,
951 )
952 .await;
953 let log = git::log_oneline(repo, &start, branch)
954 .await
955 .unwrap_or_default();
956 let log_text = if log.trim().is_empty() {
957 "(no commit messages)"
958 } else {
959 log.trim()
960 };
961 let mut instruction = format!(
962 "Review the work already on branch `{branch}`. There is no task \
963 statement: what the change claims to do is whatever its commits \
964 say.\n\n{log_text}"
965 );
966 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
970 if let Some(task) = takeover
971 .task_instruction
972 .as_deref()
973 .filter(|t| !t.trim().is_empty())
974 && rec.reviewed_head.is_some()
975 {
976 instruction = format!(
977 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
978 task.trim()
979 );
980 rec.criteria_inherited = true;
981 }
982 state.event("recovery", rec.summary());
983 state.recovery = Some(rec);
984 }
985 if let Some(selected) = &state.pr_recovery {
986 instruction = selected.instruction.clone();
987 }
988 state.task_instruction = takeover.task_instruction.clone();
989 state.instruction = instruction;
990 state.reviewed_commits = Some(
991 git::subjects(repo, &start, branch)
992 .await
993 .unwrap_or_default(),
994 );
995
996 let worktree = state.worktree_root().join("under-review");
999 if let Some(parent) = worktree.parent() {
1000 tokio::fs::create_dir_all(parent).await.ok();
1001 }
1002 let path = worktree.to_string_lossy().to_string();
1003 git::git(repo, &["worktree", "add", &path, branch])
1004 .await
1005 .with_context(|| {
1006 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1007 })?;
1008
1009 let setup = match ensure_setup_config(&mut state, repo).await {
1010 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1011 Err(e) => Err(e),
1012 };
1013 if let Err(e) = setup {
1014 git::worktree_remove(repo, &worktree).await.ok();
1015 return Err(e);
1016 }
1017 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1018 .await
1019 .unwrap_or(0);
1020 if commits == 0 {
1021 git::worktree_remove(repo, &worktree).await.ok();
1022 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1023 }
1024 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1025 .await
1026 .map(|f| f.len())
1027 .unwrap_or(0);
1028 if files == 0
1029 && let (Ok(head_tree), Ok(base_tree)) = (
1030 git::tree_of(&worktree, "HEAD").await,
1031 git::tree_of(&worktree, &base_commit).await,
1032 )
1033 && head_tree == base_tree
1034 {
1035 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1036 git::worktree_remove(repo, &worktree).await.ok();
1037 bail!(
1038 "`{branch}` at {} has a tree identical to base {}; this usually means \
1039 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1040 against `{}/{branch}`) rather than an empty change",
1041 short(&head),
1042 short(&base_commit),
1043 state.config.merge.remote
1044 );
1045 }
1046 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1047 .await
1048 .unwrap_or_default();
1049
1050 state.candidates.push(Candidate {
1051 index: 0,
1052 label: 'A',
1053 agent: EXISTING_BRANCH.to_owned(),
1056 branch: branch.to_owned(),
1057 worktree,
1058 summary: String::new(),
1059 stat,
1060 files,
1061 commits,
1062 empty: false,
1063 failed: None,
1064 verified_noop: None,
1065 duration_ms: 0,
1066 folded: false,
1067 });
1068 state.tally = Some(Tally {
1069 first_choice: BTreeMap::from([('A', 0)]),
1070 borda: BTreeMap::new(),
1071 winner: 'A',
1072 rankings: 0,
1073 unanimous_initial: false,
1074 deliberated: false,
1075 changed_votes: 0,
1076 unanimous_final: false,
1077 tie_break: None,
1078 judges: 0,
1082 present: 0,
1083 quorum: 0,
1084 met_quorum: true,
1085 uncontested: Some("review-only run: nothing competed".to_owned()),
1086 });
1087 state.status = RunStatus::Reviewing;
1088 state.prep_complete = Some(true);
1089 state.event(
1090 "start",
1091 format!(
1092 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1093 state.id
1094 ),
1095 );
1096 state.save()?;
1097 Ok(Self {
1098 state,
1099 roles,
1100 sem: Arc::new(Semaphore::new(max_parallel)),
1101 pause: Pause::new(),
1102 interrupt: Pause::new(),
1103 })
1104 }
1105
1106 pub fn resume(id: &str) -> Result<Self> {
1108 let state = RunState::load(id)?;
1109 crate::pr_head::ensure_owner(&state)?;
1110 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1111 bail!(
1112 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1113 );
1114 }
1115 if let Some(to) = &state.released_to {
1116 bail!(
1117 "run {} cannot be resumed: its worktree was released to run {}",
1118 state.short(),
1119 crate::run::short_of(to)
1120 );
1121 }
1122 let roles = state.config.resolve_roles()?;
1123 if let Some(id) = &state.config.graph.selected_implementer
1124 && state.candidates.iter().any(|c| c.agent != *id)
1125 {
1126 bail!(
1127 "replacement implementer pin does not match the saved candidates; task must stay held"
1128 );
1129 }
1130 let max_parallel = state.config.graph.max_parallel.max(1);
1131 Ok(Self {
1132 state,
1133 roles,
1134 sem: Arc::new(Semaphore::new(max_parallel)),
1135 pause: Pause::new(),
1136 interrupt: Pause::new(),
1137 })
1138 }
1139
1140 pub async fn execute(&mut self) -> Result<()> {
1147 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1150 || self.state.land_approval.is_some()
1151 || self.state.merge_decision.is_some()
1152 || !self.state.merge_repairs.is_empty()
1153 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1154 {
1155 let claim = FixClaim::acquire(&self.state.dir())?;
1156 let disk = RunState::load(&self.state.id)?;
1157 let checkpoint = |s: &RunState| {
1158 (
1159 s.merge_repairs.len(),
1160 s.merge_repairs
1161 .last()
1162 .map(|r| (r.question.clone(), r.phase)),
1163 s.land_approval
1164 .as_ref()
1165 .map(|a| (a.question.clone(), a.head.clone())),
1166 )
1167 };
1168 if checkpoint(&disk) != checkpoint(&self.state) {
1169 bail!(
1170 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1171 );
1172 }
1173 Some(claim)
1174 } else {
1175 None
1176 };
1177 let mut execution = self.claim_driver()?;
1178 let runner = &mut *execution.runner;
1179 let wait = runner.repository_wait();
1180 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1183 let result = match result {
1184 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1185 record_repository_block(&mut runner.state, &e);
1186 runner.park_here()?;
1187 Ok(())
1188 }
1189 other => other,
1190 };
1191 execution.driver.finish(&mut runner.state)?;
1192 let ended = if result.is_err() {
1193 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1194 } else {
1195 crate::notices::run_ended(&runner.state)
1196 };
1197 if let Some(notice) = ended {
1198 crate::notices::raise(notice);
1199 }
1200 result
1201 }
1202
1203 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1204 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1205 }
1206
1207 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1208 let wait = self.repository_wait();
1209 coordinate_operation(&mut self.state, cwd, &wait).await
1210 }
1211
1212 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1214 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1215 let disk = RunState::load(&self.state.id)?;
1216 crate::pr_head::ensure_owner(&disk)?;
1217 if disk.liveness(false) == crate::run::Liveness::Live {
1218 bail!("run already has a live driver; wait for its execution to end");
1219 }
1220 if let Some(to) = &disk.released_to {
1221 bail!(
1222 "run {} cannot continue: its worktree was released to run {}",
1223 self.state.short(),
1224 crate::run::short_of(to)
1225 );
1226 }
1227 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1228 let mut execution = DriverExecution {
1229 runner: self,
1230 driver,
1231 };
1232 let _owner = owner;
1235 let state = &mut execution.runner.state;
1236 state.parked = false;
1241 state.resource_wait = None;
1242 state.resource_block = None;
1243 state.clear_active();
1250 let pid = std::process::id();
1265 state.driver_pid = Some(pid);
1266 state.driver_started_at = crate::proc::process_started_at(pid);
1267 state.driver_exited = false;
1268 state.begin_pass();
1269 execution.driver.bind(state);
1270 state.save()?;
1271 execution.driver.record(state)?;
1272 Ok(execution)
1273 }
1274
1275 async fn execute_graph(&mut self) -> Result<()> {
1276 if self
1277 .state
1278 .merge_decision
1279 .as_ref()
1280 .is_some_and(|decision| !decision.closed)
1281 && self.state.status != RunStatus::Landing
1282 {
1283 if crate::merge_decision::resumable_hold(&self.state) {
1284 self.state.merge = Some(MergeOutcome {
1285 mode: MergeMode::Pr,
1286 ok: false,
1287 detail: self
1288 .state
1289 .merge_decision
1290 .as_ref()
1291 .unwrap()
1292 .target
1293 .pr
1294 .clone(),
1295 empty: false,
1296 });
1297 self.run_land().await?;
1298 self.settle_questions();
1299 return Ok(());
1300 }
1301 bail!(
1302 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1303 );
1304 }
1305 if self.state.config.graph.selected_implementer.is_some()
1306 && self.state.status == RunStatus::Blocked
1307 && self.state.tally.is_none()
1308 {
1309 bail!(
1310 "selected implementation is held; explicitly release the owning task to retry it"
1311 );
1312 }
1313
1314 if self.state.merge_repairs.last().is_some_and(|r| {
1315 matches!(
1316 r.phase,
1317 crate::run::MergeRepairPhase::Requested
1318 | crate::run::MergeRepairPhase::Fixing
1319 | crate::run::MergeRepairPhase::Reviewing
1320 )
1321 }) {
1322 if self.repair_merge_review().await? {
1323 self.run_land().await?;
1324 }
1325 self.settle_questions();
1326 return Ok(());
1327 }
1328 if self.state.pr_publication_pending {
1331 self.merge().await?;
1332 if self.resource_halt()? {
1333 return Ok(());
1334 }
1335 self.state.save()?;
1336 self.settle_questions();
1337 return Ok(());
1338 }
1339 if self.state.status == RunStatus::Stalled {
1352 if self.recover_stall().await? {
1353 self.finish_after_tally().await?;
1354 } else {
1355 self.state.save()?;
1357 }
1358 return Ok(());
1359 }
1360 if self.state.status == RunStatus::Landing {
1370 self.run_land().await?;
1371 self.settle_questions();
1376 return Ok(());
1377 }
1378 if self.state.github_text.is_some()
1383 && self.state.merge.is_none()
1384 && self.state.status == RunStatus::Gating
1385 {
1386 self.merge().await?;
1387 if self.resource_halt()? {
1388 return Ok(());
1389 }
1390 self.state.save()?;
1391 return Ok(());
1392 }
1393 if let Err(e) = self.prep().await {
1394 if e.is::<crate::repo_lock::ResourceBlocked>() {
1395 return Err(e);
1396 }
1397 self.state.status = RunStatus::Blocked;
1398 self.state
1399 .prep_failure
1400 .get_or_insert_with(|| format!("{e:#}"));
1401 self.state
1402 .event("prep", format!("preparation stopped: {e:#}"));
1403 self.state.save()?;
1404 return Err(e);
1405 }
1406 if self.park_here()? {
1407 return Ok(());
1408 }
1409 self.advise().await?;
1410 if self.park_here()? {
1411 return Ok(());
1412 }
1413 self.implement().await?;
1414 if self.park_here()? {
1415 return Ok(());
1416 }
1417 if self.state.status == RunStatus::VerifiedNoop {
1421 return Ok(());
1422 }
1423 self.judge().await?;
1424 if self.park_here()? {
1425 return Ok(());
1426 }
1427 self.deliberate().await?;
1428 if self.park_here()? {
1429 return Ok(());
1430 }
1431 self.vote().await?;
1432 if self.park_here()? {
1433 return Ok(());
1434 }
1435 self.tally()?;
1436 if self.state.status == RunStatus::Stalled {
1441 self.state.save()?;
1445 return Ok(());
1446 }
1447 self.finish_after_tally().await?;
1448 Ok(())
1449 }
1450
1451 fn stop_for_resource(&mut self) -> Result<()> {
1456 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1458 {
1459 self.state.status = RunStatus::Blocked;
1460 }
1461 self.state.save()
1462 }
1463
1464 fn resource_halt(&mut self) -> Result<bool> {
1466 if self.state.resource_block.is_none() {
1467 return Ok(false);
1468 }
1469 self.stop_for_resource()?;
1470 Ok(true)
1471 }
1472
1473 fn park_here(&mut self) -> Result<bool> {
1480 let parking = self.pause.parked() || self.interrupt.parked();
1481 if !parking && self.state.resource_block.is_some() {
1482 self.stop_for_resource()?;
1483 return Ok(true);
1484 }
1485 if !self.pause.parked() && !self.interrupt.parked() {
1490 return Ok(false);
1491 }
1492 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1493 let why = match cause.as_ref().map(ParkCause::label) {
1494 Some(reason) => format!(
1495 "parked after `{}` ({reason}) — resume to carry on from here",
1496 self.state.status.as_str()
1497 ),
1498 None => format!(
1499 "parked after `{}` — resume to carry on from here",
1500 self.state.status.as_str()
1501 ),
1502 };
1503 self.state.event("park", why);
1504 self.state
1505 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1506 self.state.parked = true;
1507 self.state.save()?;
1508 Ok(true)
1509 }
1510
1511 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1513 if self.state.prep_failure.is_none() {
1514 return Ok(());
1515 }
1516 let failed_at = self
1517 .state
1518 .events
1519 .iter()
1520 .rev()
1521 .find(|event| event.node == "prep")
1522 .map_or(self.state.updated_at, |event| event.at);
1523 anyhow::ensure!(
1524 approval_at > failed_at,
1525 "preparation retry requires a new explicit release after the failure"
1526 );
1527 self.state.prep_failure = None;
1528 self.state.prep_complete = Some(false);
1529 self.state
1530 .event("prep-retry", "operator approved retry of preparation");
1531 Ok(())
1532 }
1533
1534 pub fn on_pause(&mut self, pause: Pause) {
1536 self.pause = pause;
1537 }
1538
1539 pub fn watch_interrupt(&mut self, pause: Pause) {
1545 self.interrupt = pause;
1546 }
1547
1548 fn settle_questions(&mut self) {
1568 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1569 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1570 }
1571 }
1572
1573 async fn finish_after_tally(&mut self) -> Result<()> {
1576 self.graft().await?;
1579 if self.resource_halt()? {
1580 return Ok(());
1581 }
1582 self.fold_losers().await?;
1583 self.sync_to_base().await?;
1588 if self.state.status == RunStatus::AlreadyInBase {
1589 return Ok(());
1590 }
1591 self.review_loop().await?;
1592 if self.resource_halt()? {
1593 return Ok(());
1594 }
1595 self.sync_to_base().await?;
1596 if self.state.status == RunStatus::AlreadyInBase {
1597 return Ok(());
1598 }
1599 self.gate().await?;
1600 self.merge().await?;
1601 if self.resource_halt()? {
1602 return Ok(());
1603 }
1604 self.state.save()?;
1605 Ok(())
1606 }
1607
1608 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1615 let result = match ensure_setup_config(&mut self.state, repo).await {
1616 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1617 Err(e) => Err(e),
1618 };
1619 if let Err(e) = result {
1620 git::worktree_remove(repo, wt).await.ok();
1621 if in_prep {
1622 self.rollback_prep(repo).await;
1623 }
1624 self.state.status = RunStatus::Blocked;
1625 self.state.event(
1626 "setup",
1627 format!("worktree setup failed in {}: {e:#}", wt.display()),
1628 );
1629 self.state.save()?;
1630 return Err(e);
1631 }
1632 Ok(())
1633 }
1634
1635 async fn rollback_prep(&mut self, repo: &Path) {
1637 let root = self.state.worktree_root();
1638 for c in std::mem::take(&mut self.state.candidates) {
1639 git::worktree_remove(repo, &c.worktree).await.ok();
1640 git::branch_delete(repo, &c.branch).await.ok();
1641 }
1642 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1643 for j in 1..=seats {
1644 git::worktree_remove(repo, &root.join(format!("judge-{j}")))
1645 .await
1646 .ok();
1647 }
1648 for k in 1..=self.state.config.graph.advisors {
1649 git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
1650 .await
1651 .ok();
1652 }
1653 }
1654
1655 async fn prep(&mut self) -> Result<()> {
1656 if let Some(reason) = &self.state.prep_failure {
1657 bail!(
1658 "candidate preparation previously failed: {reason}; correct the source before retrying"
1659 );
1660 }
1661 if self
1662 .state
1663 .prep_complete
1664 .unwrap_or(!self.state.candidates.is_empty())
1665 {
1666 return Ok(());
1667 }
1668 let repo = self.state.repo.clone();
1669 let _coordination = self.coordinate(&repo).await?;
1672 self.rollback_prep(&repo).await;
1673 self.state.prep_complete = Some(false);
1674 self.state.status = RunStatus::Prep;
1675 self.state.save()?;
1676 let repo = self.state.repo.clone();
1677 let base = self.state.base_commit.clone();
1678 let plan = refs::plan(&repo, &self.state.seeds).await?;
1679 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1680 let root = self.state.worktree_root();
1681 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1682
1683 let hooks_dir = self.state.dir().join("hooks");
1686 if self.state.config.blind.commit_msg_hook {
1687 std::fs::create_dir_all(&hooks_dir)
1688 .with_context(|| format!("create {}", hooks_dir.display()))?;
1689 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1690 let path = hooks_dir.join("commit-msg");
1691 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1692 make_executable(&path)?;
1693 git::acquire_worktree_config(&repo).await?;
1701 self.state.enabled_worktree_config = true;
1702 }
1703
1704 ensure_setup_config(&mut self.state, &repo).await?;
1707
1708 for (index, (spec, label)) in self
1709 .roles
1710 .implementers
1711 .clone()
1712 .into_iter()
1713 .zip(labels)
1714 .enumerate()
1715 {
1716 let branch = self.state.branch_for(label);
1717 let worktree = root.join(format!("cand-{label}"));
1718 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1719 if self.state.config.blind.commit_msg_hook {
1720 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1721 }
1722 git::local_exclude(&worktree, "/.magi/").await?;
1723 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1724 git::worktree_remove(&repo, &worktree).await.ok();
1727 git::branch_delete(&repo, &branch).await.ok();
1728 self.rollback_prep(&repo).await;
1729 self.state.status = RunStatus::Blocked;
1730 self.state
1731 .event("prep", format!("worktree setup failed: {e:#}"));
1732 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1733 self.state.save()?;
1734 return Err(e);
1735 }
1736 self.state.candidates.push(Candidate {
1737 index,
1738 label,
1739 agent: spec.id.clone(),
1740 branch,
1741 worktree,
1742 summary: String::new(),
1743 stat: String::new(),
1744 files: 0,
1745 commits: 0,
1746 empty: false,
1747 failed: None,
1748 verified_noop: None,
1749 duration_ms: 0,
1750 folded: false,
1751 });
1752 self.state.save()?;
1753 for pick in &plan.picks {
1754 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1755 {
1756 self.state.status = RunStatus::Blocked;
1757 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1758 self.state
1759 .event("prep", format!("cannot apply referenced commit: {e}"));
1760 self.state.save()?;
1761 return Err(e);
1762 }
1763 }
1764 }
1765
1766 for j in 1..=self.roles.judges.len() {
1767 let wt = root.join(format!("judge-{j}"));
1768 if !wt.exists() {
1769 git::worktree_add_detached(&repo, &wt, &base).await?;
1770 self.setup_seat_worktree(&repo, &wt, true).await?;
1771 }
1772 }
1773
1774 if self.state.config.graph.advise {
1782 for k in 1..=self.state.config.graph.advisors {
1783 let wt = root.join(format!("advisor-{k}"));
1784 if !wt.exists() {
1785 git::worktree_add_detached(&repo, &wt, &base).await?;
1786 self.setup_seat_worktree(&repo, &wt, true).await?;
1787 }
1788 }
1789 }
1790
1791 let authors: Vec<&str> = self
1796 .roles
1797 .implementers
1798 .iter()
1799 .map(|a| a.id.as_str())
1800 .collect();
1801 let overlap: Vec<String> = self
1802 .roles
1803 .judges
1804 .iter()
1805 .enumerate()
1806 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1807 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1808 .collect();
1809 if !overlap.is_empty() {
1810 let note = format!(
1811 "{} also authored a candidate; blind, but the panel is less \
1812 independent than {} distinct agents would be",
1813 overlap.join(", "),
1814 self.roles.judges.len()
1815 );
1816 self.state.event("prep", note);
1817 }
1818
1819 self.state.event(
1820 "prep",
1821 format!(
1822 "{} candidates, {} judges, base {} ({})",
1823 self.state.candidates.len(),
1824 self.roles.judges.len(),
1825 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1826 self.state.base_branch
1827 ),
1828 );
1829 self.state.prep_complete = Some(true);
1830 self.state.status = RunStatus::Implementing;
1831 self.state.save()?;
1832 Ok(())
1833 }
1834
1835 async fn advise(&mut self) -> Result<()> {
1868 let implement_untouched = self
1869 .state
1870 .candidates
1871 .iter()
1872 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1873 if !self.state.config.graph.advise || self.state.advise_attempted {
1874 return Ok(());
1875 }
1876 if !implement_untouched {
1877 self.state.event(
1878 "advise",
1879 "skipping the design-deliberation stage: at least one \
1880 candidate already shows implementation progress, so this \
1881 run is past the point the stage exists to run before"
1882 .to_owned(),
1883 );
1884 self.state.advise_attempted = true;
1885 self.state.save()?;
1886 return Ok(());
1887 }
1888 let run_id = self.state.id.clone();
1889 let prompts = self.state.config.prompts.clone();
1890 let instruction = self.state.instruction.clone();
1891 let language = self.state.config.graph.language.clone();
1892 let root = self.state.worktree_root();
1893 let n = self.state.config.graph.advisors;
1894 let where_recorded = self.state.dir().join("run.json");
1895
1896 let seats = match self.state.config.advisors() {
1897 Ok(seats) if !seats.is_empty() => seats,
1898 Ok(_) => {
1899 self.state.event(
1900 "advise",
1901 format!(
1902 "[graph] advisors is 0; skipping the design-deliberation \
1903 stage and continuing without a synthesis brief (see {})",
1904 where_recorded.display()
1905 ),
1906 );
1907 self.state.advise_attempted = true;
1908 self.state.save()?;
1909 return Ok(());
1910 }
1911 Err(e) => {
1912 self.state.event(
1913 "advise",
1914 format!(
1915 "could not resolve advisor seats ({e:#}); continuing \
1916 without a design-deliberation brief (see {})",
1917 where_recorded.display()
1918 ),
1919 );
1920 self.state.advise_attempted = true;
1921 self.state.save()?;
1922 return Ok(());
1923 }
1924 };
1925
1926 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1927 let artifacts = agent::artifacts_dir(&self.state.dir());
1928 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1929
1930 let mut jobs = Vec::new();
1931 for (i, spec) in seats.iter().cloned().enumerate() {
1932 let seat_key = format!("advisor-{}", i + 1);
1933 let seat = self.seat(&seat_key, &spec.id);
1934 jobs.push(SeatJob {
1935 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1936 spec,
1937 seat,
1938 cwd: worktrees[i % worktrees.len()].clone(),
1939 timeout,
1940 allow_write: false,
1941 sessions: false,
1942 artifacts: artifacts.clone(),
1943 stem: seat_key,
1944 handover: None,
1945 });
1946 }
1947
1948 self.state.event(
1949 "advise",
1950 format!(
1951 "{} advisor seat(s) sketching a design in parallel",
1952 jobs.len()
1953 ),
1954 );
1955 let mut quota_losses = Vec::new();
1956 let cache = self.state.config.cache_dir();
1957 let ctx = WaveCtx {
1958 carry_seats: false,
1959 pauses: Some((&self.pause, &self.interrupt)),
1960 run: &run_id,
1961 node: "advise",
1962 prompts: &prompts,
1963 cache: cache.as_deref(),
1964 round: None,
1965 };
1966 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1967 let results = ask_json_wave::<Proposal>(
1968 jobs,
1969 Arc::clone(&self.sem),
1970 self.state.config.graph.retries,
1971 &advisor_roster,
1972 &ctx,
1973 &mut quota_losses,
1974 &mut self.state,
1975 &|p: &Proposal| p.validate(),
1976 )
1977 .await;
1978 if self.state.resource_block.is_some() {
1979 return self.stop_for_resource();
1980 }
1981 self.state.quota.extend(quota_losses);
1982
1983 let mut records = Vec::with_capacity(results.len());
1984 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1985 let agent_id = seat.agent.clone();
1986 self.state.seats.insert(seat.key.clone(), seat);
1987 match res {
1988 Ok((proposal, out)) => {
1989 self.state
1990 .event("advise", format!("advisor-{} proposed a design", i + 1));
1991 records.push(advise::AdvisorRecord::proposed(
1992 i + 1,
1993 agent_id,
1994 proposal,
1995 out.duration_ms,
1996 ));
1997 }
1998 Err(e) => {
1999 self.state.event(
2000 "advise",
2001 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2002 );
2003 records.push(advise::AdvisorRecord::failed(
2004 i + 1,
2005 agent_id,
2006 e.to_string(),
2007 ));
2008 }
2009 }
2010 }
2011
2012 let mut advice = advise::Advice {
2013 records,
2014 synthesis: None,
2015 };
2016 if advice.proposals().is_empty() {
2017 self.state.event(
2018 "advise",
2019 "no advisor produced a usable proposal; continuing without a \
2020 synthesis brief"
2021 .to_owned(),
2022 );
2023 } else {
2024 match self
2025 .synthesize_brief(
2026 &advice,
2027 &instruction,
2028 &language,
2029 &worktrees[0],
2030 &artifacts,
2031 &run_id,
2032 &prompts,
2033 cache.as_deref(),
2034 )
2035 .await
2036 {
2037 Ok(Some(text)) => {
2038 self.state.event(
2039 "advise",
2040 "synthesized a design brief for the implementer".to_owned(),
2041 );
2042 advice.synthesis = Some(text);
2043 }
2044 Ok(None) => {
2045 self.state.event(
2046 "advise",
2047 "the synthesis seat produced nothing usable; continuing \
2048 without a design brief"
2049 .to_owned(),
2050 );
2051 }
2052 Err(e) => {
2053 if e.is::<crate::repo_lock::ResourceBlocked>() {
2054 return Err(e);
2055 }
2056 self.state.event(
2057 "advise",
2058 format!("could not synthesize a design brief: {e:#}"),
2059 );
2060 }
2061 }
2062 }
2063 advise::apply_reflection(&mut advice);
2064
2065 self.state.advice = Some(advice);
2066 self.state.advise_attempted = true;
2067 self.state.save()?;
2068 Ok(())
2069 }
2070
2071 #[allow(clippy::too_many_arguments)]
2083 async fn synthesize_brief(
2084 &mut self,
2085 advice: &advise::Advice,
2086 instruction: &str,
2087 language: &str,
2088 cwd: &Path,
2089 artifacts: &Path,
2090 run_id: &str,
2091 prompts: &Prompts,
2092 cache: Option<&Path>,
2093 ) -> Result<Option<String>> {
2094 let chain = agent::pick_chain(
2095 &self.state.config.agents,
2096 self.state.config.roles.synthesizer.as_ref(),
2097 &agent::installed,
2098 "synthesizer",
2099 )?;
2100 let proposals = advice.proposals();
2101 let mut prompt = prompt::with_overlay(
2102 prompt::synthesize_brief(instruction, &proposals, language),
2103 prompts.overlay("advise"),
2104 );
2105 if cache.is_some() {
2106 prompt.push('\n');
2111 prompt.push_str(&prompt::build_cache_note("advise", false));
2112 }
2113 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2114 let mut last = None;
2119 for (n, spec) in chain.iter().enumerate() {
2120 if n > 0 {
2121 self.state
2122 .event("advise", format!("synthesis falling back to {}", spec.id));
2123 }
2124 let mut seat = self.seat("advise-synthesis", &spec.id);
2125 let wait = self.repository_wait();
2126 let outcome = wait
2127 .scope(agent::invoke(
2128 spec,
2129 &mut seat,
2130 &Invocation {
2131 coordination: self.state.config.coordination.as_ref(),
2132 cwd,
2133 prompt: &prompt,
2134 timeout,
2135 allow_write: false,
2136 unsandboxed: false,
2137 sessions: false,
2138 artifacts,
2139 stem: &if n == 0 {
2140 "advise-synthesis".to_owned()
2141 } else {
2142 format!("advise-synthesis-{}", spec.id)
2143 },
2144 run: run_id,
2145 node: "advise",
2146 cache_dir: None,
2147 attachments: &[],
2148 writable: &[],
2149 },
2150 ))
2151 .await;
2152 if outcome.is_ok() {
2153 self.state.seats.insert(seat.key.clone(), seat);
2154 }
2155 let advance = agent::chain_advances(&outcome);
2156 last = Some(outcome);
2157 if !advance {
2158 break;
2159 }
2160 }
2161 let out = last.expect("a chain holds at least one agent")?;
2164 if !out.usable() {
2165 return Ok(None);
2166 }
2167 let text =
2168 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2169 Ok((!text.trim().is_empty()).then_some(text))
2170 }
2171
2172 async fn implement(&mut self) -> Result<()> {
2175 let run_id = self.state.id.clone();
2180 let prompts = self.state.config.prompts.clone();
2181 let todo: Vec<usize> = self
2182 .state
2183 .candidates
2184 .iter()
2185 .enumerate()
2186 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2187 .map(|(i, _)| i)
2188 .collect();
2189 if todo.is_empty() {
2190 return self.after_implement();
2191 }
2192 self.state.status = RunStatus::Implementing;
2193
2194 let language = self.state.config.graph.language.clone();
2195 let github_language = self.state.config.graph.github_language.clone();
2196 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2197 let sessions = self.state.config.graph.sessions;
2198 let artifacts = agent::artifacts_dir(&self.state.dir());
2199 let brief = self
2203 .state
2204 .advice
2205 .as_ref()
2206 .and_then(|a| a.synthesis.as_deref())
2207 .map(str::to_owned);
2208 let attachments = self.state.attachments.clone();
2209
2210 let mut jobs = Vec::new();
2211 for &i in &todo {
2212 let (index, label, worktree) = {
2213 let c = &self.state.candidates[i];
2214 (c.index, c.label, c.worktree.clone())
2215 };
2216 let spec = self.roles.implementers[index].clone();
2217 let seat_key = format!("impl-{label}");
2218 let seat = self.seat(&seat_key, &spec.id);
2219 let instruction = seeded_instruction(&self.state);
2220 jobs.push(SeatJob {
2221 spec,
2222 seat,
2223 prompt: prompt::implement_in(
2224 &instruction,
2225 &worktree.to_string_lossy(),
2226 &language,
2227 &github_language,
2228 brief.as_deref(),
2229 &attachments,
2230 ),
2231 cwd: worktree,
2232 timeout,
2233 allow_write: true,
2234 sessions,
2235 artifacts: artifacts.clone(),
2236 stem: format!("impl-{label}"),
2237 handover: None,
2238 });
2239 }
2240
2241 self.state.event(
2242 "implement",
2243 format!("{} candidates in parallel", jobs.len()),
2244 );
2245 let mut sent = jobs.clone();
2251 let cache = self.state.config.cache_dir();
2252 let ctx = WaveCtx {
2253 carry_seats: false,
2254 pauses: Some((&self.pause, &self.interrupt)),
2255 run: &run_id,
2256 node: "implement",
2257 prompts: &prompts,
2258 cache: cache.as_deref(),
2259 round: None,
2260 };
2261 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2262 if !wave_blocked(&results) {
2263 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2264 .await;
2265 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2266 .await?;
2267 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2268 .await;
2269 }
2270 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2271
2272 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2273 && results
2274 .iter()
2275 .any(|(_, _, out)| FailClass::of(out).is_some());
2276 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2277 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2278 continue;
2279 }
2280 let seat_key = seat.key.clone();
2281 let agent = seat.agent.clone();
2290 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2291 self.state.seats.insert(seat.key.clone(), seat);
2292 let label = self.state.candidates[i].label;
2293 let worktree = self.state.candidates[i].worktree.clone();
2294 let base = self.state.base_commit.clone();
2295
2296 let (summary, duration, failed, verified_claim) = match out {
2297 AgentOutcome::Ok(o) => {
2298 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2299 let failed = (!o.usable()).then(|| {
2300 if o.timed_out {
2301 "agent timed out".to_owned()
2302 } else {
2303 format!("agent exited with {:?}", o.exit_code)
2304 }
2305 });
2306 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2307 (text, o.duration_ms, failed, verified_claim)
2308 }
2309 AgentOutcome::Dropped(o) => {
2315 let why = o
2316 .dropped
2317 .as_ref()
2318 .map(|d| d.why.as_str())
2319 .unwrap_or("the CLI ended the stream without delivering its answer");
2320 (
2321 String::new(),
2322 o.duration_ms,
2323 Some(format!("the CLI dropped the stream ({why})")),
2324 None,
2325 )
2326 }
2327 AgentOutcome::Quota(o) => {
2328 self.state.quota.push(QuotaLoss {
2329 seat: seat_key,
2330 node: "implement".to_owned(),
2331 at: Timestamp::now(),
2332 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2333 });
2334 (
2335 String::new(),
2336 o.duration_ms,
2337 Some("rate limited (quota); produced no change".to_owned()),
2338 None,
2339 )
2340 }
2341 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2342 (String::new(), 0, Some(e), None)
2343 }
2344 };
2345
2346 let wait = crate::repo_lock::WaitPolicy::new(
2351 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2352 || false,
2353 );
2354 let _coordination = coordinate_operation(&mut self.state, &worktree, &wait).await?;
2355 let rescued = match git::rescue_commit(
2356 &worktree,
2357 &format!("magi: candidate {label} (uncommitted work)"),
2358 )
2359 .await
2360 {
2361 Ok(r) => {
2362 self.state.note_withheld("implement", &r.withheld);
2363 r.committed
2364 }
2365 Err(_) => false,
2366 };
2367 let commits = git::commits_ahead(&worktree, &base, "HEAD")
2368 .await
2369 .unwrap_or(0);
2370 let patch = git::diff(&worktree, &base, "HEAD")
2371 .await
2372 .unwrap_or_default();
2373 let stat = git::diff_stat(&worktree, &base, "HEAD")
2374 .await
2375 .unwrap_or_default();
2376 let files = git::changed_files(&worktree, &base, "HEAD")
2377 .await
2378 .map(|f| f.len())
2379 .unwrap_or(0);
2380 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2381
2382 let c = &mut self.state.candidates[i];
2383 if !exhausted_the_fallback_chain {
2384 c.agent = agent;
2385 }
2386 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2387 c.stat = stat;
2388 c.files = files;
2389 c.commits = commits;
2390 c.duration_ms = duration;
2391 c.empty = commits == 0 || patch.trim().is_empty();
2392 c.failed = match failed {
2395 Some(_) if c.empty => failed,
2396 _ => None,
2397 };
2398 c.verified_noop = if c.empty { verified_claim } else { None };
2403 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
2404 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
2405 (None, true, Some(_), _) => {
2406 format!("candidate {label}: no change produced (agent-verified no-op)")
2407 }
2408 (None, true, None, _) => format!("candidate {label}: no change produced"),
2409 (None, false, _, true) => {
2410 format!(
2411 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
2412 )
2413 }
2414 (None, false, _, false) => {
2415 format!("candidate {label}: {files} files, {commits} commits")
2416 }
2417 };
2418 self.state.event("implement", note);
2419 self.state.save()?;
2420 }
2421
2422 if resource_blocked {
2423 return self.stop_for_resource();
2424 }
2425 if selected_failed {
2426 self.state.status = RunStatus::Blocked;
2427 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2428 self.state.save()?;
2429 bail!(
2430 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2431 );
2432 }
2433 self.after_implement()
2434 }
2435
2436 async fn resume_undelivered(
2464 &mut self,
2465 results: &mut [(usize, SeatState, AgentOutcome)],
2466 sent: &[SeatJob],
2467 prompts: &Prompts,
2468 run_id: &str,
2469 ) {
2470 for (wi, seat, out) in results.iter_mut() {
2471 let Some(dropped) = (match &*out {
2472 AgentOutcome::Dropped(o) => o.dropped.clone(),
2473 _ => None,
2474 }) else {
2475 continue;
2476 };
2477 let Some(job) = sent.get(*wi) else { continue };
2478 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2480 self.state.event(
2481 "implement",
2482 format!(
2483 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2484 work is in the tree",
2485 seat.key, dropped.output_tokens, dropped.why
2486 ),
2487 );
2488 continue;
2489 }
2490 if !has_context(&job.spec, seat, job.sessions) {
2498 self.state.event(
2499 "implement",
2500 format!(
2501 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2502 is no session left to resume",
2503 seat.key, dropped.output_tokens, dropped.why
2504 ),
2505 );
2506 continue;
2507 }
2508 self.state.event(
2509 "implement",
2510 format!(
2511 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2512 conversation",
2513 seat.key, dropped.output_tokens, dropped.why
2514 ),
2515 );
2516 let mut retry = job.clone();
2517 retry.seat = seat.clone();
2518 retry.prompt = prompt::resume_after_drop(&dropped.why);
2519 retry.timeout = retry_budget(job.timeout, true);
2520 retry.stem = format!("{}-resume", job.stem);
2521 let cache = self.state.config.cache_dir();
2522 let ctx = WaveCtx {
2523 carry_seats: false,
2524 pauses: Some((&self.pause, &self.interrupt)),
2525 run: run_id,
2526 node: "implement",
2527 prompts,
2528 cache: cache.as_deref(),
2529 round: None,
2530 };
2531 let (resumed_seat, resumed) =
2532 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2533 *seat = resumed_seat;
2534 *out = resumed;
2535 }
2536 }
2537
2538 async fn resume_seat_handovers(
2603 &mut self,
2604 results: &mut [(usize, SeatState, AgentOutcome)],
2605 sent: &mut [SeatJob],
2606 prompts: &Prompts,
2607 run_id: &str,
2608 ) -> Result<()> {
2609 let instruction = seeded_instruction(&self.state);
2610 let language = self.state.config.graph.language.clone();
2611 let github_language = self.state.config.graph.github_language.clone();
2612 let brief = self
2613 .state
2614 .advice
2615 .as_ref()
2616 .and_then(|a| a.synthesis.as_deref())
2617 .map(str::to_owned);
2618 let attachments = self.state.attachments.clone();
2619 for (wi, seat, out) in results.iter_mut() {
2620 let others: BTreeSet<String> = sent
2623 .iter()
2624 .enumerate()
2625 .filter(|(j, _)| j != wi)
2626 .map(|(_, j)| j.spec.id.clone())
2627 .collect();
2628 let Some(job) = sent.get_mut(*wi) else {
2629 continue;
2630 };
2631 let start = self
2636 .roles
2637 .implementer_roster
2638 .iter()
2639 .position(|s| s.id == job.spec.id)
2640 .unwrap_or(0);
2641 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2642 let mut fallback_attempt = 0usize;
2643 let mut prev: Option<FailClass> = None;
2644 while let Some(cur) = FailClass::of(&*out) {
2645 if !should_hand_over(prev.as_ref(), &cur) {
2646 break;
2647 }
2648 let Some(next) =
2649 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2650 .cloned()
2651 else {
2652 break;
2653 };
2654 tried.insert(next.id.clone());
2655 fallback_attempt += 1;
2656
2657 let coordination = self.coordinate(&job.cwd).await?;
2658 if let Ok(r) = git::rescue_commit(
2659 &job.cwd,
2660 &format!(
2661 "magi: candidate {} (uncommitted work before {} fallback)",
2662 seat.key,
2663 if cur == FailClass::Quota {
2664 "quota"
2665 } else {
2666 "handover"
2667 }
2668 ),
2669 )
2670 .await
2671 {
2672 self.state.note_withheld("implement", &r.withheld);
2673 }
2674 drop(coordination);
2675
2676 record_handover(
2677 &mut self.state,
2678 "implement",
2679 &seat.key,
2680 &seat.agent,
2681 &next.id,
2682 &cur,
2683 &fail_reason(&*out),
2684 );
2685 prev = Some(cur.clone());
2686
2687 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2688 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2689 job.spec = next.clone();
2697 let mut retry = job.clone();
2698 retry.seat = new_seat;
2699 retry.prompt = prompt::implement_in(
2700 &instruction,
2701 &job.cwd.to_string_lossy(),
2702 &language,
2703 &github_language,
2704 brief.as_deref(),
2705 &attachments,
2706 );
2707 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2708 let cache = self.state.config.cache_dir();
2709 let ctx = WaveCtx {
2710 carry_seats: false,
2711 pauses: Some((&self.pause, &self.interrupt)),
2712 run: run_id,
2713 node: "implement",
2714 prompts,
2715 cache: cache.as_deref(),
2716 round: None,
2717 };
2718 let (fallback_seat, fallback_out) = run_one(
2719 retry,
2720 Arc::clone(&self.sem),
2721 &ctx,
2722 &mut self.state,
2723 fallback_attempt,
2724 )
2725 .await;
2726 *seat = fallback_seat;
2727 *out = fallback_out;
2728 }
2729 }
2730 Ok(())
2731 }
2732
2733 async fn resume_unconfirmed_commands(
2757 &mut self,
2758 results: &mut [(usize, SeatState, AgentOutcome)],
2759 sent: &[SeatJob],
2760 prompts: &Prompts,
2761 run_id: &str,
2762 ) {
2763 for (wi, seat, out) in results.iter_mut() {
2764 let AgentOutcome::Ok(o) = &*out else {
2765 continue;
2766 };
2767 if !has_unconfirmed_command(&o.commands) {
2768 continue;
2769 }
2770 let Some(job) = sent.get(*wi) else { continue };
2771 if !has_context(&job.spec, seat, job.sessions) {
2772 self.state.event(
2773 "implement",
2774 format!(
2775 "{}: the reply named a command whose own CLI never confirmed the exit \
2776 status of, but there is no session left to resume",
2777 seat.key
2778 ),
2779 );
2780 continue;
2781 }
2782 self.state.event(
2783 "implement",
2784 format!(
2785 "{}: the reply named a command whose own CLI never confirmed the exit \
2786 status of; resuming the conversation",
2787 seat.key
2788 ),
2789 );
2790 let mut retry = job.clone();
2791 retry.seat = seat.clone();
2792 retry.prompt = prompt::resume_incomplete(
2793 "a command in your last reply had no confirmed exit status",
2794 );
2795 retry.timeout = retry_budget(job.timeout, true);
2796 retry.stem = format!("{}-confirm", job.stem);
2797 let cache = self.state.config.cache_dir();
2798 let ctx = WaveCtx {
2799 carry_seats: false,
2800 pauses: Some((&self.pause, &self.interrupt)),
2801 run: run_id,
2802 node: "implement",
2803 prompts,
2804 cache: cache.as_deref(),
2805 round: None,
2806 };
2807 let (resumed_seat, resumed) =
2808 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2809 *seat = resumed_seat;
2810 *out = resumed;
2811 }
2812 }
2813
2814 async fn continue_fix_report(
2835 &mut self,
2836 mut seat: SeatState,
2837 parse_err: String,
2838 job: &SeatJob,
2839 prompts: &Prompts,
2840 run_id: &str,
2841 round: usize,
2842 ) -> (
2843 SeatState,
2844 Option<FixReport>,
2845 Option<String>,
2846 ContinuationRecord,
2847 ) {
2848 let mut last_err = parse_err;
2849 let mut cumulative_wait_ms = 0u64;
2850 let mut attempts = 0usize;
2851 loop {
2852 if !has_context(&job.spec, &seat, job.sessions) {
2853 self.state.event(
2854 "fix",
2855 format!(
2856 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2857 session left to resume into"
2858 ),
2859 );
2860 let outcome = if attempts == 0 {
2861 ContinuationOutcome::NoSession
2862 } else {
2863 ContinuationOutcome::Exhausted
2864 };
2865 return (
2866 seat,
2867 None,
2868 Some(format!("unparsable fix report: {last_err}")),
2869 ContinuationRecord {
2870 attempts,
2871 cumulative_wait_ms,
2872 outcome,
2873 },
2874 );
2875 }
2876 if attempts >= MAX_FIX_CONTINUATIONS {
2877 self.state.event(
2878 "fix",
2879 format!(
2880 "round {round}: fixer's reply still had no adoption report after \
2881 {attempts} continuation(s) ({last_err}); giving up"
2882 ),
2883 );
2884 return (
2885 seat,
2886 None,
2887 Some(format!(
2888 "unparsable fix report after {attempts} continuation(s): {last_err}"
2889 )),
2890 ContinuationRecord {
2891 attempts,
2892 cumulative_wait_ms,
2893 outcome: ContinuationOutcome::Exhausted,
2894 },
2895 );
2896 }
2897 attempts += 1;
2898 self.state.event(
2899 "fix",
2900 format!(
2901 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2902 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2903 ),
2904 );
2905 let mut retry = job.clone();
2906 retry.seat = seat.clone();
2907 retry.prompt = prompt::resume_incomplete(&last_err);
2908 retry.timeout = retry_budget(job.timeout, true);
2909 retry.stem = format!("{}-continue{attempts}", job.stem);
2910 let cache = self.state.config.cache_dir();
2911 let ctx = WaveCtx {
2912 carry_seats: false,
2913 pauses: Some((&self.pause, &self.interrupt)),
2914 run: run_id,
2915 node: "fix",
2916 prompts,
2917 cache: cache.as_deref(),
2918 round: Some(round),
2919 };
2920 let (resumed_seat, resumed_out) = run_one(
2921 retry,
2922 Arc::clone(&self.sem),
2923 &ctx,
2924 &mut self.state,
2925 attempts,
2926 )
2927 .await;
2928 seat = resumed_seat;
2929 match resumed_out {
2930 AgentOutcome::Ok(o) => {
2931 cumulative_wait_ms += o.duration_ms;
2932 match verdict::extract_json::<FixReport>(&o.text) {
2933 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2934 self.state.event(
2935 "fix",
2936 format!(
2937 "round {round}: fixer's adoption report recovered after \
2938 {attempts} continuation(s)"
2939 ),
2940 );
2941 return (
2942 seat,
2943 Some(report),
2944 None,
2945 ContinuationRecord {
2946 attempts,
2947 cumulative_wait_ms,
2948 outcome: ContinuationOutcome::Resumed,
2949 },
2950 );
2951 }
2952 Ok(_) => {
2960 last_err = "the reply parsed, but it reported a command whose own CLI \
2961 never confirmed an exit status"
2962 .to_owned();
2963 }
2964 Err(e) => last_err = e.to_string(),
2965 }
2966 }
2967 AgentOutcome::Quota(o) => {
2968 cumulative_wait_ms += o.duration_ms;
2969 self.state.quota.push(QuotaLoss {
2970 seat: seat.key.clone(),
2971 node: "fix".to_owned(),
2972 at: Timestamp::now(),
2973 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2974 });
2975 self.state.event(
2976 "fix",
2977 format!(
2978 "round {round}: continuation rate limited (quota); not retrying now"
2979 ),
2980 );
2981 return (
2982 seat,
2983 None,
2984 Some("rate limited (quota) while recovering the fix report".to_owned()),
2985 ContinuationRecord {
2986 attempts,
2987 cumulative_wait_ms,
2988 outcome: ContinuationOutcome::QuotaLost,
2989 },
2990 );
2991 }
2992 AgentOutcome::Dropped(o) => {
2993 cumulative_wait_ms += o.duration_ms;
2994 let why = o
2995 .dropped
2996 .as_ref()
2997 .map(|d| d.why.as_str())
2998 .unwrap_or("the CLI ended the stream without delivering its answer");
2999 last_err = format!("the CLI dropped the stream ({why})");
3000 }
3001 AgentOutcome::Failed(e) => last_err = e,
3002 AgentOutcome::ResourceBlocked(e) => {
3005 return (
3006 seat,
3007 None,
3008 Some(format!("the continuation was not started: {e}")),
3009 ContinuationRecord {
3010 attempts: attempts.saturating_sub(1),
3011 cumulative_wait_ms,
3012 outcome: ContinuationOutcome::NoSession,
3013 },
3014 );
3015 }
3016 }
3017 }
3018 }
3019
3020 fn after_implement(&mut self) -> Result<()> {
3021 if self.state.leaks.is_empty() {
3023 let cfg = self.state.config.blind.clone();
3024 let mut leaks = Vec::new();
3025 for c in &self.state.candidates {
3026 let Some(patch) =
3027 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3028 else {
3029 continue;
3030 };
3031 leaks.extend(blind::scan(
3032 &format!("candidate {} patch", c.label),
3033 &patch,
3034 &cfg.vendor_tokens,
3035 ));
3036 }
3037 if !leaks.is_empty() {
3038 let summary = leaks
3039 .iter()
3040 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3041 .collect::<Vec<_>>()
3042 .join(", ");
3043 match cfg.on_leak {
3044 LeakPolicy::Fail => {
3045 self.state.status = RunStatus::Failed;
3046 self.state
3047 .event("blind", format!("vendor text in a patch: {summary}"));
3048 self.state.leaks = leaks;
3049 self.state.save()?;
3050 self.settle_questions();
3051 bail!(
3052 "blind.on_leak = \"fail\" and vendor text reached a \
3053 judged patch: {summary}"
3054 );
3055 }
3056 LeakPolicy::Redact => self.state.event(
3057 "blind",
3058 format!("redacting vendor text for judging: {summary}"),
3059 ),
3060 LeakPolicy::Warn => self.state.event(
3061 "blind",
3062 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3063 ),
3064 }
3065 self.state.leaks = leaks;
3066 }
3067 }
3068
3069 if self.state.viable().is_empty() {
3070 if self.state.all_candidates_verified_noop() {
3071 self.state.status = RunStatus::VerifiedNoop;
3082 self.state.save()?;
3083 self.settle_questions();
3084 return Ok(());
3085 }
3086 self.state.status = RunStatus::Failed;
3087 self.state.save()?;
3088 self.settle_questions();
3089 bail!("no candidate produced a change; nothing to judge");
3090 }
3091 self.state.status = RunStatus::Judging;
3092 self.state.save()?;
3093 Ok(())
3094 }
3095
3096 async fn judge(&mut self) -> Result<()> {
3099 let run_id = self.state.id.clone();
3104 let prompts = self.state.config.prompts.clone();
3105 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3106 return Ok(());
3107 }
3108 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3109 if viable.len() == 1 {
3110 self.state.judge_skipped = true;
3117 self.state.event(
3118 "judge",
3119 format!(
3120 "only candidate {} produced a change; judging skipped",
3121 viable[0].label
3122 ),
3123 );
3124 self.state.save()?;
3125 return Ok(());
3126 }
3127 self.state.status = RunStatus::Judging;
3128
3129 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3130 let language = self.state.config.graph.language.clone();
3131 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3132 let sessions = self.state.config.graph.sessions;
3133 let artifacts = agent::artifacts_dir(&self.state.dir());
3134 let root = self.state.worktree_root();
3135 let base_short = short(&self.state.base_commit);
3136
3137 let mut jobs = Vec::new();
3138 let mut orders = Vec::new();
3139 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3140 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3141 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3142 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3143 let seat_key = format!("judge-{}", j + 1);
3144 let seat = self.seat(&seat_key, &spec.id);
3145 jobs.push(SeatJob {
3146 prompt: prompt::judge(
3147 &self.state.instruction,
3148 &views,
3149 self.roles.judges.len(),
3150 &base_short,
3151 &language,
3152 ),
3153 spec,
3154 seat,
3155 cwd: root.join(format!("judge-{}", j + 1)),
3156 timeout,
3157 allow_write: false,
3158 sessions,
3159 artifacts: artifacts.clone(),
3160 stem: format!("judge-{}", j + 1),
3161 handover: None,
3162 });
3163 }
3164
3165 self.state.event(
3166 "judge",
3167 format!(
3168 "{} judges ranking {} candidates blind",
3169 jobs.len(),
3170 viable.len()
3171 ),
3172 );
3173 let labels_for_check = labels.clone();
3174 let mut quota_losses = Vec::new();
3175 let cache = self.state.config.cache_dir();
3176 let ctx = WaveCtx {
3177 carry_seats: false,
3178 pauses: Some((&self.pause, &self.interrupt)),
3179 run: &run_id,
3180 node: "judge",
3181 prompts: &prompts,
3182 cache: cache.as_deref(),
3183 round: None,
3184 };
3185 let results = ask_json_wave::<Ranking>(
3186 jobs,
3187 Arc::clone(&self.sem),
3188 self.state.config.graph.retries,
3189 &self.roles.judge_roster,
3190 &ctx,
3191 &mut quota_losses,
3192 &mut self.state,
3193 &move |r: &Ranking| r.validate(&labels_for_check),
3194 )
3195 .await;
3196 if self.state.resource_block.is_some() {
3197 return self.stop_for_resource();
3198 }
3199 self.state.quota.extend(quota_losses);
3200
3201 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3202 let agent_id = seat.agent.clone();
3203 self.state.seats.insert(seat.key.clone(), seat);
3204 let mut record = Judgement {
3205 judge: j + 1,
3206 seat: format!("judge-{}", j + 1),
3207 agent: agent_id,
3208 ranking: Vec::new(),
3209 reasons: BTreeMap::new(),
3210 confidence: None,
3211 strengths: BTreeMap::new(),
3212 order: orders[j].clone(),
3213 failed: None,
3214 duration_ms: 0,
3215 };
3216 match res {
3217 Ok((ranking, out)) => {
3218 record.ranking = ranking.normalized();
3219 record.reasons = ranking.reasons;
3220 record.strengths = ranking.strengths;
3221 record.confidence = ranking.confidence;
3222 record.duration_ms = out.duration_ms;
3223 self.state.event(
3224 "judge",
3225 format!(
3226 "judge {} ranked {}",
3227 j + 1,
3228 record.ranking.iter().collect::<String>()
3229 ),
3230 );
3231 }
3232 Err(e) => {
3233 record.failed = Some(e.to_string());
3234 self.state
3235 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3236 }
3237 }
3238 self.state.judgements.push(record);
3239 self.state.save()?;
3240 }
3241 Ok(())
3242 }
3243
3244 async fn deliberate(&mut self) -> Result<()> {
3247 let run_id = self.state.id.clone();
3252 let prompts = self.state.config.prompts.clone();
3253 if !self.state.deliberation.is_empty() {
3254 return Ok(());
3255 }
3256 let tops: Vec<char> = self
3257 .state
3258 .judgements
3259 .iter()
3260 .filter_map(|j| j.ranking.first().copied())
3261 .collect();
3262 let rounds = self.state.config.graph.deliberate_rounds;
3263 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3264 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3265 self.state.event(
3266 "deliberate",
3267 format!("judges agreed on {} outright; no deliberation", tops[0]),
3268 );
3269 }
3270 self.state.status = RunStatus::Voting;
3271 self.state.save()?;
3272 return Ok(());
3273 }
3274
3275 self.state.status = RunStatus::Deliberating;
3276 self.state.event(
3277 "deliberate",
3278 format!(
3279 "split: first choices were {} — opening {rounds} round(s)",
3280 tops.iter().collect::<String>()
3281 ),
3282 );
3283
3284 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3285 let language = self.state.config.graph.language.clone();
3286 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3287 let sessions = self.state.config.graph.sessions;
3288 let artifacts = agent::artifacts_dir(&self.state.dir());
3289 let root = self.state.worktree_root();
3290 let base_short = short(&self.state.base_commit);
3291
3292 for round in 1..=rounds {
3296 let mut turns: Vec<DeliberationTurn> = Vec::new();
3297 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3298 if self.state.judgements[j].failed.is_some() {
3299 continue;
3300 }
3301 let seat_key = format!("judge-{}", j + 1);
3302 let spec = self.occupant(&seat_key, spec);
3303 let mut seat = self.seat(&seat_key, &spec.id);
3304 let transcript = self.transcript(&turns, j);
3305 let build = |context: Option<&str>| {
3306 prompt::deliberate(
3307 &self.state.instruction,
3308 context,
3309 &transcript,
3310 round,
3311 rounds,
3312 &language,
3313 )
3314 };
3315 let block = self.candidate_block(&viable, &base_short);
3316 let full = build(Some(&block));
3317 let text = if has_context(&spec, &seat, sessions) {
3318 build(None)
3319 } else {
3320 full.clone()
3321 };
3322 let job = SeatJob {
3323 spec,
3324 seat: seat.clone(),
3325 prompt: text,
3326 cwd: root.join(format!("judge-{}", j + 1)),
3327 timeout,
3328 allow_write: false,
3329 sessions,
3330 artifacts: artifacts.clone(),
3331 stem: format!("delib-{round}-judge-{}", j + 1),
3332 handover: Some(full),
3333 };
3334 let cache = self.state.config.cache_dir();
3335 let ctx = WaveCtx {
3336 carry_seats: false,
3337 pauses: Some((&self.pause, &self.interrupt)),
3338 run: &run_id,
3339 node: "deliberate",
3340 prompts: &prompts,
3341 cache: cache.as_deref(),
3342 round: None,
3343 };
3344 let mut losses = Vec::new();
3348 let mut results = ask_wave_with::<String>(
3349 vec![job],
3350 Arc::clone(&self.sem),
3351 0,
3352 &self.roles.judge_roster,
3353 &ctx,
3354 &mut losses,
3355 &mut self.state,
3356 &|text: &str| {
3357 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3358 },
3359 )
3360 .await;
3361 self.state.quota.extend(losses);
3362 let (updated, res, _) = results.pop().expect("one job in, one result out");
3363 seat = updated;
3364 let agent_id = seat.agent.clone();
3365 self.state.seats.insert(seat.key.clone(), seat);
3366 let body = match res {
3367 Ok((body, _)) => body,
3368 Err(e) => {
3371 self.state
3372 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3373 continue;
3374 }
3375 };
3376 let tentative = verdict::extract_json::<Position>(&body)
3377 .ok()
3378 .and_then(|p| p.tentative)
3379 .and_then(|s| s.trim().chars().next())
3380 .map(|c| c.to_ascii_uppercase());
3381 self.state.event(
3382 "deliberate",
3383 format!(
3384 "round {round}: judge {} now favours {}",
3385 j + 1,
3386 tentative.map_or("—".to_owned(), |c| c.to_string())
3387 ),
3388 );
3389 turns.push(DeliberationTurn {
3390 judge: j + 1,
3391 agent: agent_id,
3392 body: blind::sanitize_prose(&body, &self.state.config.blind),
3393 tentative,
3394 });
3395 }
3396 self.state
3397 .deliberation
3398 .push(DeliberationRound { round, turns });
3399 self.state.save()?;
3400 }
3401
3402 self.state.status = RunStatus::Voting;
3403 self.state.save()?;
3404 Ok(())
3405 }
3406
3407 async fn vote(&mut self) -> Result<()> {
3410 let run_id = self.state.id.clone();
3415 let prompts = self.state.config.prompts.clone();
3416 if !self.state.votes.is_empty() {
3417 return Ok(());
3418 }
3419 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3420 if viable.len() == 1 {
3421 return Ok(());
3422 }
3423 self.state.status = RunStatus::Voting;
3424
3425 let language = self.state.config.graph.language.clone();
3426 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3427 let sessions = self.state.config.graph.sessions;
3428 let artifacts = agent::artifacts_dir(&self.state.dir());
3429 let root = self.state.worktree_root();
3430 let base_short = short(&self.state.base_commit);
3431 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3432
3433 let mut jobs = Vec::new();
3434 let mut seats_at = Vec::new();
3435 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3436 if self
3437 .state
3438 .judgements
3439 .get(j)
3440 .is_some_and(|r| r.failed.is_some())
3441 {
3442 continue;
3443 }
3444 let seat_key = format!("judge-{}", j + 1);
3445 let spec = self.occupant(&seat_key, spec);
3446 let seat = self.seat(&seat_key, &spec.id);
3447 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3448 let text = if has_context(&spec, &seat, sessions) {
3449 prompt::final_vote(&viable, &language)
3450 } else {
3451 full.clone()
3452 };
3453 jobs.push(SeatJob {
3454 spec,
3455 seat,
3456 prompt: text,
3457 cwd: root.join(format!("judge-{}", j + 1)),
3458 timeout,
3459 allow_write: false,
3460 sessions,
3461 artifacts: artifacts.clone(),
3462 stem: format!("vote-judge-{}", j + 1),
3463 handover: Some(full),
3464 });
3465 seats_at.push(j);
3466 }
3467
3468 self.state.event(
3469 "vote",
3470 format!(
3471 "collecting {} final votes one by one, privately",
3472 jobs.len()
3473 ),
3474 );
3475 let allowed = viable.clone();
3476 let mut quota_losses = Vec::new();
3477 let cache = self.state.config.cache_dir();
3478 let ctx = WaveCtx {
3479 carry_seats: false,
3480 pauses: Some((&self.pause, &self.interrupt)),
3481 run: &run_id,
3482 node: "vote",
3483 prompts: &prompts,
3484 cache: cache.as_deref(),
3485 round: None,
3486 };
3487 let results = ask_json_wave::<FinalVote>(
3488 jobs,
3489 Arc::clone(&self.sem),
3490 self.state.config.graph.retries,
3491 &self.roles.judge_roster,
3492 &ctx,
3493 &mut quota_losses,
3494 &mut self.state,
3495 &move |v: &FinalVote| match v.label() {
3496 Some(c) if allowed.contains(&c) => Ok(()),
3497 other => bail!("vote {other:?} is not one of {allowed:?}"),
3498 },
3499 )
3500 .await;
3501 if self.state.resource_block.is_some() {
3502 return self.stop_for_resource();
3503 }
3504 self.state.quota.extend(quota_losses);
3505
3506 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3507 let agent_id = seat.agent.clone();
3508 self.state.seats.insert(seat.key.clone(), seat);
3509 let initial = self
3510 .state
3511 .judgements
3512 .get(j)
3513 .and_then(|r| r.ranking.first().copied());
3514 let mut record = VoteRecord {
3515 judge: j + 1,
3516 agent: agent_id,
3517 vote: None,
3518 reason: String::new(),
3519 changed: false,
3520 };
3521 match res {
3522 Ok((v, _)) => {
3523 record.vote = v.label();
3524 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3525 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3526 self.state.event(
3527 "vote",
3528 format!(
3529 "judge {} voted {}{}",
3530 j + 1,
3531 record.vote.unwrap_or('?'),
3532 if record.changed { " (changed)" } else { "" }
3533 ),
3534 );
3535 }
3536 Err(e) => {
3537 self.state
3538 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3539 }
3540 }
3541 self.state.votes.push(record);
3542 self.state.save()?;
3543 }
3544 Ok(())
3545 }
3546
3547 fn tally(&mut self) -> Result<()> {
3550 if self.state.tally.is_some() {
3551 return Ok(());
3552 }
3553 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3554 let tops: Vec<char> = self
3555 .state
3556 .judgements
3557 .iter()
3558 .filter_map(|j| j.ranking.first().copied())
3559 .collect();
3560 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3561
3562 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3565 let mut cast: Vec<char> = Vec::new();
3566 for (i, j) in self.state.judgements.iter().enumerate() {
3567 let vote = self
3568 .state
3569 .votes
3570 .iter()
3571 .find(|v| v.judge == i + 1)
3572 .and_then(|v| v.vote)
3573 .or_else(|| j.ranking.first().copied());
3574 if let Some(v) = vote {
3575 *first_choice.entry(v).or_insert(0) += 1;
3576 cast.push(v);
3577 }
3578 }
3579
3580 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3581 for j in &self.state.judgements {
3582 let n = j.ranking.len();
3583 for (pos, label) in j.ranking.iter().enumerate() {
3584 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3585 }
3586 }
3587
3588 let best = first_choice.values().copied().max().unwrap_or(0);
3589 let mut leaders: Vec<char> = first_choice
3590 .iter()
3591 .filter(|(_, v)| **v == best)
3592 .map(|(k, _)| *k)
3593 .collect();
3594 let mut tie_break = None;
3595 if leaders.len() > 1 {
3596 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3597 let borda_leaders: Vec<char> = leaders
3598 .iter()
3599 .copied()
3600 .filter(|l| borda[l] == top_borda)
3601 .collect();
3602 tie_break = Some(if borda_leaders.len() == 1 {
3603 format!(
3604 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3605 leaders.len()
3606 )
3607 } else {
3608 format!(
3609 "{} way tie on both first-choice votes and Borda points, broken by label order",
3610 leaders.len()
3611 )
3612 });
3613 leaders = borda_leaders;
3614 leaders.sort_unstable();
3615 }
3616 let winner = *leaders
3617 .first()
3618 .or(viable.first())
3619 .context("no candidate to declare a winner from")?;
3620
3621 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3622 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3623 let deliberated = !self.state.deliberation.is_empty();
3624
3625 let quota_seats: std::collections::BTreeSet<&str> =
3629 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3630 let mut present = 0usize;
3631 for (i, j) in self.state.judgements.iter().enumerate() {
3632 if quota_seats.contains(j.seat.as_str()) {
3633 continue;
3634 }
3635 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3636 let voted = self
3637 .state
3638 .votes
3639 .iter()
3640 .any(|v| v.judge == i + 1 && v.vote.is_some());
3641 if ranked || voted {
3642 present += 1;
3643 }
3644 }
3645 let needs_quorum = viable.len() > 1;
3651 let judges_total = if needs_quorum {
3652 self.roles.judges.len()
3653 } else {
3654 0
3655 };
3656 let quorum = if needs_quorum {
3657 judges_total / 2 + 1
3658 } else {
3659 0
3660 };
3661 let met_quorum = !needs_quorum || present >= quorum;
3662 let uncontested = (!needs_quorum).then(|| {
3663 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3664 });
3665
3666 self.state.event(
3667 "tally",
3668 match &uncontested {
3669 Some(reason) => format!("winner {winner} — {reason}"),
3670 None => format!(
3671 "winner {winner} — votes {} | initial {} | {} changed | \
3672 {present}/{judges_total} judges{}",
3673 first_choice
3674 .iter()
3675 .map(|(k, v)| format!("{k}:{v}"))
3676 .collect::<Vec<_>>()
3677 .join(" "),
3678 if unanimous_initial {
3679 "unanimous"
3680 } else {
3681 "split"
3682 },
3683 changed_votes,
3684 if met_quorum {
3685 String::new()
3686 } else {
3687 format!(" — below quorum ({quorum} required)")
3688 },
3689 ),
3690 },
3691 );
3692 if !met_quorum {
3693 self.state.event(
3694 "stall",
3695 format!(
3696 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3697 the run stops here, resumable"
3698 ),
3699 );
3700 }
3701 self.state.tally = Some(Tally {
3702 first_choice,
3703 borda,
3704 winner,
3705 rankings: tops.len(),
3706 unanimous_initial,
3707 deliberated,
3708 changed_votes,
3709 unanimous_final,
3710 tie_break,
3711 judges: judges_total,
3712 present,
3713 quorum,
3714 met_quorum,
3715 uncontested,
3716 });
3717 self.state.status = if met_quorum {
3718 RunStatus::Reviewing
3719 } else {
3720 RunStatus::Stalled
3721 };
3722 self.state.save()?;
3723 Ok(())
3724 }
3725
3726 #[allow(clippy::too_many_lines)]
3747 async fn recover_stall(&mut self) -> Result<bool> {
3748 let run_id = self.state.id.clone();
3753 let prompts = self.state.config.prompts.clone();
3754 let quota_seats: BTreeSet<&str> =
3759 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3760 let absent: Vec<String> = self
3761 .state
3762 .judgements
3763 .iter()
3764 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3765 .map(|j| j.seat.clone())
3766 .collect();
3767 if absent.is_empty() {
3768 return Ok(false);
3769 }
3770 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3771 if viable.len() <= 1 {
3772 return Ok(false);
3773 }
3774 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3775 let language = self.state.config.graph.language.clone();
3776 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3777 let sessions = self.state.config.graph.sessions;
3778 let artifacts = agent::artifacts_dir(&self.state.dir());
3779 let root = self.state.worktree_root();
3780 let base_short = short(&self.state.base_commit);
3781 let candidates: Vec<Candidate> = viable.clone();
3782
3783 let mut positions: Vec<usize> = absent
3785 .iter()
3786 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3787 .collect();
3788 if positions.is_empty() {
3789 return Ok(false);
3790 }
3791 positions.sort_unstable();
3792 positions.dedup();
3793
3794 let mut judge_jobs = Vec::new();
3796 for &j in &positions {
3797 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3798 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3799 let seat_key = format!("judge-{}", j + 1);
3800 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3801 let seat = self.seat(&seat_key, &spec.id);
3802 judge_jobs.push(SeatJob {
3803 spec,
3804 seat,
3805 prompt: prompt::judge(
3806 &self.state.instruction,
3807 &views,
3808 self.roles.judges.len(),
3809 &base_short,
3810 &language,
3811 ),
3812 cwd: root.join(seat_key),
3813 timeout,
3814 allow_write: false,
3815 sessions,
3816 artifacts: artifacts.clone(),
3817 stem: format!("judge-{}-recover", j + 1),
3818 handover: None,
3819 });
3820 }
3821
3822 let labels_for_check = labels.clone();
3823 let mut judge_losses = Vec::new();
3824 let retries = self.state.config.graph.retries;
3825 let cache = self.state.config.cache_dir();
3826 let ctx = WaveCtx {
3827 carry_seats: false,
3828 pauses: Some((&self.pause, &self.interrupt)),
3829 run: &run_id,
3830 node: "judge",
3831 prompts: &prompts,
3832 cache: cache.as_deref(),
3833 round: None,
3834 };
3835 let results = ask_json_wave::<Ranking>(
3836 judge_jobs,
3837 Arc::clone(&self.sem),
3838 retries,
3839 &self.roles.judge_roster,
3840 &ctx,
3841 &mut judge_losses,
3842 &mut self.state,
3843 &move |r: &Ranking| r.validate(&labels_for_check),
3844 )
3845 .await;
3846 if self.state.resource_block.is_some() {
3847 self.stop_for_resource()?;
3848 return Ok(false);
3849 }
3850
3851 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3853 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3854 let agent_id = seat.agent.clone();
3855 self.state.seats.insert(seat.key.clone(), seat);
3856 let record = &mut self.state.judgements[j];
3857 match res {
3858 Ok((ranking, out)) => {
3859 record.agent = agent_id;
3860 record.ranking = ranking.normalized();
3861 record.reasons = ranking.reasons;
3862 record.strengths = ranking.strengths;
3863 record.confidence = ranking.confidence;
3864 record.failed = None;
3865 record.duration_ms = out.duration_ms;
3866 recovered.insert(j);
3867 self.state.event(
3868 "recover",
3869 format!("judge {} ranked again after the limit", j + 1),
3870 );
3871 }
3872 Err(e) => {
3873 self.state
3874 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3875 }
3876 }
3877 }
3878
3879 let mut vote_jobs = Vec::new();
3881 let mut vote_pos: Vec<usize> = Vec::new();
3882 for &j in &recovered {
3883 let seat_key = format!("judge-{}", j + 1);
3884 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3885 let seat = self.seat(&seat_key, &spec.id);
3886 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3887 let text = if has_context(&spec, &seat, sessions) {
3888 prompt::final_vote(&labels, &language)
3889 } else {
3890 full.clone()
3891 };
3892 vote_jobs.push(SeatJob {
3893 spec,
3894 seat,
3895 prompt: text,
3896 cwd: root.join(seat_key),
3897 timeout,
3898 allow_write: false,
3899 sessions,
3900 artifacts: artifacts.clone(),
3901 stem: format!("vote-judge-{}-recover", j + 1),
3902 handover: Some(full),
3903 });
3904 vote_pos.push(j);
3905 }
3906 let allowed = labels.clone();
3907 let mut vote_losses = Vec::new();
3908 let vote_retries = self.state.config.graph.retries;
3909 let vote_cache = self.state.config.cache_dir();
3910 let ctx = WaveCtx {
3911 carry_seats: false,
3912 pauses: Some((&self.pause, &self.interrupt)),
3913 run: &run_id,
3914 node: "vote",
3915 prompts: &prompts,
3916 cache: vote_cache.as_deref(),
3917 round: None,
3918 };
3919 let votes = ask_json_wave::<FinalVote>(
3920 vote_jobs,
3921 Arc::clone(&self.sem),
3922 vote_retries,
3923 &self.roles.judge_roster,
3924 &ctx,
3925 &mut vote_losses,
3926 &mut self.state,
3927 &move |v: &FinalVote| match v.label() {
3928 Some(c) if allowed.contains(&c) => Ok(()),
3929 other => bail!("vote {other:?} is not one of {allowed:?}"),
3930 },
3931 )
3932 .await;
3933 if self.state.resource_block.is_some() {
3934 self.stop_for_resource()?;
3935 return Ok(false);
3936 }
3937 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3938 let agent_id = seat.agent.clone();
3939 self.state.seats.insert(seat.key.clone(), seat);
3940 match res {
3941 Ok((v, _)) => {
3942 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3943 rec.vote = v.label();
3944 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3945 } else {
3946 self.state.votes.push(VoteRecord {
3947 judge: j + 1,
3948 agent: agent_id,
3949 vote: v.label(),
3950 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3951 changed: false,
3952 });
3953 }
3954 self.state.event(
3955 "recover",
3956 format!("judge {} voted again after the limit", j + 1),
3957 );
3958 }
3959 Err(e) => {
3960 self.state
3961 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3962 }
3963 }
3964 }
3965
3966 let recovered_keys: BTreeSet<String> = recovered
3970 .iter()
3971 .map(|&j| format!("judge-{}", j + 1))
3972 .collect();
3973 self.state
3974 .quota
3975 .retain(|q| !recovered_keys.contains(&q.seat));
3976 for loss in judge_losses.into_iter().chain(vote_losses) {
3980 if recovered_keys.contains(&loss.seat) {
3981 continue;
3982 }
3983 self.state.quota.retain(|q| q.seat != loss.seat);
3984 self.state.quota.push(loss);
3985 }
3986
3987 self.state.tally = None;
3989 self.tally()?;
3990 Ok(self
3991 .state
3992 .tally
3993 .as_ref()
3994 .map(|t| t.met_quorum)
3995 .unwrap_or(false))
3996 }
3997
3998 async fn graft(&mut self) -> Result<()> {
4015 let Err(e) = self.graft_inner().await else {
4016 return Ok(());
4017 };
4018 if e.is::<crate::repo_lock::ResourceBlocked>() {
4019 return Err(e);
4020 }
4021 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4022 self.state.status = RunStatus::Blocked;
4023 self.state.event("graft", format!("blocked: {e:#}"));
4024 self.state.save()?;
4025 return Err(e);
4026 }
4027 if let Some(rec) = self.state.graft.clone()
4032 && !rec.outcome.settled()
4033 {
4034 if let Some(winner) = self.state.winner().cloned() {
4035 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4036 if re.is::<crate::repo_lock::ResourceBlocked>() {
4037 return Err(re);
4038 }
4039 self.state.status = RunStatus::Blocked;
4040 self.state.event("graft", format!("blocked: {re:#}"));
4041 self.state.save()?;
4042 return Err(re);
4043 }
4044 }
4045 if let Some(g) = self.state.graft.as_mut() {
4046 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4047 }
4048 }
4049 self.state.event("graft", format!("skipped: {e:#}"));
4050 if let Err(e) = self.state.save() {
4051 tracing::warn!("save after graft: {e:#}");
4052 }
4053 Ok(())
4054 }
4055
4056 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4059 let _coordination = self.coordinate(wt).await?;
4060 crate::recovery::restore_repair(wt, pre)
4061 .await
4062 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4063 }
4064
4065 fn graft_skipped(&mut self, why: &str) {
4067 let message = format!("skipped: {why}");
4068 if !self
4069 .state
4070 .events
4071 .iter()
4072 .any(|e| e.node == "graft" && e.message == message)
4073 {
4074 self.state.event("graft", message);
4075 }
4076 }
4077
4078 async fn graft_inner(&mut self) -> Result<()> {
4079 if let Some(rec) = self.state.graft.clone()
4082 && !rec.outcome.settled()
4083 {
4084 if self.state.reviews.is_empty()
4088 && let Some(winner) = self.state.winner().cloned()
4089 {
4090 self.graft_restore(&winner.worktree, &rec.pre).await?;
4091 }
4092 if rec.outcome == GraftOutcome::WaitingForRestore {
4093 self.state.graft = None;
4094 self.state.save()?;
4095 } else {
4096 if let Some(g) = self.state.graft.as_mut() {
4097 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4098 }
4099 self.state.event(
4100 "graft",
4101 "rolled back: the previous round was interrupted; the winner is untouched",
4102 );
4103 self.state.save()?;
4104 return Ok(());
4105 }
4106 }
4107 let plan = match graft::plan(&self.state) {
4108 Ok(plan) => plan,
4109 Err(skip) => {
4110 if skip != graft::Skip::AlreadyDone {
4111 self.graft_skipped(skip.reason());
4112 self.state.save()?;
4113 }
4114 return Ok(());
4115 }
4116 };
4117 let Some(winner) = self.state.winner().cloned() else {
4118 return Ok(());
4119 };
4120 let wt = winner.worktree.clone();
4121 if !git::is_clean(&wt).await? {
4122 self.graft_skipped("the winner's worktree has uncommitted changes");
4123 self.state.save()?;
4124 return Ok(());
4125 }
4126 let pre = git::rev_parse(&wt, "HEAD").await?;
4127 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4128 else {
4129 self.graft_skipped("the winner's diff could not be measured");
4130 self.state.save()?;
4131 return Ok(());
4132 };
4133 let limit = graft::limit(winner_lines);
4134 self.state.graft = Some(GraftRecord {
4137 pre: pre.clone(),
4138 post: None,
4139 lines: None,
4140 limit: Some(limit),
4141 agent: None,
4142 outcome: GraftOutcome::Running,
4143 requested: graft::requested(&plan),
4144 adoption: None,
4145 adoption_error: None,
4146 });
4147 self.state.event(
4148 "graft",
4149 format!(
4150 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4151 plan.strengths
4152 .iter()
4153 .map(|(l, _, _)| l.to_string())
4154 .collect::<Vec<_>>()
4155 .join(", "),
4156 plan.winner
4157 ),
4158 );
4159 self.state.save()?;
4160
4161 let fix_prompt = prompt::graft(
4162 &self.state.instruction,
4163 plan.winner,
4164 &plan.strengths,
4165 limit,
4166 &self.state.config.graph.language,
4167 );
4168 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4169 let sessions = self.state.config.graph.sessions;
4170 let artifacts = agent::artifacts_dir(&self.state.dir());
4171 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4172 let (_, seat, out) = self
4173 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4174 SeatJob {
4175 prompt: fix_prompt.clone(),
4176 spec,
4177 seat,
4178 cwd: wt.clone(),
4179 timeout,
4180 allow_write: true,
4181 sessions,
4182 artifacts: artifacts.clone(),
4183 stem: "graft".to_owned(),
4184 handover: None,
4185 }
4186 })
4187 .await;
4188 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4189 self.state.graft.as_mut().expect("graft record").outcome =
4192 GraftOutcome::WaitingForRestore;
4193 return self.stop_for_resource();
4194 }
4195 let mut why: Option<String> = None;
4196 match &out {
4197 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4198 why = Some("the fixer produced nothing usable".to_owned());
4199 }
4200 AgentOutcome::Ok(_) => {}
4201 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4202 AgentOutcome::Quota(o) => {
4203 self.state.quota.push(QuotaLoss {
4204 seat: seat.key.clone(),
4205 node: "graft".to_owned(),
4206 at: Timestamp::now(),
4207 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4208 });
4209 why = Some("rate limited (quota)".to_owned());
4210 }
4211 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4212 why = Some(format!(
4213 "the fixer failed: {}",
4214 e.lines()
4215 .next()
4216 .unwrap_or("")
4217 .chars()
4218 .take(160)
4219 .collect::<String>()
4220 ));
4221 }
4222 }
4223 let agent_id = seat.agent.clone();
4224 self.state.seats.insert(seat.key.clone(), seat);
4225
4226 let mut post = None;
4227 let mut lines = None;
4228 let _coordination = self.coordinate(&wt).await?;
4229 if why.is_none() {
4230 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4231 if let Ok(r) = &rescued {
4232 self.state.note_withheld("graft", &r.withheld);
4233 }
4234 let head = git::rev_parse(&wt, "HEAD").await?;
4235 let dirty = match &rescued {
4239 Ok(r) => git::status_porcelain(&wt)
4240 .await
4241 .map(|st| {
4242 st.lines().any(|l| {
4243 !r.withheld
4244 .iter()
4245 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4246 })
4247 })
4248 .unwrap_or(true),
4249 Err(_) => true,
4250 };
4251 if dirty {
4252 why = Some("the fixer left changes that could not be committed".to_owned());
4253 } else if head == pre {
4254 why = Some("the fixer changed nothing".to_owned());
4255 } else {
4256 lines = git::changed_lines(&wt, &pre, &head).await;
4257 if graft::within_budget(lines, limit) {
4258 post = Some(head);
4259 } else {
4260 why = Some(format!(
4261 "{} changed lines against a limit of {limit}",
4262 lines.map_or_else(
4263 || "an unmeasurable number of".to_owned(),
4264 |l| l.to_string()
4265 )
4266 ));
4267 }
4268 }
4269 }
4270 let (adoption, adoption_error) = match &out {
4271 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4272 _ => (None, None),
4273 };
4274 let requested = graft::requested(&plan);
4275 let outcome = match why {
4276 None => {
4277 self.state.event(
4278 "graft",
4279 format!(
4280 "applied: {} changed lines (limit {limit})",
4281 lines.unwrap_or(0)
4282 ),
4283 );
4284 GraftOutcome::Applied
4285 }
4286 Some(why) => {
4287 crate::recovery::restore_repair(&wt, &pre)
4290 .await
4291 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4292 self.state.event(
4293 "graft",
4294 format!("rolled back: {why}; the winner is untouched"),
4295 );
4296 GraftOutcome::RolledBack(why)
4297 }
4298 };
4299 self.state.graft = Some(GraftRecord {
4300 pre,
4301 post,
4302 lines,
4303 limit: Some(limit),
4304 agent: Some(agent_id),
4305 outcome,
4306 requested,
4307 adoption,
4308 adoption_error,
4309 });
4310 self.state.save()?;
4311 Ok(())
4312 }
4313
4314 async fn fold_losers(&mut self) -> Result<()> {
4317 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4318 return Ok(());
4319 };
4320 let repo = self.state.repo.clone();
4321 let _coordination = self.coordinate(&repo).await?;
4322 let mut folded = Vec::new();
4323 for i in 0..self.state.candidates.len() {
4324 let c = &self.state.candidates[i];
4325 if c.label == winner || c.folded {
4326 continue;
4327 }
4328 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4329 git::worktree_remove(&repo, &wt).await.ok();
4330 git::branch_delete(&repo, &branch).await.ok();
4331 self.state.candidates[i].folded = true;
4332 folded.push(label.to_string());
4333 }
4334 let root = self.state.worktree_root();
4336 for j in 1..=self.roles.judges.len() {
4337 let wt = root.join(format!("judge-{j}"));
4338 if wt.exists() {
4339 git::worktree_remove(&repo, &wt).await.ok();
4340 }
4341 }
4342 if self.state.config.graph.advise {
4345 for k in 1..=self.state.config.graph.advisors {
4346 let wt = root.join(format!("advisor-{k}"));
4347 if wt.exists() {
4348 git::worktree_remove(&repo, &wt).await.ok();
4349 }
4350 }
4351 }
4352 if !folded.is_empty() {
4353 self.state
4354 .event("fold", format!("folded candidates {}", folded.join(", ")));
4355 self.state.save()?;
4356 }
4357 Ok(())
4358 }
4359
4360 async fn sync_to_base(&mut self) -> Result<()> {
4400 if self.state.status == RunStatus::AlreadyInBase {
4401 return Ok(());
4402 }
4403 let conflicted = self
4404 .state
4405 .base_sync
4406 .as_ref()
4407 .is_some_and(|s| s.conflict.is_some());
4408 let Some(winner) = self.state.winner().cloned() else {
4409 return Ok(());
4410 };
4411
4412 let repo = self.state.repo.clone();
4413 let remote = self.state.config.merge.remote.clone();
4414 let base_branch = self.state.base_branch.clone();
4415 let tracking = format!("{remote}/{base_branch}");
4416
4417 git::fetch(&repo, &remote, &base_branch).await.ok();
4418 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4423 return Ok(());
4424 };
4425
4426 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4427 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4428 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4429
4430 if (behind > 0 || conflicted || head == tip)
4439 && self
4440 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4441 .await?
4442 {
4443 return Ok(());
4444 }
4445 if conflicted {
4446 return Ok(());
4447 }
4448
4449 if behind == 0 {
4450 if let Some(from) = self
4459 .state
4460 .rebase_fixes
4461 .iter()
4462 .rev()
4463 .find_map(|r| r.from.clone())
4464 && from != head
4465 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4466 .await
4467 .is_ok_and(|o| o.ok())
4468 {
4469 git::sync_to_head(&winner.worktree).await?;
4470 }
4471 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4477 if let Some(why) = &conflict {
4478 self.state.status = RunStatus::Blocked;
4479 self.state.event("land", why.clone());
4480 }
4481 self.state.base_sync = Some(BaseSync {
4482 tip,
4483 behind: 0,
4484 attempts,
4485 conflict,
4486 already_in: None,
4487 });
4488 self.state.save()?;
4489 return Ok(());
4490 }
4491
4492 if attempts >= BASE_SYNC_ROUNDS {
4493 let why = format!(
4494 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4495 rebase(s); rebasing again would only race it",
4496 winner.branch
4497 );
4498 self.state.status = RunStatus::Blocked;
4499 self.state.base_sync = Some(BaseSync {
4500 tip,
4501 behind,
4502 attempts,
4503 conflict: Some(why.clone()),
4504 already_in: None,
4505 });
4506 self.state.event("land", why);
4507 self.state.save()?;
4508 return Ok(());
4509 }
4510
4511 self.state.event(
4512 "land",
4513 format!(
4514 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4515 winner.branch
4516 ),
4517 );
4518 self.state.save()?;
4519
4520 let branch_tracking = format!("{remote}/{}", winner.branch);
4526 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4527 let remote_tip = if self.state.task_pr_branch().is_none()
4528 && matches!(&fetched_branch, Ok(o) if o.ok())
4529 {
4530 git::rev_parse(&repo, &branch_tracking).await.ok()
4531 } else {
4532 None
4533 };
4534 if let Some(theirs) = &remote_tip
4538 && !git::is_ancestor(&repo, theirs, &head).await
4539 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4540 .await
4541 .is_ok_and(|missing| missing.is_empty())
4542 {
4543 let why = format!(
4544 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4545 them",
4546 short(theirs),
4547 winner.branch
4548 );
4549 self.state.status = RunStatus::Blocked;
4550 self.state.base_sync = Some(BaseSync {
4551 tip,
4552 behind,
4553 attempts,
4554 conflict: Some(why.clone()),
4555 already_in: None,
4556 });
4557 self.state.event("land", why);
4558 self.state.save()?;
4559 return Ok(());
4560 }
4561
4562 let scratch = self.state.dir().join("base-sync");
4563 let rebased = match crate::rebase::rebase_with_fixer(
4564 &mut self.state,
4565 &scratch,
4566 &winner.branch,
4567 &tracking,
4568 )
4569 .await
4570 {
4571 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4572 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4573 Err(e) => Err(e),
4574 };
4575 let attempts = attempts + 1;
4576 match rebased {
4577 Ok(None) => {
4578 git::sync_to_head(&winner.worktree).await?;
4582 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4583 let mut conflict = None;
4584 if let Some(pinned) = &remote_tip {
4585 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4586 match pushed {
4587 Ok(o) if o.ok() => self.state.event(
4588 "land",
4589 format!("pushed rebased {} to {remote}", winner.branch),
4590 ),
4591 Ok(o) => {
4592 conflict = Some(format!(
4593 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4594 winner.branch,
4595 short(pinned),
4596 o.stderr.chars().take(600).collect::<String>()
4597 ));
4598 }
4599 Err(e) => {
4600 conflict = Some(format!(
4601 "rebased {} locally but could not push it: {e:#}",
4602 winner.branch
4603 ));
4604 }
4605 }
4606 }
4607 if let Some(why) = &conflict {
4608 self.state.status = RunStatus::Blocked;
4609 self.state.event("land", why.clone());
4610 }
4611 self.state.base_sync = Some(BaseSync {
4612 tip: tip.clone(),
4613 behind: 0,
4614 attempts,
4615 conflict,
4616 already_in: None,
4617 });
4618 self.state
4619 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4620 }
4621 Ok(Some(conflict)) => {
4622 let why = format!(
4623 "{} conflicts with {tracking} and did not rebase: {}",
4624 winner.branch,
4625 conflict.chars().take(600).collect::<String>()
4626 );
4627 self.state.status = RunStatus::Blocked;
4628 self.state.base_sync = Some(BaseSync {
4629 tip,
4630 behind,
4631 attempts,
4632 conflict: Some(why.clone()),
4633 already_in: None,
4634 });
4635 self.state.event("land", why);
4636 }
4637 Err(e) => {
4638 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4639 self.state.status = RunStatus::Blocked;
4640 self.state.base_sync = Some(BaseSync {
4641 tip,
4642 behind,
4643 attempts,
4644 conflict: Some(why.clone()),
4645 already_in: None,
4646 });
4647 self.state.event("land", why);
4648 }
4649 }
4650 self.state.save()?;
4651 Ok(())
4652 }
4653
4654 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4660 if self.state.task_pr_branch().is_some() {
4663 return None;
4664 }
4665 let repo = self.state.repo.clone();
4666 let remote = self.state.config.merge.remote.clone();
4667 let fetched = git::fetch(&repo, &remote, branch).await;
4668 if !matches!(&fetched, Ok(o) if o.ok()) {
4669 return None;
4670 }
4671 let branch_tracking = format!("{remote}/{branch}");
4672 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4673 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4674 return None;
4675 }
4676 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4677 .await
4678 .is_ok_and(|missing| missing.is_empty())
4679 {
4680 return Some(format!(
4681 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4682 them",
4683 short(&theirs)
4684 ));
4685 }
4686 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4687 Ok(o) if o.ok() => {
4688 self.state
4689 .event("land", format!("pushed rebased {branch} to {remote}"));
4690 None
4691 }
4692 Ok(o) => Some(format!(
4693 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4694 someone may have pushed): {}",
4695 short(&theirs),
4696 o.stderr.chars().take(600).collect::<String>()
4697 )),
4698 Err(e) => Some(format!(
4699 "{branch} is rebased locally but could not be pushed: {e:#}"
4700 )),
4701 }
4702 }
4703
4704 async fn settle_already_in(
4721 &mut self,
4722 branch: &str,
4723 tip: &str,
4724 head: &str,
4725 attempts: usize,
4726 behind: usize,
4727 ) -> Result<bool> {
4728 let repo = self.state.repo.clone();
4729 let remote = self.state.config.merge.remote.clone();
4730 let start = self.state.base_commit.clone();
4731 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4732 Ok(Some(e)) => e,
4733 Ok(None) => return Ok(false),
4734 Err(e) => {
4735 tracing::warn!("already-in-base check for {branch}: {e:#}");
4736 return Ok(false);
4737 }
4738 };
4739 let mut verified = vec![head.to_owned()];
4740 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4741 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4742 if matches!(&fetched, Ok(o) if o.ok())
4743 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4744 && theirs != head
4745 {
4746 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4747 Ok(Some(_)) => verified.push(theirs),
4748 _ => return Ok(false),
4749 }
4750 }
4751 let base_branch = self.state.base_branch.clone();
4752 let message = format!(
4753 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4754 land",
4755 evidence.names(),
4756 evidence.proof.as_str()
4757 );
4758 let closed =
4759 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4760 .await;
4761 match closed {
4762 Ok(Ok(url)) => self
4763 .state
4764 .event("land", format!("closed {url}: superseded on {base_branch}")),
4765 Ok(Err(why)) => self
4766 .state
4767 .event("land", format!("did not close a pull request: {why}")),
4768 Err(e) => {
4769 let why = format!(
4770 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4771 not be closed ({e:#}); resume to retry"
4772 );
4773 self.state.status = RunStatus::Blocked;
4774 self.state.base_sync = Some(BaseSync {
4775 tip: tip.to_owned(),
4776 behind,
4777 attempts,
4778 conflict: Some(why.clone()),
4779 already_in: None,
4780 });
4781 self.state.event("land", why);
4782 self.state.save()?;
4783 return Ok(true);
4784 }
4785 }
4786 self.state.status = RunStatus::AlreadyInBase;
4787 self.state.base_sync = Some(BaseSync {
4788 tip: tip.to_owned(),
4789 behind,
4790 attempts,
4791 conflict: None,
4792 already_in: Some(evidence),
4793 });
4794 self.state.event("land", message);
4795 self.state.save()?;
4796 self.settle_questions();
4797 Ok(true)
4798 }
4799
4800 fn landing_base(&self) -> String {
4810 self.state
4811 .base_sync
4812 .as_ref()
4813 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4814 }
4815
4816 pub async fn fix_selected(
4849 &mut self,
4850 ids: &[String],
4851 reason: &str,
4852 allow_stale: bool,
4853 ) -> Result<()> {
4854 let reason = reason.trim();
4855 if reason.is_empty() {
4856 bail!("a fix request needs a reason — that is the operator's own record of why");
4857 }
4858 if ids.is_empty() {
4859 bail!("no finding id given");
4860 }
4861 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4862 bail!(
4863 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4864 has already concluded — can be given a targeted fix. A run still \
4865 in progress should simply be resumed; a `merged` run's branch has \
4866 already landed, so its answer is a fresh `magi review <branch>`, \
4867 not reopening this run's own record",
4868 self.state.id,
4869 self.state.status.as_str()
4870 );
4871 }
4872 let Some(winner) = self.state.winner().cloned() else {
4873 bail!("run {} has no winning candidate to fix", self.state.id);
4874 };
4875 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4876 bail!(
4877 "branch `{}` no longer exists; this run cannot be extended",
4878 winner.branch
4879 );
4880 }
4881 let home = crate::run::home();
4882 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4883 bail!(
4884 "run {} is currently being worked on by another magi process",
4885 self.state.id
4886 );
4887 }
4888 let _claim = FixClaim::acquire(&self.state.dir())?;
4894
4895 let mut seen = BTreeSet::new();
4899 let mut findings = Vec::new();
4900 let mut missing = Vec::new();
4901 for id in ids {
4902 if !seen.insert(id.clone()) {
4903 continue;
4904 }
4905 match self.state.finding(id) {
4906 Some((round, rec, f)) => findings.push(OperatorFixFinding {
4907 id: f.id.clone(),
4908 severity: f.severity,
4909 reviewer_vote: rec.vote,
4910 round: round.round,
4911 round_head: round.head.clone(),
4912 reviewer: rec.reviewer,
4913 agent: rec.agent.clone(),
4914 file: f.file.clone(),
4915 line: f.line,
4916 title: f.title.clone(),
4917 detail: f.detail.clone(),
4918 outcome: OperatorFixOutcome::Pending,
4919 }),
4920 None => missing.push(id.clone()),
4921 }
4922 }
4923 if !missing.is_empty() {
4924 bail!(
4925 "unknown finding id(s): {}; nothing was changed",
4926 missing.join(", ")
4927 );
4928 }
4929
4930 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
4931 let stale_details: Vec<(String, String)> = findings
4932 .iter()
4933 .filter(|f| f.round_head != head_at_request)
4934 .map(|f| (f.id.clone(), f.round_head.clone()))
4935 .collect();
4936 let stale = !stale_details.is_empty();
4937 if stale && !allow_stale {
4938 bail!(
4939 "the branch has moved since some finding(s) were raised — {} — now \
4940 at {}; pass --allow-stale to fix anyway, or re-run review first",
4941 stale_details
4942 .iter()
4943 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4944 .collect::<Vec<_>>()
4945 .join(", "),
4946 short(&head_at_request)
4947 );
4948 }
4949
4950 let request = OperatorFixRequest {
4951 requested_at: Timestamp::now(),
4952 reason: reason.to_owned(),
4953 findings,
4954 head_at_request: head_at_request.clone(),
4955 allow_stale,
4956 stale,
4957 fix: None,
4958 result_head: None,
4959 follow_up_review_run: None,
4960 };
4961 self.state.event(
4962 "fix",
4963 format!(
4964 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4965 request.findings.len(),
4966 request
4967 .findings
4968 .iter()
4969 .map(|f| f.id.as_str())
4970 .collect::<Vec<_>>()
4971 .join(", "),
4972 ),
4973 );
4974 self.state.operator_fixes.push(request);
4981 self.state.save()?;
4982 let request_index = self.state.operator_fixes.len() - 1;
4983
4984 let repo = self.state.repo.clone();
4993 let coordination = self.coordinate(&repo).await?;
4994 if winner.worktree.exists() {
4995 let dirty = git::git(
4998 &winner.worktree,
4999 &["status", "--porcelain", "--untracked-files=all"],
5000 )
5001 .await?;
5002 let only_withheld = dirty.lines().all(|l| {
5003 l.strip_prefix("?? ")
5004 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5005 });
5006 if !only_withheld {
5007 bail!(
5008 "`{}` has uncommitted changes; refusing to touch it — commit or \
5009 discard them first",
5010 winner.worktree.display()
5011 );
5012 }
5013 git::worktree_remove(&self.state.repo, &winner.worktree)
5014 .await
5015 .ok();
5016 }
5017 let fix_worktree = self.state.worktree_root().join("operator-fix");
5018 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5019 git::git(
5020 &self.state.repo,
5021 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5022 )
5023 .await
5024 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5025 if !git::is_clean(&fix_worktree).await? {
5026 git::worktree_remove(&self.state.repo, &fix_worktree)
5027 .await
5028 .ok();
5029 bail!(
5030 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5031 winner.branch
5032 );
5033 }
5034 let repo_for_setup = self.state.repo.clone();
5035 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5036 Ok(()) => {
5037 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5038 }
5039 Err(e) => Err(e),
5040 };
5041 if let Err(e) = setup {
5042 git::worktree_remove(&self.state.repo, &fix_worktree)
5043 .await
5044 .ok();
5045 return Err(e);
5046 }
5047
5048 let run_id = self.state.id.clone();
5049 let prompts = self.state.config.prompts.clone();
5050 let language = self.state.config.graph.language.clone();
5051 let sessions = self.state.config.graph.sessions;
5052 let artifacts = agent::artifacts_dir(&self.state.dir());
5053 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5054 .findings
5055 .iter()
5056 .map(|f| Finding {
5057 id: f.id.clone(),
5058 severity: f.severity,
5059 file: f.file.clone(),
5060 line: f.line,
5061 title: f.title.clone(),
5062 detail: f.detail.clone(),
5063 })
5064 .collect();
5065 let fix_prompt = prompt::operator_fix_in(
5066 &self.state.instruction,
5067 &finding_list,
5068 reason,
5069 &stale_details,
5070 &head_at_request,
5071 &language,
5072 &self.state.config.graph.github_language,
5073 );
5074 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5075 drop(coordination);
5076 let (job, seat, out) = self
5077 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5078 prompt: fix_prompt.clone(),
5079 spec,
5080 seat,
5081 cwd: fix_worktree.clone(),
5082 timeout,
5083 allow_write: true,
5084 sessions,
5085 artifacts: artifacts.clone(),
5086 stem: "operator-fix".to_owned(),
5087 handover: None,
5088 })
5089 .await;
5090 let agent_id = seat.agent.clone();
5091
5092 let mut fix = FixRecord {
5093 agent: agent_id,
5094 addressed: Vec::new(),
5095 rejected: Vec::new(),
5096 notes: String::new(),
5097 committed: false,
5098 failed: None,
5099 duration_ms: 0,
5100 continuation: None,
5101 };
5102 let mut final_seat = seat.clone();
5103 match out {
5104 AgentOutcome::Ok(o) => {
5105 fix.duration_ms = o.duration_ms;
5106 let parsed = verdict::extract_json::<FixReport>(&o.text);
5107 let incomplete_reason = match &parsed {
5108 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5109 "the reply parsed, but it reported a command whose own CLI \
5110 never confirmed an exit status"
5111 .to_owned(),
5112 ),
5113 Ok(_) => None,
5114 Err(e) => Some(e.to_string()),
5115 };
5116 match incomplete_reason {
5117 None => {
5118 let report = parsed.expect("checked Ok above");
5119 fix.addressed = report.addressed;
5120 fix.rejected = report.rejected;
5121 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5122 }
5123 Some(reason) => {
5124 let (resumed_seat, resolved, failure, cont) = self
5125 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5126 .await;
5127 fix.duration_ms += cont.cumulative_wait_ms;
5128 fix.continuation = Some(cont);
5129 final_seat = resumed_seat;
5130 match resolved {
5131 Some(report) => {
5132 fix.addressed = report.addressed;
5133 fix.rejected = report.rejected;
5134 fix.notes =
5135 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5136 }
5137 None => fix.failed = failure,
5138 }
5139 }
5140 }
5141 }
5142 AgentOutcome::Dropped(o) => {
5143 fix.duration_ms = o.duration_ms;
5144 let why = o
5145 .dropped
5146 .as_ref()
5147 .map(|d| d.why.as_str())
5148 .unwrap_or("the CLI ended the stream without delivering its answer");
5149 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5150 }
5151 AgentOutcome::Quota(o) => {
5152 self.state.quota.push(QuotaLoss {
5153 seat: final_seat.key.clone(),
5154 node: "fix".to_owned(),
5155 at: Timestamp::now(),
5156 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5157 });
5158 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5159 }
5160 AgentOutcome::Failed(e) => fix.failed = Some(e),
5161 AgentOutcome::ResourceBlocked(e) => {
5162 fix.failed = Some(format!("the fixer was not started: {e}"));
5163 }
5164 }
5165 if fix.continuation.is_none() {
5166 fix.continuation = Some(ContinuationRecord::not_needed());
5167 }
5168 self.state.seats.insert(final_seat.key.clone(), final_seat);
5169
5170 let rescue_message = format!(
5171 "magi: operator-selected fix ({}) (uncommitted work)",
5172 self.state.operator_fixes[request_index]
5173 .findings
5174 .iter()
5175 .map(|f| f.id.as_str())
5176 .collect::<Vec<_>>()
5177 .join(", ")
5178 );
5179 let repo = self.state.repo.clone();
5180 let coordination = self.coordinate(&repo).await?;
5181 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5182 self.state.note_withheld("fix", &r.withheld);
5183 }
5184 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5185 fix.committed = after != head_at_request;
5186 git::worktree_remove(&self.state.repo, &fix_worktree)
5187 .await
5188 .ok();
5189 drop(coordination);
5190
5191 self.state.event(
5192 "fix",
5193 match &fix.failed {
5194 Some(reason) => format!(
5195 "operator fix: adoption report was lost ({reason}); {}",
5196 if fix.committed {
5197 "committed"
5198 } else {
5199 "NO new commit"
5200 }
5201 ),
5202 None => format!(
5203 "operator fix: {} addressed, {} rejected, {}",
5204 fix.addressed.len(),
5205 fix.rejected.len(),
5206 if fix.committed {
5207 "committed"
5208 } else {
5209 "NO new commit"
5210 }
5211 ),
5212 },
5213 );
5214
5215 for f in &mut self.state.operator_fixes[request_index].findings {
5222 f.outcome = if fix.failed.is_some() {
5223 OperatorFixOutcome::Unreported
5224 } else if fix.addressed.contains(&f.id) {
5225 OperatorFixOutcome::Addressed
5226 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5227 OperatorFixOutcome::Rejected { why: r.why.clone() }
5228 } else {
5229 OperatorFixOutcome::Unreported
5230 };
5231 }
5232
5233 let committed = fix.committed;
5234 if committed {
5235 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5236 }
5237 self.state.operator_fixes[request_index].fix = Some(fix);
5238 self.state.save()?;
5241
5242 if committed {
5243 self.state.event(
5244 "fix",
5245 format!(
5246 "operator fix committed {}; opening a follow-up review-only run",
5247 short(&after)
5248 ),
5249 );
5250 let origin =
5253 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5254 match Self::review(
5255 &self.state.repo,
5256 &winner.branch,
5257 self.state.config.clone(),
5258 origin,
5259 )
5260 .await
5261 {
5262 Ok(mut follow_up) => {
5263 follow_up.state.event(
5264 "start",
5265 format!(
5266 "requested by an operator fix on run {} for finding(s) {}",
5267 self.state.id,
5268 self.state.operator_fixes[request_index]
5269 .findings
5270 .iter()
5271 .map(|f| f.id.as_str())
5272 .collect::<Vec<_>>()
5273 .join(", "),
5274 ),
5275 );
5276 follow_up.state.save()?;
5277 let follow_up_id = follow_up.state.id.clone();
5278 let adopted = match crate::direct::adopt(
5281 &follow_up.state,
5282 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5283 ) {
5284 Ok(a) => a,
5285 Err(e) => {
5286 self.state.event(
5289 "fix",
5290 format!(
5291 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5292 ),
5293 );
5294 self.state.save()?;
5295 return Ok(());
5296 }
5297 };
5298 let executed = follow_up.execute().await;
5299 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5300 if let Some(a) = adopted {
5301 a.finish(&follow_up.state, executed);
5302 }
5303 if let Some(e) = failure {
5304 self.state.event(
5305 "fix",
5306 format!(
5307 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5308 ),
5309 );
5310 }
5311 self.state.operator_fixes[request_index].follow_up_review_run =
5312 Some(follow_up_id);
5313 }
5314 Err(e) => {
5315 self.state.event(
5316 "fix",
5317 format!("committed the fix but could not open a follow-up review: {e:#}"),
5318 );
5319 }
5320 }
5321 self.state.save()?;
5322 }
5323
5324 Ok(())
5325 }
5326
5327 async fn ask_fixer(
5347 &mut self,
5348 winner: &Candidate,
5349 node: &'static str,
5350 round: Option<usize>,
5351 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5352 ) -> (SeatJob, SeatState, AgentOutcome) {
5353 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5354 self.ask_fixer_among(attempts, node, round, None, build)
5355 .await
5356 }
5357
5358 async fn ask_fixer_among(
5363 &mut self,
5364 attempts: Vec<(AgentSpec, String)>,
5365 node: &'static str,
5366 round: Option<usize>,
5367 restore: Option<(&Path, &str)>,
5368 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5369 ) -> (SeatJob, SeatState, AgentOutcome) {
5370 let run_id = self.state.id.clone();
5371 let prompts = self.state.config.prompts.clone();
5372 let cache = self.state.config.cache_dir();
5373 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5374 let mut last = None;
5375 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5376 let seat = self.seat(&key, &spec.id);
5377 let mut job = build(spec.clone(), seat);
5378 if i > 0 {
5379 job.stem = format!("{}-{}", job.stem, spec.id);
5380 }
5381 let ctx = WaveCtx {
5382 carry_seats: false,
5383 pauses: Some((&self.pause, &self.interrupt)),
5384 run: &run_id,
5385 node,
5386 prompts: &prompts,
5387 cache: cache.as_deref(),
5388 round,
5389 };
5390 let (seat, out) =
5391 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5392 let advances = match &out {
5395 AgentOutcome::Ok(o) => agent::output_advances(o),
5396 AgentOutcome::ResourceBlocked(_) => false,
5397 _ => true,
5398 };
5399 if let Some(next) = ids.get(i + 1)
5400 && advances
5401 {
5402 self.state.seats.insert(seat.key.clone(), seat.clone());
5403 let class =
5404 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5405 let reason = match &out {
5406 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5407 other => fail_reason(other),
5408 };
5409 if let Some((wt, rev)) = restore
5413 && let Err(e) = self.graft_restore(wt, rev).await
5414 {
5415 let why =
5416 format!("could not restore the worktree before the next fixer: {e:#}");
5417 tracing::warn!("{why}");
5418 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5419 AgentOutcome::ResourceBlocked(why)
5420 } else {
5421 AgentOutcome::Failed(why)
5422 };
5423 last = Some((job, seat, out));
5424 break;
5425 }
5426 record_handover(
5427 &mut self.state,
5428 node,
5429 &seat.key,
5430 &spec.id,
5431 next,
5432 &class,
5433 &reason,
5434 );
5435 continue;
5436 }
5437 last = Some((job, seat, out));
5438 break;
5439 }
5440 last.expect("the fixer chain always has an entry")
5441 }
5442
5443 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5454 use crate::recovery::{Mode, Repair};
5455 if !self.state.reviews.is_empty() {
5456 return Ok(());
5457 }
5458 let Some(mut rec) = self.state.recovery.clone() else {
5459 return Ok(());
5460 };
5461 if rec.mode != Mode::VerifyRepair {
5462 return Ok(());
5463 }
5464 let wt = winner.worktree.clone();
5465 let mut head = git::rev_parse(&wt, "HEAD").await?;
5466 match rec.repair.clone() {
5467 Repair::Done { .. } => return Ok(()),
5468 Repair::WaitingForRestore { before } => {
5469 self.graft_restore(&wt, &before).await?;
5470 head = before;
5471 rec.repair = Repair::NotStarted;
5472 self.state.recovery = Some(rec.clone());
5473 self.state.save()?;
5474 }
5475 Repair::Started { before } => {
5476 if let Err(e) = self.graft_restore(&wt, &before).await {
5480 if e.is::<crate::repo_lock::ResourceBlocked>() {
5481 return Err(e);
5482 }
5483 return self
5484 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5485 .await;
5486 }
5487 rec.repair = Repair::Done {
5488 before: before.clone(),
5489 after: before,
5490 committed: false,
5491 failed: Some("interrupted; rolled back".to_owned()),
5492 };
5493 self.state
5494 .event("recovery", "the repair was interrupted and rolled back");
5495 self.state.recovery = Some(rec);
5496 self.state.save()?;
5497 return Ok(());
5498 }
5499 Repair::NotStarted => {}
5500 }
5501 if crate::recovery::recheck(&mut rec, &head) {
5502 self.state.event("recovery", rec.summary());
5503 self.state.recovery = Some(rec);
5504 self.state.save()?;
5505 return Ok(());
5506 }
5507 rec.repair = Repair::Started {
5508 before: head.clone(),
5509 };
5510 self.state.recovery = Some(rec.clone());
5511 self.state.save()?;
5512
5513 let language = self.state.config.graph.language.clone();
5514 let sessions = self.state.config.graph.sessions;
5515 let artifacts = agent::artifacts_dir(&self.state.dir());
5516 let fix_prompt = prompt::recovery_fix(
5517 &self.state.instruction,
5518 &rec,
5519 self.state.config.graph.review_rounds,
5520 &language,
5521 );
5522 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5523 self.state.event(
5524 "recovery",
5525 format!(
5526 "repair: the fixer starts from the failed verification on {}",
5527 short(&head)
5528 ),
5529 );
5530 let (_job, seat, out) = self
5531 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5532 prompt: fix_prompt.clone(),
5533 spec,
5534 seat,
5535 cwd: wt.clone(),
5536 timeout,
5537 allow_write: true,
5538 sessions,
5539 artifacts: artifacts.clone(),
5540 stem: "fix-recovery".to_owned(),
5541 handover: None,
5542 })
5543 .await;
5544 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5545 rec.repair = Repair::WaitingForRestore { before: head };
5546 self.state.recovery = Some(rec);
5547 return self.stop_for_resource();
5548 }
5549 let failed = match &out {
5550 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5551 AgentOutcome::Quota(o) => {
5552 self.state.quota.push(QuotaLoss {
5553 seat: seat.key.clone(),
5554 node: "fix".to_owned(),
5555 at: Timestamp::now(),
5556 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5557 });
5558 Some("rate limited (quota); the fixer could not run".to_owned())
5559 }
5560 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5561 AgentOutcome::Failed(e) => Some(e.clone()),
5562 };
5563 self.state.seats.insert(seat.key.clone(), seat);
5564 let coordination = self.coordinate(&wt).await?;
5565 let failed = match failed {
5566 Some(why) => {
5568 if let Err(e) = git::restore_to(&wt, &head).await {
5569 return self
5570 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5571 .await;
5572 }
5573 Some(why)
5574 }
5575 None => {
5576 if let Ok(r) = git::rescue_commit(
5577 &wt,
5578 "magi: recovery repair of the failed verification (uncommitted work)",
5579 )
5580 .await
5581 {
5582 self.state.note_withheld("fix", &r.withheld);
5583 }
5584 None
5585 }
5586 };
5587 let after = git::rev_parse(&wt, "HEAD").await?;
5588 drop(coordination);
5589 self.state.event(
5590 "recovery",
5591 match &failed {
5592 Some(why) => format!("repair: nothing kept ({why})"),
5593 None if after != head => {
5594 format!("repair: committed {}", short(&after))
5595 }
5596 None => "repair: the fixer made no commit".to_owned(),
5597 },
5598 );
5599 rec.repair = Repair::Done {
5600 before: head.clone(),
5601 committed: after != head,
5602 after,
5603 failed,
5604 };
5605 self.state.recovery = Some(rec);
5606 self.state.save()?;
5607 Ok(())
5608 }
5609
5610 async fn stop_recovery(
5613 &mut self,
5614 mut rec: crate::recovery::RecoveryRecord,
5615 head: &str,
5616 why: String,
5617 ) -> Result<()> {
5618 rec.repair = crate::recovery::Repair::Started {
5620 before: head.to_owned(),
5621 };
5622 self.state.recovery = Some(rec);
5623 self.state
5624 .event("recovery", format!("repair could not be undone: {why}"));
5625 self.state.status = RunStatus::Blocked;
5626 self.state.save()?;
5627 bail!("recovery repair could not be rolled back: {why}")
5628 }
5629
5630 async fn review_loop(&mut self) -> Result<()> {
5631 if self
5636 .state
5637 .base_sync
5638 .as_ref()
5639 .is_some_and(|s| s.conflict.is_some())
5640 {
5641 return Ok(());
5642 }
5643 let run_id = self.state.id.clone();
5648 let prompts = self.state.config.prompts.clone();
5649 let Some(winner) = self.state.winner().cloned() else {
5650 return Ok(());
5651 };
5652 let max_rounds = self.state.config.graph.review_rounds;
5653 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5663 if status == RunStatus::Gating {
5666 self.record_contested_handoff();
5667 }
5668 self.state.status = status;
5669 self.state.save()?;
5670 return Ok(());
5671 }
5672 self.state.status = RunStatus::Reviewing;
5673 if self
5683 .state
5684 .reviews
5685 .last()
5686 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5687 {
5688 let shell = self.state.config.shell();
5689 return self
5690 .stop_reviewing(
5691 "the last round's own verification never resolved",
5692 &shell,
5693 &winner.worktree,
5694 )
5695 .await;
5696 }
5697
5698 self.recovery_repair(&winner).await?;
5699 if self.resource_halt()? {
5700 return Ok(());
5701 }
5702
5703 let repo = self.state.repo.clone();
5704 let root = self.state.worktree_root();
5705 let language = self.state.config.graph.language.clone();
5706 let sessions = self.state.config.graph.sessions;
5707 let artifacts = self.merge_repair_artifacts();
5708 let base = self.landing_base();
5709 let base_short = short(&base);
5710 let reviewers = self.roles.reviewers.clone();
5711 let shell = self.state.config.shell();
5712
5713 for round in (self.state.reviews.len() + 1)..=max_rounds {
5714 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5715 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5716 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5717 let prev_verification = self
5726 .state
5727 .reviews
5728 .last()
5729 .and_then(|r| r.verification_summary(&head));
5730 let prior_section = match (&self.state.recovery, round) {
5733 (Some(rec), 1) if self.state.reviews.is_empty() => {
5734 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5735 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5736 Some(crate::recovery::head_of(
5737 &git::diff(&winner.worktree, reviewed, "HEAD")
5738 .await
5739 .unwrap_or_default(),
5740 crate::recovery::SINCE_DIFF_MAX,
5741 ))
5742 }
5743 _ => None,
5744 };
5745 Some(prompt::recovery_review(rec, since.as_deref()))
5746 }
5747 _ => None,
5748 };
5749
5750 let mut jobs = Vec::new();
5754 let coordination = self.coordinate(&repo).await?;
5755 for (r, spec) in reviewers.iter().cloned().enumerate() {
5756 let wt = root.join(format!("review-{}", r + 1));
5757 if wt.exists() {
5760 git::reset_detached(&wt, &head).await?;
5761 } else {
5762 git::worktree_add_detached(&repo, &wt, &head).await?;
5763 }
5764 self.setup_seat_worktree(&repo, &wt, false).await?;
5765 let seat_key = format!("review-{}", r + 1);
5766 let spec = pick_start_spec(
5769 &self.roles.reviewer_roster,
5770 spec,
5771 self.state.seat_history.get(&seat_key),
5772 );
5773 let seat = self.seat(&seat_key, &spec.id);
5774 jobs.push(SeatJob {
5775 prompt: prompt::review(&prompt::ReviewCtx {
5776 instruction: &self.state.instruction,
5777 branch: &winner.branch,
5778 base_short: &base_short,
5779 stat: &stat,
5780 patch: &patch,
5781 verification: prev_verification.as_ref(),
5782 prior: prior_section.as_deref(),
5783 reviewers: reviewers.len(),
5784 round,
5785 rounds: max_rounds,
5786 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5789 lens: Lens::for_seat(r),
5790 language: &language,
5791 github_language: &self.state.config.graph.github_language,
5792 }),
5793 spec,
5794 seat,
5795 cwd: wt,
5796 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5797 allow_write: false,
5798 sessions,
5799 artifacts: artifacts.clone(),
5800 stem: format!("review-{round}-{}", r + 1),
5801 handover: None,
5802 });
5803 }
5804
5805 drop(coordination);
5806 self.state.event(
5807 "review",
5808 format!(
5809 "round {round}: {} reviewers on {}",
5810 jobs.len(),
5811 short(&head)
5812 ),
5813 );
5814 let mut quota_losses = Vec::new();
5815 let review_retries = self.state.config.graph.retries;
5816 let review_cache = self.state.config.cache_dir();
5817 let ctx = WaveCtx {
5818 carry_seats: true,
5819 pauses: Some((&self.pause, &self.interrupt)),
5820 run: &run_id,
5821 node: "review",
5822 prompts: &prompts,
5823 cache: review_cache.as_deref(),
5824 round: Some(round),
5825 };
5826 let results = ask_json_wave::<Review>(
5827 jobs,
5828 Arc::clone(&self.sem),
5829 review_retries,
5830 &self.roles.reviewer_roster,
5831 &ctx,
5832 &mut quota_losses,
5833 &mut self.state,
5834 &|_: &Review| Ok(()),
5835 )
5836 .await;
5837 if self.state.resource_block.is_some() {
5838 return self.stop_for_resource();
5839 }
5840 let round_quota_missing = quota_losses.len();
5844 self.state.quota.extend(quota_losses);
5845
5846 let mut records = Vec::new();
5847 let mut all_findings = Vec::new();
5848 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5849 let agent_id = seat.agent.clone();
5850 self.state.seats.insert(seat.key.clone(), seat);
5851 let mut record = ReviewRecord {
5852 reviewer: r + 1,
5853 agent: agent_id,
5854 summary: String::new(),
5855 findings: Vec::new(),
5856 vote: None,
5857 failed: None,
5858 duration_ms: 0,
5859 attempts,
5865 };
5866 match res {
5867 Ok((review, out)) => {
5868 record.summary =
5876 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5877 record.vote = Some(review.vote);
5878 record.duration_ms = out.duration_ms;
5879 for (n, mut f) in review.findings.into_iter().enumerate() {
5880 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5883 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5884 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5885 f.file = f
5891 .file
5892 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
5893 all_findings.push(f.clone());
5894 record.findings.push(f);
5895 }
5896 self.state.event(
5897 "review",
5898 format!(
5899 "round {round}: reviewer {} voted {} with {} finding(s)",
5900 r + 1,
5901 review.vote.label(),
5902 record.findings.len()
5903 ),
5904 );
5905 }
5906 Err(e) => {
5907 record.failed = Some(e.to_string());
5908 self.state.event(
5909 "review",
5910 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
5911 );
5912 }
5913 }
5914 records.push(record);
5915 }
5916
5917 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
5924 let vote_split =
5925 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
5926 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
5927 if vote_split {
5928 self.state.event(
5929 "review",
5930 format!(
5931 "round {round}: votes split ({}) — one round of reconsideration",
5932 initial_votes
5933 .iter()
5934 .map(|v| v.label())
5935 .collect::<Vec<_>>()
5936 .join(", ")
5937 ),
5938 );
5939 let panel: Vec<ReviewSeatReport<'_>> = records
5942 .iter()
5943 .filter_map(|r| {
5944 r.vote.map(|vote| ReviewSeatReport {
5945 reviewer: r.reviewer,
5946 vote,
5947 summary: &r.summary,
5948 findings: &r.findings,
5949 })
5950 })
5951 .collect();
5952
5953 let mut jobs = Vec::new();
5954 let mut seats_at = Vec::new();
5955 for (r, spec) in reviewers.iter().cloned().enumerate() {
5956 if records[r].vote.is_none() {
5960 continue;
5961 }
5962 let wt = root.join(format!("review-{}", r + 1));
5963 let seat_key = format!("review-{}", r + 1);
5964 let spec = self.occupant(&seat_key, spec);
5965 let seat = self.seat(&seat_key, &spec.id);
5966 let build = |with_patch: bool| {
5974 prompt::review_reconsider(&ReviewReconsiderCtx {
5975 instruction: &self.state.instruction,
5976 reviewer: r + 1,
5977 lens: Lens::for_seat(r),
5978 panel: &panel,
5979 patch: with_patch.then_some(ReviewPatch {
5980 branch: &winner.branch,
5981 base_short: &base_short,
5982 stat: &stat,
5983 patch: &patch,
5984 }),
5985 round,
5986 rounds: max_rounds,
5987 language: &language,
5988 })
5989 };
5990 let full = build(true);
5991 let prompt = if has_context(&spec, &seat, sessions) {
5992 build(false)
5993 } else {
5994 full.clone()
5995 };
5996 jobs.push(SeatJob {
5997 prompt,
5998 spec,
5999 seat,
6000 cwd: wt,
6001 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6002 allow_write: false,
6003 sessions,
6004 artifacts: artifacts.clone(),
6005 stem: format!("review-{round}-reconsider-{}", r + 1),
6006 handover: Some(full),
6007 });
6008 seats_at.push(r);
6009 }
6010
6011 let mut recon_quota_losses = Vec::new();
6012 let recon_cache = self.state.config.cache_dir();
6013 let recon_ctx = WaveCtx {
6014 carry_seats: true,
6015 pauses: Some((&self.pause, &self.interrupt)),
6016 run: &run_id,
6017 node: "review",
6018 prompts: &prompts,
6019 cache: recon_cache.as_deref(),
6020 round: Some(round),
6021 };
6022 let recon_results = ask_json_wave::<ReviewRevote>(
6023 jobs,
6024 Arc::clone(&self.sem),
6025 review_retries,
6026 &self.roles.reviewer_roster,
6027 &recon_ctx,
6028 &mut recon_quota_losses,
6029 &mut self.state,
6030 &|_: &ReviewRevote| Ok(()),
6031 )
6032 .await;
6033 if self.state.resource_block.is_some() {
6034 return self.stop_for_resource();
6035 }
6036 self.state.quota.extend(recon_quota_losses);
6037
6038 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6039 let agent_id = seat.agent.clone();
6040 self.state.seats.insert(seat.key.clone(), seat);
6041 let mut rec = ReviewRevoteRecord {
6042 reviewer: r + 1,
6043 agent: agent_id,
6044 vote: None,
6045 reason: String::new(),
6046 failed: None,
6047 };
6048 match res {
6049 Ok((rv, _)) => {
6050 rec.vote = Some(rv.vote);
6051 rec.reason =
6052 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6053 self.state.event(
6054 "review",
6055 format!(
6056 "round {round}: reviewer {} revoted {}",
6057 r + 1,
6058 rv.vote.label()
6059 ),
6060 );
6061 }
6062 Err(e) => {
6063 rec.failed = Some(e.to_string());
6064 self.state.event(
6065 "review",
6066 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6067 );
6068 }
6069 }
6070 reconsideration.push(rec);
6071 }
6072 } else if initial_votes.len() > 1 {
6073 self.state.event(
6074 "review",
6075 format!(
6076 "round {round}: votes agreed ({}) — no reconsideration",
6077 initial_votes[0].label()
6078 ),
6079 );
6080 }
6081
6082 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6083 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6084 let defer_e2e =
6095 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6096 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6097 let reason =
6098 format!("{blocking} blocking finding(s) already required a fix this round");
6099 self.state.event(
6100 "verify",
6101 format!(
6102 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6103 {}); it will run once a round has none left",
6104 short(&head)
6105 ),
6106 );
6107 (Vec::new(), false, true, Some(reason))
6108 } else {
6109 let e2e_commands = self.state.config.verify.e2e.clone();
6110 let cache_dir = self.state.config.cache_dir();
6111 let context = format!("round {round}");
6112 let wait = self.repository_wait();
6113 let (e2e, verify_retried) = with_cache_lease(
6114 &mut self.state,
6115 cache_dir.as_deref(),
6116 "e2e",
6117 "e2e",
6118 &winner.worktree,
6119 &head,
6120 verify_timeout,
6121 &context,
6122 &wait,
6123 |state, budget| {
6124 let shell = shell.clone();
6125 let e2e_commands = e2e_commands.clone();
6126 let worktree = winner.worktree.clone();
6127 let context = context.clone();
6128 async move {
6129 run_e2e_with_retry(
6130 state,
6131 &shell,
6132 &e2e_commands,
6133 &worktree,
6134 budget,
6135 &context,
6136 )
6137 .await
6138 }
6139 },
6140 )
6141 .await?;
6142 (e2e, verify_retried, false, None)
6143 };
6144
6145 let expected = records.len();
6146 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6147 let incomplete = answered < expected;
6148 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6149 let policy = self.state.config.graph.incomplete_review;
6150 let clean = round_is_clean(
6151 blocking,
6152 e2e_ok,
6153 answered,
6154 expected,
6155 round_quota_missing,
6156 policy,
6157 );
6158
6159 let mut round_record = ReviewRound {
6160 round,
6161 head: head.clone(),
6162 verified_head: None,
6163 verified_at: None,
6164 reviews: records,
6165 e2e,
6166 verify_retried,
6167 e2e_deferred,
6168 e2e_defer_reason,
6169 fix: None,
6170 blocking,
6171 answered,
6172 expected,
6173 clean,
6174 progressed: false,
6175 vote_split,
6176 reconsideration,
6177 verdict: None,
6178 };
6179 round_record.verdict = ReviewVote::worst(
6183 round_record
6184 .final_votes()
6185 .into_iter()
6186 .map(|(_, _, vote)| vote),
6187 );
6188 if !matches!(
6199 round_record.e2e_status(),
6200 E2eStatus::Deferred | E2eStatus::NotConfigured
6201 ) {
6202 round_record.verified_head = Some(head.clone());
6203 round_record.verified_at = Some(Timestamp::now());
6204 }
6205 let this_round_verification = round_record.verification_summary(&head);
6206
6207 if incomplete {
6208 let missing: Vec<String> = round_record
6209 .reviews
6210 .iter()
6211 .filter(|r| r.failed.is_some())
6212 .map(|r| format!("review-{}", r.reviewer))
6213 .collect();
6214 self.state.event(
6215 "review",
6216 format!(
6217 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6218 missing.join(", ")
6219 ),
6220 );
6221 }
6222
6223 if clean {
6224 self.state.event(
6225 "review",
6226 if incomplete && policy == IncompleteReviewPolicy::Warn {
6227 format!(
6228 "round {round}: clean (warn policy, incomplete panel) — no \
6229 blocking findings from the seats that answered, verification green"
6230 )
6231 } else if incomplete {
6232 format!(
6233 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6234 quorum) — no blocking findings from the seats that answered, \
6235 verification green",
6236 expected - answered
6237 )
6238 } else {
6239 format!("round {round}: clean — no blocking findings, verification green")
6240 },
6241 );
6242 self.state.reviews.push(round_record);
6243 self.state.status = RunStatus::Gating;
6244 self.state.save()?;
6245 return Ok(());
6246 }
6247
6248 if incomplete && blocking == 0 && e2e_ok {
6256 self.state.reviews.push(round_record);
6257 self.state.save()?;
6258 if round == max_rounds {
6259 self.state.status = RunStatus::Blocked;
6260 self.state.event(
6261 "review",
6262 format!(
6263 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6264 refusing to call it clean",
6265 expected - answered
6266 ),
6267 );
6268 return Ok(());
6269 }
6270 continue;
6271 }
6272
6273 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6285 self.state.reviews.push(round_record);
6286 return self
6287 .stop_reviewing(
6288 "the round's own verification could not run",
6289 &shell,
6290 &winner.worktree,
6291 )
6292 .await;
6293 }
6294
6295 if round == max_rounds {
6296 self.state.reviews.push(round_record);
6297 return self
6298 .stop_reviewing(
6299 &format!(
6300 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6301 ),
6302 &shell,
6303 &winner.worktree,
6304 )
6305 .await;
6306 }
6307
6308 let blocking_findings: Vec<_> = all_findings
6311 .iter()
6312 .filter(|f| f.severity.blocks())
6313 .cloned()
6314 .collect();
6315 let fix_prompt = prompt::fix_in(
6316 &self.state.instruction,
6317 &blocking_findings,
6318 this_round_verification.as_ref(),
6319 round,
6320 max_rounds,
6321 &language,
6322 &self.state.config.graph.github_language,
6323 );
6324 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6325 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6326 let (job, seat, out) = self
6327 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6328 prompt: fix_prompt.clone(),
6329 spec,
6330 seat,
6331 cwd: winner.worktree.clone(),
6332 timeout,
6333 allow_write: true,
6334 sessions,
6335 artifacts: artifacts.clone(),
6336 stem: format!("fix-{round}"),
6337 handover: None,
6338 })
6339 .await;
6340 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6343 return self.stop_for_resource();
6344 }
6345 let agent_id = seat.agent.clone();
6346
6347 let mut fix = FixRecord {
6348 agent: agent_id,
6349 addressed: Vec::new(),
6350 rejected: Vec::new(),
6351 notes: String::new(),
6352 committed: false,
6353 failed: None,
6354 duration_ms: 0,
6355 continuation: None,
6356 };
6357 let mut continuation = ContinuationRecord::not_needed();
6358 let mut final_seat = seat.clone();
6359 match out {
6360 AgentOutcome::Ok(o) => {
6361 fix.duration_ms = o.duration_ms;
6362 let parsed = verdict::extract_json::<FixReport>(&o.text);
6363 let incomplete_reason = match &parsed {
6370 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6371 "the reply parsed, but it reported a command whose own CLI never \
6372 confirmed an exit status"
6373 .to_owned(),
6374 ),
6375 Ok(_) => None,
6376 Err(e) => Some(e.to_string()),
6377 };
6378 match incomplete_reason {
6379 None => {
6380 let report = parsed.expect("checked Ok above");
6381 fix.addressed = report.addressed;
6382 fix.rejected = report.rejected;
6383 fix.notes =
6384 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6385 }
6386 Some(reason) => {
6387 let (resumed_seat, resolved, failure, cont) = self
6388 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6389 .await;
6390 fix.duration_ms += cont.cumulative_wait_ms;
6391 continuation = cont;
6392 final_seat = resumed_seat;
6393 match resolved {
6394 Some(report) => {
6395 fix.addressed = report.addressed;
6396 fix.rejected = report.rejected;
6397 fix.notes = blind::sanitize_prose(
6398 &report.notes,
6399 &self.state.config.blind,
6400 );
6401 }
6402 None => fix.failed = failure,
6403 }
6404 }
6405 }
6406 }
6407 AgentOutcome::Dropped(o) => {
6409 fix.duration_ms = o.duration_ms;
6410 let why = o
6411 .dropped
6412 .as_ref()
6413 .map(|d| d.why.as_str())
6414 .unwrap_or("the CLI ended the stream without delivering its answer");
6415 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6416 }
6417 AgentOutcome::Quota(o) => {
6418 self.state.quota.push(QuotaLoss {
6419 seat: final_seat.key.clone(),
6420 node: "fix".to_owned(),
6421 at: Timestamp::now(),
6422 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6423 });
6424 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6425 }
6426 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6427 }
6428 fix.continuation = Some(continuation);
6429 self.state.seats.insert(final_seat.key.clone(), final_seat);
6430 let coordination = self.coordinate(&winner.worktree).await?;
6431 if let Ok(r) = git::rescue_commit(
6432 &winner.worktree,
6433 &format!("magi: review round {round} fixes (uncommitted work)"),
6434 )
6435 .await
6436 {
6437 self.state.note_withheld("fix", &r.withheld);
6438 }
6439 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6440 fix.committed = after != before;
6441 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6449 drop(coordination);
6450 let progressed = diff_after != patch;
6451 let commit_note = if fix.committed {
6452 "committed"
6453 } else {
6454 "NO new commit"
6455 };
6456 let tree_note = if progressed {
6457 "changed vs base"
6458 } else {
6459 "unchanged vs base"
6460 };
6461 self.state.event(
6462 "fix",
6463 match &fix.failed {
6464 Some(reason) => {
6470 format!(
6471 "round {round}: fixer's adoption report was lost ({reason}); \
6472 {commit_note}, tree {tree_note}"
6473 )
6474 }
6475 None => format!(
6476 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6477 {tree_note}{}",
6478 fix.addressed.len(),
6479 fix.rejected.len(),
6480 if continuation.outcome == ContinuationOutcome::Resumed {
6481 format!(
6482 " (adoption report recovered after {} continuation(s))",
6483 continuation.attempts
6484 )
6485 } else {
6486 String::new()
6487 },
6488 ),
6489 },
6490 );
6491 round_record.fix = Some(fix);
6492 round_record.progressed = progressed;
6493 self.state.reviews.push(round_record);
6494 self.state.save()?;
6495
6496 if matches!(
6509 continuation.outcome,
6510 ContinuationOutcome::Exhausted
6511 | ContinuationOutcome::QuotaLost
6512 | ContinuationOutcome::NoSession
6513 ) {
6514 return self
6515 .stop_reviewing(
6516 "the fixer's adoption report never came back, even after resuming its \
6517 own seat; refusing to start another round against the same worktree \
6518 while that is unresolved",
6519 &shell,
6520 &winner.worktree,
6521 )
6522 .await;
6523 }
6524
6525 let streak = self
6526 .state
6527 .reviews
6528 .iter()
6529 .rev()
6530 .take_while(|r| !r.progressed)
6531 .count();
6532 if streak >= STAGNANT_LIMIT {
6533 return self
6534 .stop_reviewing(
6535 &format!(
6536 "the tree has not moved against base for {streak} round(s) in a row"
6537 ),
6538 &shell,
6539 &winner.worktree,
6540 )
6541 .await;
6542 }
6543 }
6544 Ok(())
6545 }
6546
6547 fn record_contested_handoff(&mut self) {
6581 if self.state.contested_handoff.is_some() {
6582 return;
6583 }
6584 let Some(contested) = self
6585 .state
6586 .reviews
6587 .last()
6588 .and_then(ReviewRound::contested_handoff)
6589 else {
6590 return;
6591 };
6592 self.state.event(
6593 "review",
6594 format!(
6595 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6596 wait for the owner's approval",
6597 contested.findings.len(),
6598 contested.rejecters.len()
6599 ),
6600 );
6601 self.state.contested_handoff = Some(contested);
6602 }
6603
6604 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6605 let round_idx = self.state.reviews.len() - 1;
6606 let needs_catchup_run = matches!(
6614 self.state.reviews[round_idx].e2e_status(),
6615 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6616 );
6617 if needs_catchup_run {
6618 let round = self.state.reviews[round_idx].round;
6619 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6620 let commands = self.state.config.verify.e2e.clone();
6621 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6622 let cache_dir = self.state.config.cache_dir();
6623 let context = format!(
6624 "round {round}: verification unresolved, catching up before the final decision"
6625 );
6626 let wait = self.repository_wait();
6627 let (outcomes, verify_retried) = with_cache_lease(
6628 &mut self.state,
6629 cache_dir.as_deref(),
6630 "e2e",
6631 "e2e",
6632 worktree,
6633 &attempted_head,
6634 timeout,
6635 &context,
6636 &wait,
6637 |state, budget| {
6638 let shell = shell.to_vec();
6639 let commands = commands.clone();
6640 let context = context.clone();
6641 async move {
6642 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6643 .await
6644 }
6645 },
6646 )
6647 .await?;
6648 let last = &mut self.state.reviews[round_idx];
6649 last.e2e = outcomes;
6650 last.verify_retried = verify_retried;
6651 last.verified_head = Some(attempted_head);
6658 last.verified_at = Some(Timestamp::now());
6659 if verify_inconclusive(&last.e2e) {
6660 self.state.save()?;
6667 return Ok(());
6668 }
6669 last.e2e_deferred = false;
6670 }
6671 let last = &self.state.reviews[round_idx];
6672 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6673
6674 match last.e2e_status() {
6675 E2eStatus::Failed => {
6676 let red: Vec<String> = last
6677 .e2e
6678 .iter()
6679 .filter(|o| !o.ok())
6680 .map(|o| {
6681 format!(
6682 "`{}` -> {:?}\n{}",
6683 o.command,
6684 o.code,
6685 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6686 )
6687 })
6688 .collect();
6689 self.state
6690 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6691 self.state.status = RunStatus::Blocked;
6692 }
6693 E2eStatus::ResourceBlocked => {
6698 self.state.event(
6699 "review",
6700 format!(
6701 "{why}; e2e could not run (shared build cache unavailable); not \
6702 deciding yet"
6703 ),
6704 );
6705 }
6706 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6707 self.state.event(
6708 "review",
6709 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6710 );
6711 self.record_contested_handoff();
6712 self.state.status = RunStatus::Gating;
6713 }
6714 }
6715 self.state.save()?;
6716 Ok(())
6717 }
6718
6719 async fn gate(&mut self) -> Result<()> {
6722 if self.state.status == RunStatus::Failed
6734 || self
6735 .state
6736 .base_sync
6737 .as_ref()
6738 .is_some_and(|s| s.conflict.is_some())
6739 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6740 != Some(RunStatus::Gating)
6741 {
6742 return Ok(());
6743 }
6744 if self.state.gate_ran {
6745 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6756 self.state.status = RunStatus::Blocked;
6757 self.state.save()?;
6758 }
6759 return Ok(());
6760 }
6761 let Some(winner) = self.state.winner().cloned() else {
6762 return Ok(());
6763 };
6764 self.state.status = RunStatus::Gating;
6765 let mut outcomes = self.run_gate(&winner).await?;
6766 loop {
6767 if verify_inconclusive(&outcomes) {
6778 self.state.save()?;
6779 return Ok(());
6780 }
6781 if outcomes.iter().all(CommandOutcome::ok) {
6782 break;
6783 }
6784 match self.gate_fix_round(&winner, &outcomes).await? {
6785 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6786 GateFix::Stop => break,
6787 GateFix::Defer => {
6788 self.state.save()?;
6789 return Ok(());
6790 }
6791 }
6792 }
6793 let passed = outcomes.iter().all(CommandOutcome::ok);
6794 self.state.gate = outcomes;
6795 self.state.gate_ran = true;
6796 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6797 if !passed {
6798 self.state.status = RunStatus::Blocked;
6799 let spent = self.state.gate_fixes.len();
6800 self.state.event(
6801 "gate",
6802 if spent == 0 {
6803 "gate failed; not merging".to_owned()
6804 } else {
6805 format!("gate failed after {spent} gate-fix round(s); not merging")
6806 },
6807 );
6808 }
6809 self.state.save()?;
6810 Ok(())
6811 }
6812
6813 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6823 let commands = state.config.verify.pre_gate.clone();
6824 if commands.is_empty() {
6825 return;
6826 }
6827 let shell = state.config.shell();
6828 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6829 let (outcomes, _) = run_commands(
6830 state,
6831 "pre_gate",
6832 "pre_gate",
6833 0,
6834 &shell,
6835 &commands,
6836 &winner.worktree,
6837 timeout,
6838 )
6839 .await;
6840 for o in &outcomes {
6841 if !o.ok() {
6842 tracing::warn!(
6843 "pre_gate `{}` failed ({:?}); the gate decides",
6844 o.command,
6845 o.code
6846 );
6847 }
6848 state.event(
6849 "pre_gate",
6850 format!(
6851 "`{}` -> {}",
6852 o.command,
6853 if o.ok() {
6854 "pass".to_owned()
6855 } else {
6856 format!(
6857 "FAIL ({:?})\n{}",
6858 o.code,
6859 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6860 )
6861 }
6862 ),
6863 );
6864 }
6865 state.pre_gate = outcomes;
6866 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6867 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6868 Ok(head) => {
6869 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6870 state.pre_gate_commit = Some(head);
6871 }
6872 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6873 },
6874 Ok(false) => {}
6875 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6876 }
6877 if let Err(e) = state.save() {
6878 tracing::warn!("could not persist the pre_gate record: {e:#}");
6879 }
6880 }
6881
6882 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6885 let shell = self.state.config.shell();
6886 let gate_commands = self.state.config.verify.gate.clone();
6887 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
6888 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
6889 Vec::new()
6890 } else {
6891 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6892 let cache_dir = self.state.config.cache_dir();
6893 let wait = self.repository_wait();
6894 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6895 let (outcomes, _) = with_cache_lease(
6896 &mut self.state,
6897 cache_dir.as_deref(),
6898 "gate",
6899 "gate",
6900 &winner.worktree,
6901 &head,
6902 timeout,
6903 "final gate",
6904 &wait,
6905 |state, budget| {
6906 let shell = shell.clone();
6907 let gate_commands = gate_commands.clone();
6908 let winner = winner.clone();
6909 async move {
6910 Self::run_pre_gate(state, &winner).await;
6911 let (outcomes, timed_out_pids) = run_commands(
6912 state,
6913 "gate",
6914 "gate",
6915 0,
6916 &shell,
6917 &gate_commands,
6918 &winner.worktree,
6919 budget,
6920 )
6921 .await;
6922 (outcomes, false, timed_out_pids)
6923 }
6924 },
6925 )
6926 .await?;
6927 outcomes
6928 };
6929 if outcomes.is_empty() {
6930 self.state.event(
6935 "gate",
6936 "no gate commands configured; nothing to check, passing",
6937 );
6938 }
6939 for o in &outcomes {
6940 self.state.event(
6941 "gate",
6942 format!(
6943 "`{}` -> {}",
6944 o.command,
6945 if o.ok() {
6946 "pass".to_owned()
6947 } else {
6948 format!(
6949 "FAIL ({:?})\n{}",
6950 o.code,
6951 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6952 )
6953 }
6954 ),
6955 );
6956 }
6957 Ok(outcomes)
6958 }
6959
6960 async fn gate_fix_round(
6973 &mut self,
6974 winner: &Candidate,
6975 outcomes: &[CommandOutcome],
6976 ) -> Result<GateFix> {
6977 let cap = self.state.config.graph.gate_fix_rounds;
6978 let spent = self.state.gate_fixes.len();
6979 if spent >= cap {
6980 if cap > 0 {
6981 self.state.event(
6982 "gate",
6983 format!("{spent} gate-fix round(s) spent and the gate still fails"),
6984 );
6985 }
6986 return Ok(GateFix::Stop);
6987 }
6988 if !gate_fixable(outcomes) {
6989 self.state.event(
6990 "gate",
6991 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
6992 command or similar); not spending a fix round on it",
6993 );
6994 return Ok(GateFix::Stop);
6995 }
6996 let min_free = self.state.config.disk.min_free_bytes;
6997 if min_free > 0 {
6998 match crate::disk::free_bytes(&winner.worktree) {
6999 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7000 Ok(free) => {
7001 self.state.event(
7002 "gate",
7003 format!(
7004 "only {free} bytes free ({min_free} required by `[disk] \
7005 min_free_bytes`); not spending a fix round on a failure the disk \
7006 may explain"
7007 ),
7008 );
7009 return Ok(GateFix::Stop);
7010 }
7011 Err(e) => {
7012 self.state.event(
7013 "gate",
7014 format!("free disk space could not be measured ({e:#}); no fix round"),
7015 );
7016 return Ok(GateFix::Stop);
7017 }
7018 }
7019 }
7020
7021 let attempt = spent + 1;
7022 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7023 let base = self.landing_base();
7024 let fix_prompt = prompt::gate_fix_in(
7025 &self.state.instruction,
7026 &failed,
7027 attempt,
7028 cap,
7029 &self.state.config.graph.language,
7030 &self.state.config.graph.github_language,
7031 );
7032 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7033 let sessions = self.state.config.graph.sessions;
7034 let artifacts = self.merge_repair_artifacts();
7035 self.state.event(
7036 "gate",
7037 format!("gate failed; gate-fix round {attempt} of {cap}"),
7038 );
7039 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7040 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7041 let (_, seat, out) = self
7042 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7043 prompt: fix_prompt.clone(),
7044 spec,
7045 seat,
7046 cwd: winner.worktree.clone(),
7047 timeout,
7048 allow_write: true,
7049 sessions,
7050 artifacts: artifacts.clone(),
7051 stem: format!("gate-fix-{attempt}"),
7052 handover: None,
7053 })
7054 .await;
7055 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7058 return Ok(GateFix::Defer);
7059 }
7060 let mut record = GateFixRecord {
7061 agent: seat.agent.clone(),
7062 failed,
7063 notes: String::new(),
7064 committed: false,
7065 error: None,
7066 };
7067 match out {
7068 AgentOutcome::Ok(o) => {
7069 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7072 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7073 }
7074 }
7075 AgentOutcome::Dropped(_) => {
7076 record.error = Some("the CLI dropped the stream".to_owned());
7077 }
7078 AgentOutcome::Quota(o) => {
7079 self.state.quota.push(QuotaLoss {
7080 seat: seat.key.clone(),
7081 node: "gate-fix".to_owned(),
7082 at: Timestamp::now(),
7083 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7084 });
7085 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7086 }
7087 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7088 }
7089 self.state.seats.insert(seat.key.clone(), seat);
7090 let coordination = self.coordinate(&winner.worktree).await?;
7091 if let Ok(r) = git::rescue_commit(
7092 &winner.worktree,
7093 &format!("magi: gate fix {attempt} (uncommitted work)"),
7094 )
7095 .await
7096 {
7097 self.state.note_withheld("gate-fix", &r.withheld);
7098 }
7099 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7100 record.committed = after != before;
7101 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7102 let note = record.error.clone();
7103 self.state.gate_fixes.push(record);
7104 self.state.save()?;
7105 if !changed {
7106 self.state.event(
7107 "gate",
7108 match note {
7109 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7110 None => format!("gate-fix round {attempt}: the tree did not change"),
7111 },
7112 );
7113 return Ok(GateFix::Stop);
7114 }
7115 self.state.event(
7116 "gate",
7117 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7118 );
7119
7120 drop(coordination);
7121 let commands = self.state.config.verify.e2e.clone();
7122 if !commands.is_empty() {
7123 let shell = self.state.config.shell();
7124 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7125 let cache_dir = self.state.config.cache_dir();
7126 let context = format!("gate-fix round {attempt}");
7127 let wait = self.repository_wait();
7128 let (e2e, _) = with_cache_lease(
7129 &mut self.state,
7130 cache_dir.as_deref(),
7131 "e2e",
7132 "e2e",
7133 &winner.worktree,
7134 &after,
7135 timeout,
7136 &context,
7137 &wait,
7138 |state, budget| {
7139 let shell = shell.clone();
7140 let commands = commands.clone();
7141 let context = context.clone();
7142 let worktree = winner.worktree.clone();
7143 async move {
7144 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7145 .await
7146 }
7147 },
7148 )
7149 .await?;
7150 if verify_inconclusive(&e2e) {
7151 return Ok(GateFix::Defer);
7152 }
7153 if e2e.iter().any(|o| !o.ok()) {
7154 self.state.event(
7155 "gate",
7156 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7157 );
7158 return Ok(GateFix::Stop);
7159 }
7160 }
7161 Ok(GateFix::Retry)
7162 }
7163
7164 async fn guarded_pr_message(
7171 &mut self,
7172 winner: &Candidate,
7173 facts: Option<&BranchFacts>,
7174 posting: bool,
7175 ) -> Result<Option<PrMessage>> {
7176 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7177 let head = match final_head.as_deref() {
7178 Some(h) => crate::pr_outcome::FinalHead::At(h),
7179 None => crate::pr_outcome::FinalHead::Unknown,
7180 };
7181 let summarized = if posting {
7183 Some(self.summarize_change(winner, final_head.as_deref()).await)
7184 } else {
7185 None
7186 };
7187 if self.state.resource_block.is_some() {
7192 self.state.event(
7193 "pr-summary",
7194 "a shared resource was busy; nothing was published, resume to summarize again",
7195 );
7196 return Ok(None);
7197 }
7198 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7199 anyhow::bail!(
7200 "the final commit moved during public description generation; retry before publishing"
7201 );
7202 }
7203 let note = match (&summarized, &self.state.change_summary) {
7204 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7205 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7206 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7207 (None, _) => ChangeNote::NotAsked,
7208 };
7209 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7210 if !posting {
7211 let identity = crate::scrub::Identity::current();
7212 return Ok(Some(PrMessage {
7213 title: crate::scrub::scrub(&pr.title, &identity),
7214 body: crate::scrub::scrub(&pr.body, &identity),
7215 }));
7216 }
7217 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7220 if let Some(known) = self
7221 .state
7222 .github_text
7223 .clone()
7224 .filter(|g| g.fingerprint == raw_fp)
7225 {
7226 return self.settle_withheld(winner, &pr, known).await;
7227 }
7228 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7229 let mut violations =
7230 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7231 let identity = crate::scrub::Identity::current();
7232 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7233 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7234 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7235 {
7236 violations.push(crate::github_text::Violation::SensitiveData);
7237 }
7238 let has_language = violations
7242 .iter()
7243 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7244 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7245 self.state.event(
7246 "github-text",
7247 if has_language {
7248 format!("description rejected: {violations:?}; requesting one rewrite")
7249 } else {
7250 format!("description rejected: {violations:?}; asking the owner")
7251 },
7252 );
7253 let mut rewritten = false;
7254 let mut rejected_rewrite = None;
7255 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7256 {
7257 let output = text;
7258 if let Some(title) = summary_title(&output) {
7259 let raw_body = summary_without_title(&output);
7262 let clean_title = crate::scrub::scrub(&title, &identity);
7263 let mut body = crate::scrub::scrub(&raw_body, &identity);
7264 let redacted = clean_title != title || body != raw_body;
7265 let title = clean_title;
7266 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7267 if !complete {
7268 self.state.event(
7269 "github-text",
7270 "description rewrite omitted required findings",
7271 );
7272 }
7273 if !body.trim().is_empty() {
7274 body.push_str("\n\n");
7275 body.push_str(&public_pr_evidence(&self.state, head));
7276 }
7277 let marker = format!("magi:run/{}", self.state.id);
7278 if !body.trim().is_empty() && !body.contains(&marker) {
7279 body.push_str(&format!("\n\n{marker}"));
7280 }
7281 let rewrite = PrMessage {
7282 title: title.clone(),
7283 body: body.clone(),
7284 };
7285 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7286 let mut failed = crate::github_text::check_for(
7287 &self.state.config.graph,
7288 &title,
7289 &body,
7290 redecision,
7291 );
7292 if (!crate::github_text::shareable(&title)
7293 || !crate::github_text::shareable(&body))
7294 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7295 {
7296 failed.push(crate::github_text::Violation::SensitiveData);
7297 }
7298 if complete
7299 && !title.trim().is_empty()
7300 && !body.trim().is_empty()
7301 && !failed.is_empty()
7302 {
7303 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7304 title: title.clone(),
7305 body: body.clone(),
7306 failed: failed
7307 .iter()
7308 .map(|v| crate::github_text::category(*v).to_owned())
7309 .collect(),
7310 });
7311 }
7312 if complete
7313 && !title.trim().is_empty()
7314 && !raw_body.trim().is_empty()
7315 && failed.is_empty()
7316 {
7317 decision = redecision;
7318 pr = PrMessage { title, body };
7319 rewritten = true;
7320 self.state
7321 .event("github-text", "description rewrite passed");
7322 if redacted {
7323 self.state.event(
7324 "github-text",
7325 "sensitive data removed from description rewrite",
7326 );
7327 }
7328 }
7329 }
7330 }
7331 if !rewritten {
7332 self.state.event(
7333 "github-text",
7334 "description rewrite unavailable or rejected; asking the owner",
7335 );
7336 if let Some(w) =
7337 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7338 {
7339 self.retire_github_text_question("the withheld text changed");
7342 let record = crate::run::GithubTextAsk {
7343 rewrite: rejected_rewrite,
7344 fingerprint: raw_fp,
7345 title: w.title,
7346 body: w.body,
7347 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7348 question: None,
7349 asks: 0,
7350 resolved: false,
7351 chosen_title: None,
7352 choice: None,
7353 };
7354 return self.settle_withheld(winner, &pr, record).await;
7355 }
7356 }
7357 }
7358 let (title, body) =
7359 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7360 Ok(Some(PrMessage { title, body }))
7361 }
7362
7363 async fn summarize_change(
7374 &mut self,
7375 winner: &Candidate,
7376 head: Option<&str>,
7377 ) -> std::result::Result<(), String> {
7378 let Some(head) = head else {
7379 return Err("the final commit could not be read".to_owned());
7380 };
7381 let remote = self.state.config.merge.remote.clone();
7382 let tracking = format!("{remote}/{}", self.state.base_branch);
7383 if !git::rev_exists(&winner.worktree, &tracking).await {
7384 return Err(format!("the remote base `{tracking}` could not be read"));
7385 }
7386 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7387 Ok(b) if !b.is_empty() => b,
7388 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7389 };
7390 let evidence = crate::pr_outcome::summary_evidence(
7391 &self.state,
7392 crate::pr_outcome::FinalHead::At(head),
7393 );
7394 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7395 if self
7396 .state
7397 .change_summary
7398 .as_ref()
7399 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7400 {
7401 return Ok(());
7402 }
7403 if self.state.change_summary.take().is_some() {
7404 self.state.event(
7405 "pr-summary",
7406 "the final diff or evidence changed; the earlier change summary was discarded",
7407 );
7408 }
7409 if let Err(e) = write_artifact(
7411 &self.state,
7412 &format!("public-description-{fingerprint}.md"),
7413 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7414 ) {
7415 tracing::warn!("could not save public description evidence: {e:#}");
7416 }
7417 let chain = match agent::pick_chain(
7418 &self.state.config.agents,
7419 self.state.config.roles.synthesizer.as_ref(),
7420 &agent::installed,
7421 "synthesizer",
7422 ) {
7423 Ok(chain) => chain,
7424 Err(e) => {
7425 tracing::warn!("no agent for the change summary: {e:#}");
7426 self.state.event("pr-summary", "no summary agent available");
7427 return Err("no agent was available to summarize it".to_owned());
7428 }
7429 };
7430 let subjects = git::subjects(&winner.worktree, &base, head)
7431 .await
7432 .unwrap_or_default();
7433 let stat = git::diff_stat(&winner.worktree, &base, head)
7434 .await
7435 .unwrap_or_default();
7436 let stat: String = stat.chars().take(6_000).collect();
7437 let mut prompt = crate::prompt::change_summary_in(
7438 &base,
7439 head,
7440 &subjects,
7441 stat.trim(),
7442 &self.state.config.graph.github_language,
7443 );
7444 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7445 let sessions = self.state.config.graph.sessions;
7446 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7447 let prompts = self.state.config.prompts.clone();
7448 let cache = self.state.config.cache_dir();
7449 let run = self.state.id.clone();
7450 for (i, spec) in chain.into_iter().enumerate() {
7451 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7452 let job = SeatJob {
7453 spec: spec.clone(),
7454 seat,
7455 cwd: winner.worktree.clone(),
7456 prompt: prompt.clone(),
7457 timeout,
7458 allow_write: false,
7459 sessions,
7460 artifacts: agent::artifacts_dir(&self.state.dir()),
7461 stem: if i == 0 {
7462 "change-summary".to_owned()
7463 } else {
7464 format!("change-summary-{}", spec.id)
7465 },
7466 handover: None,
7467 };
7468 let ctx = WaveCtx {
7469 run: &run,
7470 node: "pr-summary",
7471 prompts: &prompts,
7472 cache: cache.as_deref(),
7473 round: None,
7474 carry_seats: false,
7475 pauses: Some((&self.pause, &self.interrupt)),
7476 };
7477 let (seat, outcome) =
7478 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7479 self.state.seats.insert(seat.key.clone(), seat);
7480 let text = match outcome {
7481 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7482 AgentOutcome::ResourceBlocked(why) => {
7483 if self.state.resource_block.is_none() {
7486 self.state.resource_block = Some(crate::run::ResourceBlock {
7487 node: "pr-summary".to_owned(),
7488 kind: "shared resource".to_owned(),
7489 detail: why,
7490 at: Timestamp::now(),
7491 });
7492 }
7493 return Err("a shared resource was busy".to_owned());
7494 }
7495 _ => continue,
7496 };
7497 let blind = &self.state.config.blind;
7498 let Some(title) = summary_title(&text) else {
7499 continue;
7500 };
7501 let body: String = summary_without_title(&text)
7502 .lines()
7503 .filter(|l| !l.contains("magi:run/"))
7504 .collect::<Vec<_>>()
7505 .join("\n");
7506 let body = blind::sanitize_prose(body.trim(), blind);
7507 let title = blind::sanitize_prose(&title, blind);
7508 if body.trim().is_empty() || title.trim().is_empty() {
7509 continue;
7510 }
7511 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7512 continue;
7513 }
7514 if git::rev_parse(&winner.worktree, "HEAD")
7516 .await
7517 .ok()
7518 .as_deref()
7519 != Some(head)
7520 {
7521 return Err("the branch moved while the summary was written".to_owned());
7522 }
7523 if git::merge_base(&winner.worktree, &tracking, head)
7524 .await
7525 .ok()
7526 .as_deref()
7527 != Some(base.as_str())
7528 || crate::pr_outcome::summary_evidence(
7529 &self.state,
7530 crate::pr_outcome::FinalHead::At(head),
7531 ) != evidence
7532 {
7533 return Err("the summary evidence changed while it was written".to_owned());
7534 }
7535 self.state.change_summary = Some(crate::run::ChangeSummary {
7536 head: head.to_owned(),
7537 base,
7538 evidence: fingerprint,
7539 title,
7540 body,
7541 });
7542 self.state.event(
7543 "pr-summary",
7544 format!("change summary written by {} for the final diff", spec.id),
7545 );
7546 if let Err(e) = self.state.save() {
7547 tracing::warn!("could not save the change summary: {e:#}");
7548 }
7549 return Ok(());
7550 }
7551 self.state
7552 .event("pr-summary", "no agent produced a change summary");
7553 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7554 }
7555
7556 async fn rewrite_pr_text(
7564 &mut self,
7565 winner: &Candidate,
7566 pr: &PrMessage,
7567 violations: &[crate::github_text::Violation],
7568 ) -> Option<String> {
7569 let prompt = format!(
7570 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7571 pr.title, pr.body
7572 );
7573 let sessions = self.state.config.graph.sessions;
7574 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7575 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7577 let key = format!("impl-{}", winner.label);
7578 if let Some(spec) = self
7579 .state
7580 .config
7581 .agents
7582 .iter()
7583 .find(|a| a.id == winner.agent)
7584 .cloned()
7585 && let Some(existing) = self.state.seats.get(&key)
7586 && existing.agent == spec.id
7587 && agent::has_session(spec.kind, existing, sessions)
7588 {
7589 attempts.push((spec, existing.clone(), true));
7590 } else {
7591 let chain = agent::pick_chain(
7592 &self.state.config.agents,
7593 self.state.config.roles.language_judge.as_ref(),
7594 &agent::installed,
7595 "language_judge",
7596 );
7597 match chain {
7598 Ok(chain) => {
7599 for spec in chain {
7600 let seat = crate::fixer::seat_for(
7601 &mut self.state,
7602 "github-text-rewrite",
7603 &spec.id,
7604 );
7605 attempts.push((spec, seat, false));
7606 }
7607 }
7608 Err(e) => {
7609 tracing::warn!("no agent for the description rewrite: {e:#}");
7610 self.state
7611 .event("github-text", "no rewrite agent available");
7612 return None;
7613 }
7614 }
7615 }
7616 let prompts = self.state.config.prompts.clone();
7617 let cache = self.state.config.cache_dir();
7618 let run = self.state.id.clone();
7619 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7620 let job = SeatJob {
7621 spec: spec.clone(),
7622 seat,
7623 cwd: winner.worktree.clone(),
7624 prompt: prompt.clone(),
7625 timeout: retry_budget(timeout_full, resumed),
7626 allow_write: false,
7627 sessions,
7628 artifacts: agent::artifacts_dir(&self.state.dir()),
7629 stem: if i == 0 {
7630 "github-text-rewrite".to_owned()
7631 } else {
7632 format!("github-text-rewrite-{}", spec.id)
7633 },
7634 handover: None,
7635 };
7636 let ctx = WaveCtx {
7637 run: &run,
7638 node: "github-text",
7639 prompts: &prompts,
7640 cache: cache.as_deref(),
7641 round: None,
7642 carry_seats: false,
7643 pauses: Some((&self.pause, &self.interrupt)),
7644 };
7645 let (seat, outcome) =
7646 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7647 self.state.seats.insert(seat.key.clone(), seat);
7648 match outcome {
7649 AgentOutcome::Ok(o)
7650 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7651 {
7652 return Some(o.text);
7653 }
7654 AgentOutcome::ResourceBlocked(_) => return None,
7655 _ => {}
7656 }
7657 }
7658 None
7659 }
7660
7661 fn retire_github_text_question(&mut self, why: &str) {
7663 let Some(id) = self
7664 .state
7665 .github_text
7666 .as_ref()
7667 .and_then(|g| g.question.clone())
7668 else {
7669 return;
7670 };
7671 let _ = ask::Questions::open().update(&id, |q| {
7672 q.abandon(why);
7673 Ok(())
7674 });
7675 }
7676
7677 async fn settle_withheld(
7681 &mut self,
7682 winner: &Candidate,
7683 pr: &PrMessage,
7684 mut g: crate::run::GithubTextAsk,
7685 ) -> Result<Option<PrMessage>> {
7686 use crate::github_text as gt;
7687 let store = ask::Questions::open();
7688 let timeout = self.state.config.graph.answer_timeout;
7689 if !g.resolved {
7690 let mut standing = g
7691 .question
7692 .as_ref()
7693 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7694 if let Some(q) = standing.as_ref()
7695 && q.status.open()
7696 && gt::expired(q, timeout)
7697 {
7698 let why = format!("no answer within {}s of asking", timeout.max(1));
7699 standing = store
7700 .update(&q.id, |q| {
7701 q.abandon(&why);
7702 Ok(())
7703 })
7704 .ok()
7705 .map(|(q, ())| q);
7706 }
7707 let mut ask_again: Option<Option<Vec<String>>> = None;
7708 match standing {
7709 None if g.asks <= 1 => {
7712 g.asks = 0;
7713 ask_again = Some(None);
7714 }
7715 None => {
7716 g.resolved = true;
7717 g.chosen_title = None;
7718 g.choice = Some(GithubTextChoice::Fallback);
7719 }
7720 Some(q) if q.status.open() => {
7721 self.park_on_github_text(&g)?;
7722 return Ok(None);
7723 }
7724 Some(q) => match gt::read_reply(&q) {
7725 gt::Reply::Fallback => {
7726 g.resolved = true;
7727 g.chosen_title = None;
7728 g.choice = Some(GithubTextChoice::Fallback);
7729 }
7730 gt::Reply::Original => {
7731 g.resolved = true;
7732 g.chosen_title = None;
7733 g.choice = Some(GithubTextChoice::Original);
7734 }
7735 gt::Reply::Rewrite => {
7736 g.resolved = true;
7737 g.chosen_title = None;
7738 g.choice = Some(if g.rewrite.is_some() {
7739 GithubTextChoice::Rewrite
7740 } else {
7741 GithubTextChoice::Fallback
7742 });
7743 }
7744 gt::Reply::Title(text) => {
7745 let first = text
7746 .lines()
7747 .map(str::trim)
7748 .find(|l| !l.is_empty())
7749 .unwrap_or("");
7750 let decision = if first.is_empty() {
7751 None
7752 } else {
7753 let candidate = PrMessage {
7754 title: first.to_owned(),
7755 body: String::new(),
7756 };
7757 self.judge_pr_language(&winner.worktree, &candidate).await
7758 };
7759 match gt::vet_title(
7760 &text,
7761 decision,
7762 self.state.config.graph.github_english(),
7763 ) {
7764 Ok(title) => {
7765 g.resolved = true;
7766 g.chosen_title = Some(title);
7767 g.choice = Some(GithubTextChoice::MyTitle);
7768 }
7769 Err(cats) if g.asks < 2 => {
7770 ask_again = Some(Some(
7771 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7772 ))
7773 }
7774 Err(_) => {
7775 g.resolved = true;
7776 g.chosen_title = None;
7777 g.choice = Some(GithubTextChoice::Fallback);
7778 }
7779 }
7780 }
7781 },
7782 }
7783 if let Some(rejected) = ask_again {
7784 let w = gt::Withheld {
7787 title: g.title,
7788 body: g.body,
7789 categories: g.categories.clone(),
7790 };
7791 let artifact = match crate::run::write_artifact(
7794 &self.state,
7795 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7796 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7797 ) {
7798 Ok(p) => Some(p),
7799 Err(e) => {
7800 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7801 None
7802 }
7803 };
7804 let (detail, comparison) = gt::question_content(
7805 &w,
7806 &pr.title,
7807 &pr.body,
7808 &self.state.id,
7809 rejected.as_deref(),
7810 artifact.as_deref(),
7811 g.rewrite.as_ref(),
7812 );
7813 let mut q = ask::Question::new(
7814 self.state.id.clone(),
7815 gt::ASK_NODE.to_owned(),
7816 gt::ASK_SEAT.to_owned(),
7817 gt::question_summary(&self.state.config.graph.language, &w),
7818 detail,
7819 gt::question_choices(&w, g.rewrite.as_ref()),
7820 );
7821 q.posting_comparison = Some(comparison);
7822 q.answer_timeout = timeout;
7823 g.question = Some(q.id.clone());
7826 g.asks += 1;
7827 self.state.github_text = Some(g.clone());
7828 self.state.event(
7829 "github-text",
7830 format!("asking the owner about the withheld text ({})", q.short()),
7831 );
7832 self.state.save()?;
7833 store
7834 .put(&mut q)
7835 .context("file the posting-gate question")?;
7836 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7837 tracing::warn!(
7838 "could not notify about posting-gate question {}: {e:#}",
7839 q.short()
7840 );
7841 }
7842 self.park_on_github_text(&g)?;
7843 return Ok(None);
7844 }
7845 }
7846 let resolved = g.chosen_title.clone();
7847 let original = g.choice == Some(GithubTextChoice::Original);
7848 self.state.event(
7850 "github-text",
7851 if original {
7852 format!(
7853 "posting the original text for: {}",
7854 match (g.title, g.body) {
7855 (true, true) => "title, description",
7856 (true, false) => "title",
7857 _ => "description",
7858 }
7859 )
7860 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7861 "posting the saved rewrite title and description".to_owned()
7862 } else if resolved.is_some() {
7863 "posting the owner's replacement title".to_owned()
7864 } else {
7865 "posting the neutral text for the withheld fields".to_owned()
7866 },
7867 );
7868 self.state.github_text = Some(g.clone());
7869 self.state.save()?;
7870 if g.choice == Some(GithubTextChoice::Rewrite)
7871 && let Some(r) = g.rewrite.as_ref()
7872 {
7873 return Ok(Some(PrMessage {
7874 title: r.title.clone(),
7875 body: r.body.clone(),
7876 }));
7877 }
7878 let w = gt::Withheld {
7879 title: g.title,
7880 body: g.body,
7881 categories: g.categories.clone(),
7882 };
7883 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7884 if let Some(t) = resolved {
7885 title = t;
7886 }
7887 let english = gt::LanguageDecision {
7889 title_english: true,
7890 body_english: true,
7891 body_complete: true,
7892 };
7893 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7894 if original {
7895 if g.title {
7898 title = pr.title.clone();
7899 }
7900 if g.body {
7901 body = pr.body.clone();
7902 }
7903 }
7904 Ok(Some(PrMessage { title, body }))
7905 }
7906
7907 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7908 self.state.github_text = Some(g.clone());
7909 self.state.end_pass_parked(ParkCause::Question);
7910 self.state.parked = true;
7911 self.state.event(
7912 "github-text",
7913 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7914 );
7915 self.state.save()
7916 }
7917
7918 async fn judge_pr_language(
7921 &mut self,
7922 cwd: &Path,
7923 pr: &PrMessage,
7924 ) -> Option<crate::github_text::LanguageDecision> {
7925 self.state.config.roles.language_judge.as_ref()?;
7926 if !self.state.config.graph.language_gate() {
7927 return None;
7928 }
7929 let decision =
7930 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
7931 self.state.event(
7932 "github-text",
7933 if decision.is_some() {
7934 "language decided by the language judge"
7935 } else {
7936 "language judge unavailable; using built-in heuristics"
7937 },
7938 );
7939 decision
7940 }
7941
7942 async fn merge(&mut self) -> Result<()> {
7943 if self
7958 .state
7959 .base_sync
7960 .as_ref()
7961 .is_some_and(|s| s.conflict.is_some())
7962 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7963 != Some(RunStatus::Gating)
7964 || !self.state.gate_status().ok()
7973 {
7974 return Ok(());
7975 }
7976 if self.state.merge.is_some() && !self.state.pr_publication_pending {
7985 return Ok(());
7986 }
7987 let Some(winner) = self.state.winner().cloned() else {
7988 return Ok(());
7989 };
7990 let repo = self.state.repo.clone();
7991 let base = self.state.base_branch.clone();
7992 let mode = self.state.config.merge.mode;
7993 let style = self.state.config.merge.style;
7994 let facts = if is_review_run(&self.state) {
7995 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
7996 let start = review_base(
7997 &repo,
7998 &self.state.config.merge.remote,
7999 &base,
8000 &self.state.base_commit,
8001 &winner.branch,
8002 )
8003 .await;
8004 branch_facts(&repo, &start, &winner.branch).await
8005 } else {
8006 None
8007 };
8008 let leaked = if is_review_run(&self.state) {
8011 leaked_subjects(&self.state, &winner.branch).await
8012 } else {
8013 Some(Vec::new())
8014 };
8015 let Some(pr) = self
8019 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8020 .await?
8021 else {
8022 return Ok(());
8023 };
8024 let message = pr.commit_message();
8025
8026 let outcome = match mode {
8027 MergeMode::None => MergeOutcome {
8028 mode,
8029 ok: true,
8030 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8031 empty: false,
8032 },
8033 MergeMode::Pr | MergeMode::Local
8034 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8035 {
8036 MergeOutcome {
8037 mode,
8038 ok: false,
8039 detail: empty_candidate_detail(&self.state, &base),
8040 empty: true,
8041 }
8042 }
8043 MergeMode::Local => {
8044 let on = git::current_branch(&repo).await?;
8045 if on.as_deref() != Some(base.as_str()) {
8046 MergeOutcome {
8047 mode,
8048 ok: false,
8049 detail: format!(
8050 "{} has {} checked out, not the base branch {base}",
8051 repo.display(),
8052 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8053 ),
8054 empty: false,
8055 }
8056 } else if !git::is_clean(&repo).await? {
8057 MergeOutcome {
8058 mode,
8059 ok: false,
8060 detail: format!("{} is dirty; refusing to merge", repo.display()),
8061 empty: false,
8062 }
8063 } else {
8064 let out = match style {
8065 MergeStyle::Merge => {
8066 git::merge_no_ff(&repo, &winner.branch, &message).await?
8067 }
8068 MergeStyle::Squash => {
8069 git::merge_squash(&repo, &winner.branch, &message).await?
8070 }
8071 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8072 };
8073 MergeOutcome {
8074 mode,
8075 ok: out.ok(),
8076 detail: if out.ok() { out.stdout } else { out.stderr },
8077 empty: false,
8078 }
8079 }
8080 }
8081 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8082 self.state.pr_publication_pending = true;
8083 self.state.save()?;
8084 let out =
8085 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8086 self.state.pr_publication_pending = out.is_err();
8087 MergeOutcome {
8088 mode,
8089 ok: out.is_ok(),
8090 detail: match out {
8091 Ok(url) => url,
8092 Err(e) => format!("{e:#}"),
8093 },
8094 empty: false,
8095 }
8096 }
8097 MergeMode::Pr => {
8098 let remote = self.state.config.merge.remote.clone();
8099 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8100 if !pushed.ok() {
8101 MergeOutcome {
8102 mode,
8103 ok: false,
8104 detail: pushed.stderr,
8105 empty: false,
8106 }
8107 } else {
8108 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8114 let out = match pr_merge_plan(found) {
8115 PrPlan::Create => {
8116 gh_pr_create(
8117 &winner.worktree,
8118 &base,
8119 &winner.branch,
8120 &pr.title,
8121 &pr.body,
8122 )
8123 .await
8124 }
8125 PrPlan::Adopt { url, title } => {
8126 self.state
8127 .event("merge", format!("Pr: adopted open pull request {url}"));
8128 if self.state.github_text.as_ref().is_some_and(|g| {
8129 g.resolved
8130 && g.choice == Some(GithubTextChoice::Rewrite)
8131 && g.rewrite.is_some()
8132 }) {
8133 land::set_pr_text_verbatim(
8136 &winner.worktree,
8137 &url,
8138 &pr.title,
8139 &pr.body,
8140 )
8141 .await
8142 .map(|()| url)
8143 } else {
8144 if title != pr.title
8145 && (!is_review_run(&self.state)
8146 || leaked
8147 .as_deref()
8148 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8149 && let Err(e) =
8150 if self.state.github_text.as_ref().is_some_and(|g| {
8151 g.resolved
8152 && g.title
8153 && g.choice == Some(GithubTextChoice::Original)
8154 }) {
8155 land::set_pr_title_verbatim(
8157 &winner.worktree,
8158 &url,
8159 &pr.title,
8160 )
8161 .await
8162 } else {
8163 land::set_pr_title(
8164 &mut self.state,
8165 &winner.worktree,
8166 &url,
8167 &pr.title,
8168 )
8169 .await
8170 }
8171 {
8172 tracing::warn!("could not refresh title of {url}: {e:#}");
8173 self.state
8174 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8175 }
8176 Ok(url)
8177 }
8178 }
8179 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8180 };
8181 match out {
8182 Ok(url) => MergeOutcome {
8183 mode,
8184 ok: true,
8185 detail: url,
8186 empty: false,
8187 },
8188 Err(e) => MergeOutcome {
8189 mode,
8190 ok: false,
8191 detail: e.to_string(),
8192 empty: false,
8193 },
8194 }
8195 }
8196 }
8197 };
8198
8199 self.state.status = match (mode, outcome.ok) {
8200 (MergeMode::None, _) => RunStatus::Ready,
8201 (_, true) => RunStatus::Merged,
8202 (_, false) => RunStatus::Blocked,
8203 };
8204 self.state.event(
8205 "merge",
8206 format!(
8207 "{:?}: {}",
8208 mode,
8209 outcome.detail.lines().next().unwrap_or("")
8210 ),
8211 );
8212 self.state.merge = Some(outcome);
8213 self.state.save()?;
8214
8215 if self.state.config.graph.land
8221 && mode == MergeMode::Pr
8222 && self.state.status == RunStatus::Merged
8223 {
8224 self.run_land().await?;
8225 }
8226 self.settle_questions();
8231 Ok(())
8232 }
8233
8234 async fn run_land(&mut self) -> Result<()> {
8245 crate::pr_head::ensure_owner(&self.state)?;
8246 let url = self
8247 .state
8248 .merge
8249 .as_ref()
8250 .map(|m| m.detail.clone())
8251 .unwrap_or_default();
8252 let url = url.lines().next().unwrap_or("").trim().to_owned();
8253 if !url.starts_with("http") {
8254 return Ok(());
8255 }
8256 loop {
8259 match land::land(&mut self.state, &url).await {
8260 Ok(_)
8261 if self
8262 .state
8263 .merge_repairs
8264 .last()
8265 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8266 {
8267 if self.repair_merge_review().await? {
8268 continue;
8269 }
8270 }
8271 Ok(pr) if self.state.parked => {
8272 let _ = pr;
8276 }
8277 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8278 self.state.status = RunStatus::Superseded;
8279 self.state.save()?;
8280 }
8281 Ok(pr) => {
8282 self.state.status = match pr.state {
8283 land::PrLifecycle::Merged => RunStatus::Merged,
8284 _ => RunStatus::Blocked,
8285 };
8286 if bump::should_release_bump(self.state.status)
8293 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8294 {
8295 self.state
8301 .event("bump", format!("release bump skipped: {e:#}"));
8302 }
8303 if self.state.status == RunStatus::Merged {
8306 crate::followup::after_merge(&mut self.state, &pr.url).await;
8307 }
8308 self.state.save()?;
8309 }
8310 Err(e) => {
8311 if e.is::<crate::repo_lock::ResourceBlocked>() {
8312 return Err(e);
8313 }
8314 self.state.status = RunStatus::Blocked;
8315 self.state.event("land", format!("gave up: {e}"));
8316 self.state.save()?;
8317 }
8318 }
8319 break;
8320 }
8321 Ok(())
8322 }
8323
8324 async fn repair_merge_review(&mut self) -> Result<bool> {
8327 use crate::run::MergeRepairPhase as Phase;
8328 crate::pr_head::ensure_owner(&self.state)?;
8329 let i = self.state.merge_repairs.len() - 1;
8330 let request = self.state.merge_repairs[i].clone();
8331 if request.phase == Phase::Fixing {
8332 let restored = match self.state.winner().cloned() {
8333 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8334 None => Err(anyhow::anyhow!("repair has no winning branch")),
8335 };
8336 let why = match restored {
8337 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8338 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8339 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8340 };
8341 self.hold_merge_repair(i, why)?;
8342 return Ok(false);
8343 }
8344 let failure = if i >= self.state.config.graph.land_rounds
8345 || self.state.config.graph.review_rounds == 0
8346 {
8347 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8348 } else {
8349 None
8350 };
8351 if let Some(why) = failure {
8352 self.hold_merge_repair(i, why)?;
8353 return Ok(false);
8354 }
8355 if request.phase == Phase::Requested {
8356 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8357 Ok(pr) => pr,
8358 Err(e) => {
8359 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8360 return Ok(false);
8361 }
8362 };
8363 self.state.merge_repairs[i].phase = Phase::Fixing;
8364 self.state.land_approval = None;
8365 self.state.land_armed_head = None;
8366 self.state.save()?;
8367 let evidence = serde_json::to_string_pretty(&request)?;
8370 let reason = format!(
8371 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8372 );
8373 let budget = self.state.config.graph.land_rounds;
8374 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8375 match result {
8376 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8377 Ok(land::Fixed::Failed(why)) => {
8378 self.hold_merge_repair(
8379 i,
8380 format!(
8381 "repair failed: {why}; inspect the branch and run an explicit review"
8382 ),
8383 )?;
8384 return Ok(false);
8385 }
8386 Err(e) => {
8387 self.hold_merge_repair(
8388 i,
8389 format!(
8390 "repair failed: {e:#}; inspect the branch and run an explicit review"
8391 ),
8392 )?;
8393 return Ok(false);
8394 }
8395 }
8396 self.begin_merge_repair_review(i)?;
8400 }
8401 self.review_loop().await?;
8402 self.gate().await?;
8403 if self.resource_halt()? || self.state.parked {
8404 return Ok(false);
8405 }
8406 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8407 != Some(RunStatus::Gating)
8408 || !self.state.gate_status().ok()
8409 {
8410 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8411 return Ok(false);
8412 }
8413 let winner = self
8416 .state
8417 .winner()
8418 .cloned()
8419 .context("repair has no winning branch")?;
8420 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8421 if self.state.gate_head.as_deref() != Some(after.as_str())
8422 || !git::is_clean(&winner.worktree).await?
8423 {
8424 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8425 return Ok(false);
8426 }
8427 let expected = self
8428 .state
8429 .pr_pushed_head
8430 .as_deref()
8431 .unwrap_or(&request.head)
8432 .to_owned();
8433 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8434 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8437 let refspec = format!("{after}:refs/heads/{public}");
8438 let push = git::git_raw(
8439 &winner.worktree,
8440 &["push", &lease, &self.state.config.merge.remote, &refspec],
8441 )
8442 .await;
8443 match push {
8444 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8445 Ok(out) => {
8446 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8447 return Ok(false);
8448 }
8449 Err(e) => {
8450 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8451 return Ok(false);
8452 }
8453 }
8454 self.state.merge_repairs[i].phase = Phase::Complete;
8455 self.state.status = RunStatus::Landing;
8456 self.state.event(
8457 "repair",
8458 "new review and gate completed; fresh owner merge decision required",
8459 );
8460 self.state.save()?;
8461 Ok(true)
8462 }
8463
8464 fn merge_repair_artifacts(&self) -> PathBuf {
8465 let artifacts = agent::artifacts_dir(&self.state.dir());
8466 match self
8467 .state
8468 .merge_repairs
8469 .last()
8470 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8471 {
8472 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8473 None => artifacts,
8474 }
8475 }
8476
8477 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8478 self.state.reviews.clear();
8479 self.state.gate.clear();
8480 self.state.gate_ran = false;
8481 self.state.gate_head = None;
8482 self.state.gate_fixes.clear();
8483 self.state.pre_gate.clear();
8484 self.state.pre_gate_commit = None;
8485 self.state.contested_handoff = None;
8486 self.state.land_reruns = None;
8487 self.state.land_approval = None;
8488 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8489 self.state.status = RunStatus::Reviewing;
8490 self.state.save()
8491 }
8492
8493 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8494 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8495 self.state.merge_repairs[index].failure = Some(why.clone());
8496 self.state.status = RunStatus::Blocked;
8497 self.state.event("repair", why);
8498 self.state.save()
8499 }
8500
8501 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8505 if let Some(existing) = self.state.seats.get(key)
8506 && existing.agent == agent
8507 {
8508 return existing.clone();
8509 }
8510 let fresh = if self.state.seats.contains_key(key) {
8513 handover_seat(key, agent, self.state.next_seat_seed())
8514 } else {
8515 SeatState::new(key, agent, self.state.seed)
8516 };
8517 self.state.seats.insert(key.to_owned(), fresh.clone());
8518 fresh
8519 }
8520
8521 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8527 match self.state.seats.get(key) {
8528 Some(s) if s.agent != spec.id => {
8529 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8530 }
8531 _ => spec,
8532 }
8533 }
8534
8535 fn view(&self, c: &Candidate) -> CandidateView {
8537 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8538 .unwrap_or_default();
8539 let (patch, _) = blind::sanitize_patch(
8540 &format!("candidate {} patch", c.label),
8541 &raw,
8542 &self.state.config.blind,
8543 );
8544 CandidateView {
8545 label: c.label,
8546 branch: c.branch.clone(),
8547 summary: c.summary.clone(),
8548 stat: c.stat.clone(),
8549 patch,
8550 }
8551 }
8552
8553 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8555 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8556 prompt::judge(
8557 "(see above)",
8558 &views,
8559 self.roles.judges.len(),
8560 base_short,
8561 "en",
8562 )
8563 }
8564
8565 fn vote_prompt_full(
8571 &self,
8572 j: usize,
8573 labels: &[char],
8574 language: &str,
8575 candidates: &[Candidate],
8576 base_short: &str,
8577 ) -> String {
8578 let mut text = format!(
8579 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8580 prompt::final_vote(labels, language),
8581 self.state.instruction,
8582 self.candidate_block(candidates, base_short)
8583 );
8584 if let Some(own) = self
8585 .state
8586 .judgements
8587 .get(j)
8588 .filter(|r| !r.ranking.is_empty())
8589 {
8590 let reasons = own
8591 .reasons
8592 .iter()
8593 .map(|(k, v)| format!("- {k}: {v}"))
8594 .collect::<Vec<_>>()
8595 .join("\n");
8596 text.push_str(&format!(
8597 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8598 own.ranking.iter().collect::<String>(),
8599 if reasons.is_empty() {
8600 ""
8601 } else {
8602 ", because:\n"
8603 }
8604 ));
8605 }
8606 if !self.state.deliberation.is_empty() {
8607 text.push_str("\n# What was argued before this vote\n");
8608 for t in self.transcript(&[], j) {
8609 text.push_str(&format!(
8610 "\n## {}{}\n\n{}\n",
8611 t.who,
8612 if t.is_self { " (you)" } else { "" },
8613 t.body.trim()
8614 ));
8615 }
8616 }
8617 text
8618 }
8619
8620 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8627 let mut turns = Vec::new();
8628 for j in &self.state.judgements {
8629 if j.ranking.is_empty() {
8630 continue;
8631 }
8632 let reasons = j
8633 .reasons
8634 .iter()
8635 .map(|(k, v)| format!("- {k}: {v}"))
8636 .collect::<Vec<_>>()
8637 .join("\n");
8638 turns.push(Turn {
8639 who: format!("Judge {} (opening ranking)", j.judge),
8640 is_self: j.judge == self_idx + 1,
8641 body: format!(
8642 "Ranked {}{}{reasons}",
8643 j.ranking.iter().collect::<String>(),
8644 if reasons.is_empty() {
8645 ""
8646 } else {
8647 ", because:\n"
8648 }
8649 ),
8650 });
8651 }
8652 for t in self
8653 .state
8654 .deliberation
8655 .iter()
8656 .flat_map(|r| r.turns.iter())
8657 .chain(current)
8658 {
8659 turns.push(Turn {
8660 who: format!("Judge {}", t.judge),
8661 is_self: t.judge == self_idx + 1,
8662 body: t.body.clone(),
8663 });
8664 }
8665 turns
8666 }
8667}
8668
8669fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8671 agent::has_session(spec.kind, seat, sessions)
8672}
8673
8674fn next_untried_in_roster<'a>(
8695 roster: &'a [AgentSpec],
8696 start: usize,
8697 tried: &BTreeSet<String>,
8698) -> Option<&'a AgentSpec> {
8699 roster
8700 .get(start + 1..)?
8701 .iter()
8702 .find(|s| !tried.contains(&s.id))
8703}
8704
8705fn pick_successor<'a>(
8726 roster: &'a [AgentSpec],
8727 start: usize,
8728 tried: &BTreeSet<String>,
8729 carried: Option<&BTreeSet<String>>,
8730 others: &BTreeSet<String>,
8731) -> Option<&'a AgentSpec> {
8732 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8733 roster
8734 .get(start + 1..)
8735 .and_then(|tail| {
8736 tail.iter()
8737 .filter(free)
8738 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8739 })
8740 .or_else(|| {
8741 carried
8742 .is_some()
8743 .then(|| roster.iter().find(free))
8744 .flatten()
8745 })
8746 .or_else(|| match carried {
8747 Some(c) => next_for_seat(roster, start, tried, c),
8748 None => next_untried_in_roster(roster, start, tried),
8749 })
8750}
8751
8752fn next_for_seat<'a>(
8762 roster: &'a [AgentSpec],
8763 start: usize,
8764 round_tried: &BTreeSet<String>,
8765 carried_failed: &BTreeSet<String>,
8766) -> Option<&'a AgentSpec> {
8767 roster
8768 .get(start + 1..)?
8769 .iter()
8770 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8771 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8772}
8773
8774fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8780 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8781 return spec;
8782 };
8783 let ok = |id: &str| !h.failed.contains(id);
8784 if let Some(last) = h.last_ok.as_deref()
8785 && ok(last)
8786 && let Some(s) = roster.iter().find(|s| s.id == last)
8787 {
8788 return s.clone();
8789 }
8790 if ok(&spec.id) {
8791 return spec;
8792 }
8793 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8794 roster
8795 .get(from + 1..)
8796 .into_iter()
8797 .flatten()
8798 .chain(roster.iter())
8799 .find(|s| ok(&s.id))
8800 .cloned()
8801 .unwrap_or(spec)
8802}
8803
8804pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8808 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8809}
8810
8811const TIMED_OUT: &str = "timed out";
8814
8815impl FailClass {
8816 fn of(out: &AgentOutcome) -> Option<Self> {
8818 match out {
8819 AgentOutcome::Ok(_) => None,
8820 AgentOutcome::Quota(_) => Some(Self::Quota),
8821 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8822 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8823 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8824 AgentOutcome::ResourceBlocked(_) => None,
8827 }
8828 }
8829
8830 fn stem_word(&self) -> &'static str {
8832 match self {
8833 Self::Quota => "quota",
8834 _ => "handover",
8835 }
8836 }
8837}
8838
8839fn failure_signature(msg: &str) -> String {
8843 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8844 let mut out = Vec::new();
8845 for word in line.split_whitespace() {
8846 if word.contains('/') || word.contains('\\') {
8847 out.push("<path>".to_owned());
8848 continue;
8849 }
8850 let mut w = String::new();
8851 let mut in_digits = false;
8852 for c in word.chars() {
8853 if c.is_ascii_digit() {
8854 if !in_digits {
8855 w.push('#');
8856 }
8857 in_digits = true;
8858 } else {
8859 in_digits = false;
8860 w.push(c);
8861 }
8862 }
8863 out.push(w);
8864 }
8865 out.join(" ").chars().take(120).collect()
8866}
8867
8868fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8875 match cur {
8876 FailClass::Quota | FailClass::Timeout => true,
8877 FailClass::Other(_) => prev != Some(cur),
8878 }
8879}
8880
8881fn fail_reason(out: &AgentOutcome) -> String {
8883 match out {
8884 AgentOutcome::Ok(_) => String::new(),
8885 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8886 AgentOutcome::Dropped(o) => format!(
8887 "the CLI dropped the stream ({})",
8888 o.dropped
8889 .as_ref()
8890 .map(|d| d.why.as_str())
8891 .unwrap_or("it ended without delivering its answer")
8892 ),
8893 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8894 e.lines().next().unwrap_or("").chars().take(160).collect()
8895 }
8896 }
8897}
8898
8899pub(crate) fn record_handover(
8902 state: &mut RunState,
8903 node: &str,
8904 seat: &str,
8905 from: &str,
8906 to: &str,
8907 class: &FailClass,
8908 reason: &str,
8909) {
8910 let message = if *class == FailClass::Quota {
8911 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
8912 } else {
8913 format!("{seat}: handed over {from} -> {to} ({reason})")
8914 };
8915 state.event(node, message);
8916 state.handovers.push(Handover {
8917 at: Timestamp::now(),
8918 node: node.to_owned(),
8919 seat: seat.to_owned(),
8920 from: from.to_owned(),
8921 to: to.to_owned(),
8922 reason: reason.to_owned(),
8923 });
8924}
8925
8926fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
8940 commands.iter().any(|c| c.exit_code.is_none())
8941}
8942
8943fn verified_noop_claim(
8956 usable: bool,
8957 commands: &[agent::CommandEvidence],
8958 text: &str,
8959) -> Option<String> {
8960 (usable && !has_unconfirmed_command(commands))
8961 .then(|| verdict::verified_noop(text))
8962 .flatten()
8963}
8964
8965fn short(commit: &str) -> String {
8966 commit.chars().take(7).collect()
8967}
8968
8969fn make_executable(path: &Path) -> Result<()> {
8970 #[cfg(unix)]
8971 {
8972 use std::os::unix::fs::PermissionsExt as _;
8973 let mut perms = std::fs::metadata(path)?.permissions();
8974 perms.set_mode(0o755);
8975 std::fs::set_permissions(path, perms)?;
8976 }
8977 #[cfg(not(unix))]
8978 {
8979 let _ = path;
8980 }
8981 Ok(())
8982}
8983
8984struct WaveCtx<'a> {
8991 run: &'a str,
8994 node: &'a str,
8996 prompts: &'a Prompts,
8997 cache: Option<&'a Path>,
8999 round: Option<usize>,
9002 carry_seats: bool,
9006 pauses: Option<(&'a Pause, &'a Pause)>,
9009}
9010
9011fn repository_wait(
9013 state: &RunState,
9014 pauses: Option<(&Pause, &Pause)>,
9015) -> crate::repo_lock::WaitPolicy {
9016 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9017 let inherited = crate::repo_lock::WaitPolicy::current();
9018 crate::repo_lock::WaitPolicy::new(
9019 Duration::from_secs(state.config.graph.timeout_resource_wait),
9020 move || {
9021 inherited.cancelled()
9022 || pauses
9023 .as_ref()
9024 .is_some_and(|(a, b)| a.parked() || b.parked())
9025 },
9026 )
9027}
9028
9029fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9030 if error.is::<crate::repo_lock::ResourceBlocked>() {
9031 state.resource_block = Some(crate::run::ResourceBlock {
9032 node: "repository".into(),
9033 kind: "repository coordination".into(),
9034 detail: error.to_string(),
9035 at: Timestamp::now(),
9036 });
9037 state.event("resource", error.to_string());
9038 }
9039}
9040
9041async fn coordinate_operation(
9042 state: &mut RunState,
9043 cwd: &Path,
9044 wait: &crate::repo_lock::WaitPolicy,
9045) -> Result<Option<crate::repo_lock::SeatLease>> {
9046 let result =
9047 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9048 .await;
9049 if let Err(e) = &result {
9050 record_repository_block(state, e);
9051 }
9052 result
9053}
9054
9055async fn run_one(
9057 job: SeatJob,
9058 sem: Arc<Semaphore>,
9059 ctx: &WaveCtx<'_>,
9060 state: &mut RunState,
9061 attempt: usize,
9062) -> (SeatState, AgentOutcome) {
9063 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9064 .await
9065 .pop()
9066 .expect("one job in, one result out");
9067 (seat, out)
9068}
9069
9070async fn wave(
9076 jobs: Vec<SeatJob>,
9077 sem: Arc<Semaphore>,
9078 ctx: &WaveCtx<'_>,
9079 state: &mut RunState,
9080 attempt: usize,
9081) -> Vec<(usize, SeatState, AgentOutcome)> {
9082 let WaveCtx {
9083 run,
9084 node,
9085 prompts,
9086 cache,
9087 round,
9088 carry_seats: _,
9089 pauses,
9090 } = *ctx;
9091 for job in &jobs {
9092 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9093 }
9094 if let Err(e) = state.save() {
9095 tracing::warn!("could not persist in-progress seats: {e:#}");
9100 }
9101 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9117 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9121 let cache_guard = if let Some(cache_dir) = cache {
9122 if jobs_had_a_writer {
9123 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9124 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9125 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9126 {
9127 Ok(Some(g)) => Some(g),
9128 Ok(None) => {
9131 state.resource_block = Some(crate::run::ResourceBlock {
9132 node: node.to_owned(),
9133 kind: "shared build cache".to_owned(),
9134 detail: "the run is parking".to_owned(),
9135 at: Timestamp::now(),
9136 });
9137 return block_wave(
9138 jobs,
9139 state,
9140 node,
9141 attempt,
9142 "parked while waiting for the shared build cache",
9143 );
9144 }
9145 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9148 Err(e) => {
9149 let why = format!("{e:#}");
9153 state.resource_block = Some(crate::run::ResourceBlock {
9154 node: node.to_owned(),
9155 kind: "shared build cache".to_owned(),
9156 detail: why.clone(),
9157 at: Timestamp::now(),
9158 });
9159 state.event(
9160 "resource",
9161 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9162 );
9163 return block_wave(jobs, state, node, attempt, &why);
9164 }
9165 }
9166 } else {
9167 None
9168 }
9169 } else {
9170 None
9171 };
9172 state.seats_spawning(node, attempt);
9175 if let Err(e) = state.save() {
9176 tracing::warn!("could not persist the start of a wave: {e:#}");
9177 }
9178 let slot_limit = resource_limit
9182 + jobs
9183 .iter()
9184 .map(|j| j.timeout)
9185 .max()
9186 .unwrap_or_default()
9187 .saturating_mul(jobs.len() as u32);
9188 let mut set = tokio::task::JoinSet::new();
9189 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9192 let overlay = prompts.overlay(node);
9193 let repository_wait = repository_wait(state, pauses);
9194 for (i, mut job) in jobs.into_iter().enumerate() {
9195 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9196 if cache.is_some() {
9197 job.prompt.push('\n');
9198 job.prompt
9199 .push_str(&prompt::build_cache_note(node, job.allow_write));
9200 }
9201 let sem = Arc::clone(&sem);
9202 let run = run.to_owned();
9203 let node = node.to_owned();
9204 let attachments = if node == "implement" {
9207 state.attachments.clone()
9208 } else {
9209 Vec::new()
9210 };
9211 let cache = cache
9222 .filter(|_| job.allow_write && cache_guard.is_some())
9223 .map(Path::to_path_buf);
9224 let spawned_tx = spawned_tx.clone();
9225 let repository_wait = repository_wait.clone();
9226 let coordination = state.config.coordination.clone();
9227 set.spawn(async move {
9228 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9232 Ok(p) => p,
9233 Err(_) => {
9234 return (
9235 i,
9236 job.seat,
9237 AgentOutcome::ResourceBlocked(format!(
9238 "no parallel slot became free within {}s",
9239 slot_limit.as_secs()
9240 )),
9241 );
9242 }
9243 };
9244 spawned_tx.send(job.seat.key.clone()).ok();
9245 let mut seat = job.seat;
9246 let out = repository_wait
9247 .scope(agent::invoke(
9248 &job.spec,
9249 &mut seat,
9250 &Invocation {
9251 coordination: coordination.as_ref(),
9252 cwd: &job.cwd,
9253 prompt: &job.prompt,
9254 timeout: job.timeout,
9255 allow_write: job.allow_write,
9256 unsandboxed: false,
9257 sessions: job.sessions,
9258 artifacts: &job.artifacts,
9259 stem: &job.stem,
9260 run: &run,
9261 node: &node,
9262 cache_dir: cache.as_deref(),
9263 attachments: &attachments,
9264 writable: &[],
9265 },
9266 ))
9267 .await;
9268 let out = match out {
9269 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9270 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9271 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9279 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9280 Ok(o) => AgentOutcome::Failed(format!(
9281 "exited with {:?} and no usable output",
9282 o.exit_code
9283 )),
9284 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9285 AgentOutcome::ResourceBlocked(e.to_string())
9286 }
9287 Err(e) => AgentOutcome::Failed(e.to_string()),
9288 };
9289 (i, seat, out)
9290 });
9291 }
9292 drop(spawned_tx);
9293 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9294 loop {
9295 let joined = tokio::select! {
9296 Some(key) = spawned_rx.recv() => {
9297 state.seat_spawned(&key);
9299 if let Err(e) = state.save() {
9300 tracing::warn!("could not persist a seat's start: {e:#}");
9301 }
9302 continue;
9303 }
9304 joined = set.join_next() => joined,
9305 };
9306 let Some(joined) = joined else {
9307 break;
9308 };
9309 let (i, seat, out) = match joined {
9310 Ok(v) => v,
9311 Err(e) => {
9315 tracing::error!("agent task panicked: {e}");
9316 continue;
9317 }
9318 };
9319 state.seat_finished(&seat.key);
9320 if let AgentOutcome::ResourceBlocked(why) = &out
9321 && state.resource_block.is_none()
9322 {
9323 state.resource_block = Some(crate::run::ResourceBlock {
9324 node: node.to_owned(),
9325 kind: if why.contains("repository coordination") {
9326 "repository coordination"
9327 } else {
9328 "parallel slot"
9329 }
9330 .to_owned(),
9331 detail: why.clone(),
9332 at: Timestamp::now(),
9333 });
9334 state.event("resource", format!("{node}: no agent was started - {why}"));
9335 }
9336 record_jobs(state, node, round, &seat.key, &out);
9337 if let Err(e) = state.save() {
9338 tracing::warn!("could not persist a seat's completion: {e:#}");
9339 }
9340 if collected.len() <= i {
9341 collected.resize_with(i + 1, || None);
9342 }
9343 collected[i] = Some((i, seat, out));
9344 }
9345 if state
9351 .active
9352 .values()
9353 .any(|a| a.node == node && a.attempt == attempt)
9354 {
9355 state
9356 .active
9357 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9358 if let Err(e) = state.save() {
9359 tracing::warn!("could not persist the end of a wave: {e:#}");
9360 }
9361 }
9362 if let Some(cache_dir) = cache
9369 && jobs_had_a_writer
9370 {
9371 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9372 }
9373 if let Some(guard) = cache_guard {
9374 guard.release();
9375 }
9376 collected.into_iter().flatten().collect()
9377}
9378
9379fn record_jobs(
9390 state: &mut RunState,
9391 node: &str,
9392 round: Option<usize>,
9393 seat: &str,
9394 out: &AgentOutcome,
9395) {
9396 let commands: &[agent::CommandEvidence] = match out {
9397 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9398 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9399 };
9400 let checked_at = Timestamp::now();
9401 for c in commands {
9402 state.jobs.push(JobRecord {
9403 node: node.to_owned(),
9404 round,
9405 seat: seat.to_owned(),
9406 id: c.id.clone(),
9407 description: c.description.clone(),
9408 checked_at,
9409 status: match c.exit_code {
9410 Some(0) => JobStatus::Completed,
9411 Some(_) => JobStatus::Failed,
9412 None => JobStatus::Unknown,
9413 },
9414 exit_code: c.exit_code,
9415 result_summary: c.result_summary.clone(),
9416 source: c.source.clone(),
9417 });
9418 }
9419}
9420
9421fn round_is_clean(
9442 blocking: usize,
9443 e2e_ok: bool,
9444 answered: usize,
9445 expected: usize,
9446 quota_missing: usize,
9447 policy: IncompleteReviewPolicy,
9448) -> bool {
9449 if blocking != 0 || !e2e_ok {
9450 return false;
9451 }
9452 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9453 return true;
9454 }
9455 answered > 0 && expected - answered <= quota_missing
9456}
9457
9458fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9482 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9483 return Some(RunStatus::Gating);
9484 }
9485 let last = reviews.last()?;
9486 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9487 if reviews.len() < max_rounds && !stagnant {
9488 return None;
9489 }
9490 if last.incomplete() && last.blocking == 0 {
9491 return Some(RunStatus::Blocked);
9492 }
9493 if last.e2e_status() == E2eStatus::ResourceBlocked {
9494 return None;
9495 }
9496 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9497 RunStatus::Gating
9498 } else {
9499 RunStatus::Blocked
9500 })
9501}
9502
9503pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9518 if nudged {
9519 (full / 4).max(Duration::from_secs(120)).min(full)
9520 } else {
9521 full
9522 }
9523}
9524
9525#[allow(clippy::too_many_arguments)]
9554async fn ask_json_wave<T>(
9555 jobs: Vec<SeatJob>,
9556 sem: Arc<Semaphore>,
9557 retries: usize,
9558 roster: &[AgentSpec],
9559 ctx: &WaveCtx<'_>,
9560 losses: &mut Vec<QuotaLoss>,
9561 state: &mut RunState,
9562 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9563) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9564where
9565 T: serde::de::DeserializeOwned + Send + 'static,
9566{
9567 ask_wave_with(
9568 jobs,
9569 sem,
9570 retries,
9571 roster,
9572 ctx,
9573 losses,
9574 state,
9575 &|text: &str| {
9576 let v = verdict::extract_json::<T>(text)?;
9577 validate(&v)?;
9578 Ok(v)
9579 },
9580 )
9581 .await
9582}
9583
9584#[allow(clippy::too_many_arguments)]
9593async fn ask_wave_with<T>(
9594 jobs: Vec<SeatJob>,
9595 sem: Arc<Semaphore>,
9596 retries: usize,
9597 roster: &[AgentSpec],
9598 ctx: &WaveCtx<'_>,
9599 losses: &mut Vec<QuotaLoss>,
9600 state: &mut RunState,
9601 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9602) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9603where
9604 T: Send + 'static,
9605{
9606 let n = jobs.len();
9607 let originals: Vec<SeatJob> = jobs;
9608 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9609 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9610 let mut nudges: Vec<usize> = vec![0; n];
9617 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9621 let mut tried: Vec<BTreeSet<String>> = specs
9622 .iter()
9623 .map(|s| BTreeSet::from([s.id.clone()]))
9624 .collect();
9625 let starts: Vec<usize> = specs
9626 .iter()
9627 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9628 .collect();
9629 let carry = ctx.carry_seats && !roster.is_empty();
9630 let carried: Vec<BTreeSet<String>> = originals
9633 .iter()
9634 .map(|j| {
9635 state
9636 .seat_history
9637 .get(&j.seat.key)
9638 .filter(|_| carry)
9639 .map(|h| h.failed.clone())
9640 .unwrap_or_default()
9641 })
9642 .collect();
9643 let mut prev: Vec<Option<FailClass>> = originals
9644 .iter()
9645 .map(|j| {
9646 state
9647 .seat_history
9648 .get(&j.seat.key)
9649 .filter(|_| carry)
9650 .and_then(|h| h.last_fail.clone())
9651 })
9652 .collect();
9653 let next_agent =
9654 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9655 let others: BTreeSet<String> = specs
9656 .iter()
9657 .enumerate()
9658 .filter(|(j, _)| *j != i)
9659 .map(|(_, s)| s.id.clone())
9660 .collect();
9661 pick_successor(
9662 roster,
9663 starts[i],
9664 tried,
9665 carry.then(|| &carried[i]),
9666 &others,
9667 )
9668 .cloned()
9669 };
9670 let mut fresh: Vec<Option<String>> = vec![None; n];
9671 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9672 let mut pending: Vec<usize> = (0..n).collect();
9673
9674 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9679 for round in 0..max_rounds {
9680 if pending.is_empty() {
9681 break;
9682 }
9683 let mut batch = Vec::with_capacity(pending.len());
9684 let mut renudged: Vec<&str> = Vec::new();
9685 for &i in &pending {
9686 let src = &originals[i];
9687 let full: &str = match &src.handover {
9692 Some(h) if specs[i].id != src.spec.id => h,
9693 _ => &src.prompt,
9694 };
9695 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9698 (
9699 full.to_owned(),
9700 src.timeout,
9701 format!("{}-{word}-{}", src.stem, specs[i].id),
9702 )
9703 } else if nudges[i] == 0 {
9704 (full.to_owned(), src.timeout, src.stem.clone())
9705 } else {
9706 renudged.push(src.seat.key.as_str());
9707 let why = done[i]
9708 .as_ref()
9709 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9710 .unwrap_or_else(|| "no parsable answer".to_owned());
9711 let nudge = prompt::nudge(&why);
9712 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9713 let prompt = if nudged {
9714 nudge
9715 } else {
9716 format!("{full}\n\n---\n\n{nudge}")
9717 };
9718 (
9719 prompt,
9720 retry_budget(src.timeout, nudged),
9721 format!("{}-retry{}", src.stem, nudges[i]),
9722 )
9723 };
9724 batch.push(SeatJob {
9725 spec: specs[i].clone(),
9726 seat: seats[i].clone(),
9727 cwd: src.cwd.clone(),
9728 prompt,
9729 timeout,
9730 allow_write: src.allow_write,
9731 sessions: src.sessions,
9732 artifacts: src.artifacts.clone(),
9733 stem,
9734 handover: None,
9735 });
9736 }
9737
9738 if !renudged.is_empty() {
9739 state.event(
9740 ctx.node,
9741 format!("retry {round}: re-asking {}", renudged.join(", ")),
9742 );
9743 }
9744 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9745 let mut still = Vec::new();
9746 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9747 seats[i] = seat;
9748 if let AgentOutcome::ResourceBlocked(why) = &out {
9751 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9752 continue;
9753 }
9754 let class = FailClass::of(&out);
9755 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9758 && nudges[i] < retries
9759 && !roster.is_empty();
9760 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9761 let next = should_hand_over(prev[i].as_ref(), &cur)
9762 .then(|| next_agent(i, &tried[i], &specs))
9763 .flatten();
9764 if carry {
9765 let h = state
9766 .seat_history
9767 .entry(originals[i].seat.key.clone())
9768 .or_default();
9769 h.failed.insert(specs[i].id.clone());
9770 h.last_fail = Some(cur.clone());
9771 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9772 h.last_ok = None;
9773 }
9774 if let Err(e) = state.save() {
9777 tracing::warn!("could not persist a seat's failure history: {e:#}");
9778 }
9779 }
9780 if let Some(next) = next {
9781 record_handover(
9782 state,
9783 ctx.node,
9784 &originals[i].seat.key,
9785 &specs[i].id,
9786 &next.id,
9787 &cur,
9788 &fail_reason(&out),
9789 );
9790 tried[i].insert(next.id.clone());
9791 prev[i] = Some(cur.clone());
9792 seats[i] =
9793 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9794 specs[i] = next;
9795 fresh[i] = Some(cur.stem_word().to_owned());
9796 nudges[i] = 0;
9797 done[i] = Some(Err(anyhow::anyhow!(
9798 "handed over after: {}",
9799 fail_reason(&out)
9800 )));
9801 still.push(i);
9802 continue;
9803 }
9804 }
9805 if carry
9806 && !nudge_first
9807 && let Some(cur) = class.clone()
9808 {
9809 let h = state
9811 .seat_history
9812 .entry(originals[i].seat.key.clone())
9813 .or_default();
9814 h.failed.insert(specs[i].id.clone());
9815 h.last_fail = Some(cur);
9816 }
9817 let parsed = match out {
9818 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9819 AgentOutcome::Quota(o) => {
9820 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9821 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9822 }
9823 AgentOutcome::Dropped(o) => {
9827 let why = o
9828 .dropped
9829 .as_ref()
9830 .map(|d| d.why.as_str())
9831 .unwrap_or("the CLI ended the stream without delivering its answer");
9832 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9833 }
9834 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9835 Err(anyhow::anyhow!(e))
9836 }
9837 };
9838 let quota = class == Some(FailClass::Quota);
9839 let failed = parsed.is_err();
9840 done[i] = Some(parsed);
9841 if carry && !failed {
9842 let h = state
9843 .seat_history
9844 .entry(originals[i].seat.key.clone())
9845 .or_default();
9846 h.failed.remove(&specs[i].id);
9847 h.last_ok = Some(specs[i].id.clone());
9848 h.last_fail = None;
9849 }
9850 let agent_failure = class.is_some()
9858 && !nudge_first
9859 && !roster.is_empty()
9860 && next_agent(i, &tried[i], &specs).is_some();
9861 if failed && !quota && !agent_failure && nudges[i] < retries {
9862 nudges[i] += 1;
9863 still.push(i);
9864 }
9865 }
9866 pending = still;
9867 }
9868
9869 for (i, q) in last_quota.into_iter().enumerate() {
9872 if let Some(reset) = q {
9873 losses.push(QuotaLoss {
9874 seat: originals[i].seat.key.clone(),
9875 node: ctx.node.to_owned(),
9876 at: Timestamp::now(),
9877 reset,
9878 });
9879 }
9880 }
9881
9882 seats
9883 .into_iter()
9884 .zip(done)
9885 .zip(nudges)
9886 .map(|((seat, res), attempts)| {
9887 (
9888 seat,
9889 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
9890 attempts,
9891 )
9892 })
9893 .collect()
9894}
9895
9896fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
9898 results
9899 .iter()
9900 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
9901}
9902
9903fn block_wave(
9907 jobs: Vec<SeatJob>,
9908 state: &mut RunState,
9909 node: &str,
9910 attempt: usize,
9911 why: &str,
9912) -> Vec<(usize, SeatState, AgentOutcome)> {
9913 state.resource_wait = None;
9914 state
9915 .active
9916 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9917 if let Err(e) = state.save() {
9918 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
9919 }
9920 jobs.into_iter()
9921 .enumerate()
9922 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
9923 .collect()
9924}
9925
9926async fn acquire_cache_lease(
9939 state: &mut RunState,
9940 cache_dir: &Path,
9941 owner: &crate::cache::Owner,
9942 budget: Duration,
9943 context: &str,
9944 cancel: &(dyn Fn() -> bool + Sync),
9945) -> Result<Option<crate::cache::Guard>> {
9946 let home = crate::run::home();
9947 let started = Instant::now();
9948 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
9949 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
9950 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
9951 Err(e) => {
9952 state.event(
9953 "verify",
9954 format!("{context}: could not check the shared build cache: {e:#}"),
9955 );
9956 if let Err(e2) = state.save() {
9957 tracing::warn!("could not persist a cache-check failure: {e2:#}");
9958 }
9959 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
9960 }
9961 };
9962 state.event(
9963 "verify",
9964 format!(
9965 "{context}: waiting for the shared build cache at {} ({})",
9966 cache_dir.display(),
9967 busy.describe()
9968 ),
9969 );
9970 state.resource_wait = Some(crate::run::ResourceWait {
9973 node: context.to_owned(),
9974 kind: "shared build cache".to_owned(),
9975 holder: busy.describe(),
9976 since: Timestamp::now(),
9977 limit_secs: budget.as_secs(),
9978 });
9979 if let Err(e) = state.save() {
9980 tracing::warn!("could not persist a cache wait: {e:#}");
9981 }
9982 let remaining = budget.saturating_sub(started.elapsed());
9983 let waited = crate::cache::wait_for_unless(
9984 &home,
9985 cache_dir,
9986 owner,
9987 remaining,
9988 Duration::from_secs(5),
9989 cancel,
9990 )
9991 .await;
9992 state.resource_wait = None;
9993 match waited {
9994 Ok(g) => Ok(g),
9995 Err(e) => {
9996 state.event("verify", format!("{context}: {e:#}"));
9997 if let Err(e2) = state.save() {
9998 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
9999 }
10000 Err(e)
10001 }
10002 }
10003}
10004
10005#[derive(Debug)]
10010struct CacheCheckError(String);
10011
10012impl std::fmt::Display for CacheCheckError {
10013 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10014 f.write_str(&self.0)
10015 }
10016}
10017
10018impl std::error::Error for CacheCheckError {}
10019
10020#[allow(clippy::too_many_arguments)]
10041async fn with_cache_lease<'s, F, Fut>(
10042 state: &'s mut RunState,
10043 cache_dir: Option<&Path>,
10044 node: &str,
10045 seat: &str,
10046 worktree: &Path,
10047 head: &str,
10048 budget: Duration,
10049 context: &str,
10050 wait: &crate::repo_lock::WaitPolicy,
10051 body: F,
10052) -> Result<(Vec<CommandOutcome>, bool)>
10053where
10054 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10055 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10056{
10057 let started = Instant::now();
10060 let home = crate::run::home();
10061 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10062 let guard = if let Some(cache_dir) = cache_dir {
10063 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10064 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10065 wait.cancelled()
10066 })
10067 .await
10068 .and_then(|g| {
10069 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10070 }) {
10071 Ok(g) => Some(g),
10072 Err(e) => {
10073 return Ok((
10074 vec![CommandOutcome {
10075 command: "(waiting for the shared build cache)".into(),
10076 code: None,
10077 output_tail: e.to_string(),
10078 duration_ms: started.elapsed().as_millis() as u64,
10079 resource_blocked: true,
10080 }],
10081 false,
10082 ));
10083 }
10084 }
10085 } else {
10086 None
10087 };
10088 let _coordination = coordinate_operation(state, worktree, wait).await?;
10089 if let Some(cache_dir) = cache_dir {
10090 let identity = crate::cache::Identity::new(worktree, head);
10091 if let Err(e) = crate::cache::ensure_fresh(
10092 &home,
10093 cache_dir,
10094 &identity,
10095 &state.config.verify.cargo_manifests(),
10096 ) {
10097 state.event(
10098 "verify",
10099 format!(
10100 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10101 worktree.display(),
10102 short(head)
10103 ),
10104 );
10105 if let Some(guard) = guard {
10106 guard.release();
10107 }
10108 return Ok((
10109 vec![CommandOutcome {
10110 command: "(confirming the shared build cache is fresh)".into(),
10111 code: None,
10112 output_tail: e.to_string(),
10113 duration_ms: started.elapsed().as_millis() as u64,
10114 resource_blocked: true,
10115 }],
10116 false,
10117 ));
10118 }
10119 }
10120 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10121 if !timed_out_pids.is_empty() {
10122 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10123 }
10124 drop(_coordination);
10127 if let Some(guard) = guard {
10128 guard.release();
10129 }
10130 Ok((outcomes, retried))
10131}
10132
10133async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10145 wait_for_pids_with(
10146 pids,
10147 crate::proc::pid_alive,
10148 LEASE_RELEASE_POLL,
10149 LEASE_RELEASE_MAX_WAIT,
10150 )
10151 .await;
10152}
10153
10154async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10160 pids: &[u32],
10161 alive: F,
10162 poll: Duration,
10163 max_wait: Duration,
10164) {
10165 let deadline = Instant::now() + max_wait;
10166 loop {
10167 if pids.iter().all(|&pid| !alive(pid)) {
10168 return;
10169 }
10170 if Instant::now() >= deadline {
10171 return;
10172 }
10173 tokio::time::sleep(poll).await;
10174 }
10175}
10176
10177fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10184 outcomes.iter().any(|o| o.resource_blocked)
10185}
10186
10187enum GateFix {
10189 Retry,
10191 Stop,
10194 Defer,
10197}
10198
10199fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10207 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10208 red.peek().is_some()
10209 && red.all(|o| {
10210 !o.resource_blocked
10211 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10212 && !o.output_tail.trim().is_empty()
10213 })
10214}
10215
10216fn e2e_outcome_label(o: &CommandOutcome) -> String {
10220 if o.ok() {
10221 return "pass".to_owned();
10222 }
10223 let reason = if o.build_failed() {
10224 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10225 } else {
10226 format!("FAIL ({:?})", o.code)
10227 };
10228 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10229}
10230
10231async fn run_e2e_with_retry(
10239 state: &mut RunState,
10240 shell: &[String],
10241 commands: &[String],
10242 worktree: &Path,
10243 timeout: Duration,
10244 context: &str,
10245) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10246 let (mut e2e, mut timed_out_pids) = run_commands(
10247 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10248 )
10249 .await;
10250 for o in &e2e {
10251 state.event(
10252 "verify",
10253 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10254 );
10255 }
10256 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10260 if verify_retried {
10261 state.event(
10262 "verify",
10263 format!(
10264 "{context}: verify could not build/link, not a test result — retrying once \
10265 before concluding"
10266 ),
10267 );
10268 let retried = run_commands(
10269 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10270 )
10271 .await;
10272 e2e = retried.0;
10273 timed_out_pids.extend(retried.1);
10276 for o in &e2e {
10277 state.event(
10278 "verify",
10279 format!(
10280 "{context}: retry `{}` -> {}",
10281 o.command,
10282 e2e_outcome_label(o)
10283 ),
10284 );
10285 }
10286 }
10287 (e2e, verify_retried, timed_out_pids)
10288}
10289
10290#[allow(clippy::too_many_arguments)]
10306async fn run_commands(
10307 state: &mut RunState,
10308 node: &str,
10309 task: &str,
10310 attempt: usize,
10311 shell: &[String],
10312 commands: &[String],
10313 cwd: &Path,
10314 timeout: Duration,
10315) -> (Vec<CommandOutcome>, Vec<u32>) {
10316 if commands.is_empty() {
10317 return (Vec::new(), Vec::new());
10322 }
10323 let mut out = Vec::new();
10324 let mut timed_out_pids = Vec::new();
10325 let total = commands.len();
10326 for (idx, command) in commands.iter().enumerate() {
10327 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10328 if let Err(e) = state.save() {
10329 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10330 }
10331 let started = Instant::now();
10332 let mut cmd = tokio::process::Command::new(&shell[0]);
10333 cmd.quiet();
10334 cmd.args(&shell[1..])
10335 .arg(command)
10336 .current_dir(cwd)
10337 .stdin(std::process::Stdio::null())
10338 .stdout(std::process::Stdio::piped())
10339 .stderr(std::process::Stdio::piped())
10340 .kill_on_drop(true);
10341 let spawned = cmd.spawn();
10342 let (code, body) = match spawned {
10343 Ok(child) => {
10344 let pid = child.id();
10349 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10350 Ok(Ok(o)) => {
10351 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10352 body.push_str(&String::from_utf8_lossy(&o.stderr));
10353 (o.status.code(), body)
10354 }
10355 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10356 Err(_) => {
10357 if let Some(pid) = pid {
10358 timed_out_pids.push(pid);
10359 }
10360 (None, format!("timed out after {}s", timeout.as_secs()))
10361 }
10362 }
10363 }
10364 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10365 };
10366 out.push(CommandOutcome {
10367 command: command.clone(),
10368 code,
10369 output_tail: tail(&body, OUTPUT_TAIL),
10370 duration_ms: started.elapsed().as_millis() as u64,
10371 resource_blocked: false,
10372 });
10373 }
10374 state.task_finished(task);
10375 if let Err(e) = state.save() {
10376 tracing::warn!("could not persist the end of {task}: {e:#}");
10377 }
10378 (out, timed_out_pids)
10379}
10380
10381fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10393 let repo = repo.display();
10394 match style {
10395 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10396 MergeStyle::Squash => {
10397 let subject = message
10400 .lines()
10401 .next()
10402 .unwrap_or(branch)
10403 .replace(['\\', '"', '$', '`'], "");
10404 format!(
10405 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10406 )
10407 }
10408 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10409 }
10410}
10411
10412const PR_TITLE_MAX: usize = 240;
10423
10424fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10435 let line = queue::first_line(text)?;
10436 let chars: Vec<char> = line.chars().collect();
10437 let (kept, mark) = if chars.len() <= max {
10438 (line.to_owned(), "")
10439 } else if let Some(end) = sentence_end(&chars, max) {
10440 (chars[..end].iter().collect::<String>(), "")
10441 } else {
10442 let room = max.saturating_sub(3);
10443 let cut = if chars[room].is_whitespace() {
10446 room
10447 } else {
10448 chars[..room]
10449 .iter()
10450 .rposition(|c| c.is_whitespace())
10451 .unwrap_or(room)
10452 };
10453 let head: String = chars[..cut].iter().collect();
10454 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10455 (head.to_owned(), "...")
10456 };
10457 if kept.is_empty() {
10458 return None;
10459 }
10460 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10461 return None;
10462 }
10463 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10464 return None;
10465 }
10466 Some(format!("{kept}{mark}"))
10467}
10468
10469fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10473 const MIN: usize = 20;
10474 for i in MIN..max.min(chars.len()) {
10475 if !matches!(chars[i], '.' | '!' | '?') {
10476 continue;
10477 }
10478 let Some(&next) = chars.get(i + 1) else {
10479 continue;
10480 };
10481 if !next.is_whitespace() {
10482 continue;
10483 }
10484 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10485 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10486 continue;
10487 }
10488 let word: String = chars[..i]
10489 .iter()
10490 .rev()
10491 .take_while(|c| !c.is_whitespace())
10492 .collect::<Vec<_>>()
10493 .into_iter()
10494 .rev()
10495 .collect();
10496 let word = word.to_ascii_lowercase();
10497 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10498 continue;
10499 }
10500 let end = chars[..i]
10501 .iter()
10502 .rposition(|c| !c.is_whitespace())
10503 .map_or(i, |p| p + 1);
10504 return Some(end);
10505 }
10506 None
10507}
10508
10509struct PrMessage {
10514 title: String,
10515 body: String,
10516}
10517
10518impl PrMessage {
10519 fn commit_message(&self) -> String {
10523 format!("{}\n\n{}", self.title, self.body)
10524 }
10525}
10526
10527fn title_marker(line: &str) -> Option<&str> {
10529 let line = line.trim();
10530 let head = line.get(..6)?;
10531 head.eq_ignore_ascii_case("title:")
10532 .then(|| line[6..].trim())
10533}
10534
10535fn summary_title(summary: &str) -> Option<String> {
10540 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10541 let raw = title_marker(first)?;
10542 if raw.is_empty() {
10543 return None;
10544 }
10545 let title = queue::title_from(raw, PR_TITLE_MAX);
10546 let lower = title.to_ascii_lowercase();
10547 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10548 return None;
10549 }
10550 Some(title)
10551}
10552
10553const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10555
10556pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10558
10559fn is_review_run(state: &RunState) -> bool {
10563 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10564}
10565
10566fn review_title(state: &RunState) -> Option<String> {
10571 english_subject(
10572 state.reviewed_commits.as_ref()?.first()?,
10573 state.config.graph.github_english(),
10574 )
10575}
10576
10577fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10580 let raw = raw.trim();
10581 if raw.is_empty() {
10582 return None;
10583 }
10584 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10585 return None;
10586 }
10587 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10588 return None;
10589 }
10590 let title = queue::title_from(raw, PR_TITLE_MAX);
10591 let lower = title.to_ascii_lowercase();
10592 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10593 return None;
10594 }
10595 Some(title)
10596}
10597
10598#[derive(Debug, Clone, PartialEq, Eq)]
10601struct BranchFacts {
10602 commits: Vec<(String, String)>,
10604 stat: String,
10606}
10607
10608const STAT_FILE_LINES: usize = 25;
10610const COMMITS_MAX_CHARS: usize = 20_000;
10612
10613async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10616 let commits = git::commit_log(repo, base, branch).await.ok()?;
10617 if commits.is_empty() {
10618 return None;
10619 }
10620 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10621 let lines: Vec<&str> = stat.lines().collect();
10622 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10623 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10624 let more = format!(" ... {omitted} more file(s)");
10625 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10626 kept.push(&more);
10627 kept.push(lines[lines.len() - 1]);
10628 kept.join("\n")
10629 } else {
10630 lines.join("\n")
10631 };
10632 Some(BranchFacts { commits, stat })
10633}
10634
10635fn markdown_safe(text: &str) -> String {
10638 text.replace("</details>", "</details>")
10639 .replace("\x60\x60\x60", "~~~")
10640}
10641
10642fn neutral_title(state: &RunState, winner: char) -> String {
10643 format!(
10644 "chore: land candidate {} of run {}",
10645 winner.to_ascii_uppercase(),
10646 state.id
10647 )
10648}
10649
10650pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10656 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10657 ""
10658 } else {
10659 pr_title
10660 }
10661}
10662
10663pub fn landing_subject_source(state: &RunState) -> String {
10667 if is_review_run(state) {
10668 let winner = state.candidates.first().map_or('A', |c| c.label);
10669 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10670 }
10671 state.instruction.clone()
10672}
10673
10674fn summary_without_title(summary: &str) -> String {
10677 let mut lines = summary.trim().lines().peekable();
10678 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10679 lines.next();
10680 }
10681 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10682}
10683
10684#[cfg(test)]
10696fn pr_message(state: &RunState, winner: char) -> PrMessage {
10697 pr_message_with(state, winner, None)
10698}
10699
10700#[cfg(test)]
10703fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10704 let raw = pr_message_raw(
10705 state,
10706 winner,
10707 facts,
10708 crate::pr_outcome::FinalHead::Unreported,
10709 ChangeNote::NotAsked,
10710 );
10711 let id = crate::scrub::Identity::current();
10712 PrMessage {
10713 title: crate::scrub::scrub(&raw.title, &id),
10714 body: crate::scrub::scrub(&raw.body, &id),
10715 }
10716}
10717
10718#[derive(Debug, Clone, Copy)]
10720enum ChangeNote<'a> {
10721 NotAsked,
10723 Summary(&'a crate::run::ChangeSummary),
10725 Unavailable(&'a str),
10727}
10728
10729fn pr_message_raw(
10730 state: &RunState,
10731 winner: char,
10732 facts: Option<&BranchFacts>,
10733 head: crate::pr_outcome::FinalHead<'_>,
10734 change: ChangeNote<'_>,
10735) -> PrMessage {
10736 if !matches!(change, ChangeNote::NotAsked) {
10737 let (title, mut body) = match change {
10738 ChangeNote::Summary(c) => (
10739 c.title.clone(),
10740 format!("## Summary\n\n{}\n\n", c.body.trim()),
10741 ),
10742 ChangeNote::Unavailable(why) => (
10743 neutral_title(state, winner),
10744 format!(
10745 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10746 ),
10747 ),
10748 ChangeNote::NotAsked => unreachable!(),
10749 };
10750 body.push_str(&public_pr_evidence(state, head));
10751 body.push_str(&format!(
10752 "\n---\nmagi:run/{} magi:candidate-{}\n",
10753 state.id,
10754 winner.to_ascii_lowercase()
10755 ));
10756 return PrMessage { title, body };
10757 }
10758 pr_preview(state, winner, facts, head)
10759}
10760
10761fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10764 let mut body = crate::pr_outcome::review_rounds(state);
10765 body.push_str(&crate::pr_outcome::final_verification(state, head));
10766 let open = state.last_round_findings();
10767 if !open.is_empty() {
10768 body.push_str("## Open review findings\n\n");
10769 for f in open {
10770 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10771 }
10772 body.push('\n');
10773 }
10774 if let Some(c) = state.change_summary.as_ref()
10775 && head == crate::pr_outcome::FinalHead::At(&c.head)
10776 {
10777 body.push_str(&format!(
10778 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10779 c.base, c.head
10780 ));
10781 }
10782 body
10783}
10784
10785fn pr_preview(
10787 state: &RunState,
10788 winner: char,
10789 facts: Option<&BranchFacts>,
10790 head: crate::pr_outcome::FinalHead<'_>,
10791) -> PrMessage {
10792 let summary = state
10793 .candidates
10794 .iter()
10795 .find(|c| c.label == winner)
10796 .map(|c| c.summary.as_str())
10797 .unwrap_or_default();
10798 let review = is_review_run(state);
10801 let title = if review {
10802 facts
10803 .and_then(|f| {
10804 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10805 })
10806 .or_else(|| review_title(state))
10807 .or_else(|| {
10808 state
10809 .candidates
10810 .iter()
10811 .find(|c| c.label == winner)
10812 .filter(|c| !c.branch.starts_with("magi/"))
10813 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10814 })
10815 .unwrap_or_else(|| neutral_title(state, winner))
10816 } else {
10817 summary_title(summary).unwrap_or_else(|| {
10818 english_title(
10819 &state.instruction,
10820 PR_TITLE_MAX,
10821 state.config.graph.github_english(),
10822 )
10823 .unwrap_or_else(|| neutral_title(state, winner))
10824 })
10825 };
10826
10827 let mut body = String::new();
10828 let what = summary_without_title(summary);
10829 if !what.is_empty() {
10830 body.push_str("## Summary\n\n");
10831 body.push_str(&what);
10832 body.push_str("\n\n");
10833 }
10834
10835 body.push_str(&crate::pr_outcome::review_rounds(state));
10837 body.push_str(&crate::pr_outcome::final_verification(state, head));
10838 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10839
10840 let open = state.open_findings();
10841 if !open.is_empty() {
10842 body.push_str("## Open review findings\n\n");
10843 for f in &open {
10844 body.push_str(&format!(
10845 "- `{}` [{:?}] {}\n",
10846 f.id, f.severity, "Context retained in local review records."
10847 ));
10848 }
10849 body.push('\n');
10850 }
10851
10852 if let Some(fix) = fix
10853 && !fix.rejected.is_empty()
10854 {
10855 body.push_str("## Declined by the fixer\n\n");
10856 for r in &fix.rejected {
10857 body.push_str(&format!(
10858 "- `{}`: {}\n",
10859 r.id, "Reason retained in local review records."
10860 ));
10861 }
10862 body.push('\n');
10863 }
10864
10865 if review {
10866 body.push_str("## Commits under review\n\n");
10868 if let Some(facts) = facts {
10869 let mut left = COMMITS_MAX_CHARS;
10870 for (i, (subject, text)) in facts.commits.iter().enumerate() {
10871 let mut entry = format!("- {}\n", markdown_safe(subject));
10872 for l in markdown_safe(text).lines() {
10873 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
10874 }
10875 if left == 0 {
10876 body.push_str(&format!(
10877 "- ... {} more commit(s)\n",
10878 facts.commits.len() - i
10879 ));
10880 break;
10881 }
10882 if entry.len() > left {
10883 let mut end = left;
10886 while !entry.is_char_boundary(end) {
10887 end -= 1;
10888 }
10889 entry.truncate(end);
10890 entry.push_str("\n ... (truncated)\n");
10891 left = 0;
10892 } else {
10893 left -= entry.len();
10894 }
10895 body.push_str(&entry);
10896 }
10897 if !facts.stat.trim().is_empty() {
10898 body.push_str(&format!(
10899 "\n## Diff stat\n\n```\n{}\n```\n",
10900 markdown_safe(facts.stat.trim())
10901 ));
10902 }
10903 } else {
10904 match &state.reviewed_commits {
10905 Some(subjects) => {
10906 for s in subjects {
10907 body.push_str(&format!("- {}\n", s.trim()));
10908 }
10909 }
10910 None => {
10911 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
10914 body.push_str(rest.trim());
10915 body.push('\n');
10916 }
10917 }
10918 }
10919 } else {
10920 let task = state.instruction.trim();
10921 let task = if task.is_empty() {
10922 "(empty task)"
10923 } else {
10924 task
10925 };
10926 body.push_str(&format!(
10927 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
10928 task.replace("</details>", "</details>")
10929 ));
10930 }
10931
10932 body.push_str(&format!(
10933 "\n---\nmagi:run/{} magi:candidate-{}\n",
10934 state.id,
10935 winner.to_ascii_lowercase()
10936 ));
10937
10938 PrMessage { title, body }
10939}
10940
10941fn seeded_instruction(state: &RunState) -> String {
10945 match refs::describe(&state.seeds) {
10946 Some(facts) => format!(
10947 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
10948 Candidates start from the unmerged branch named above, when there \
10949 is one, and carry any unmerged commit named by sha as a \
10950 cherry-pick. Check that this is what the task meant before \
10951 building on it.",
10952 state.instruction
10953 ),
10954 None => state.instruction.clone(),
10955 }
10956}
10957
10958async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
10962 let base = &state.base_branch;
10963 let mut against = base.clone();
10964 if mode == MergeMode::Pr {
10965 let remote = &state.config.merge.remote;
10969 let tracking = format!("{remote}/{base}");
10970 let fetched = git::fetch(repo, remote, base).await;
10971 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
10972 against = tracking;
10973 } else {
10974 return false;
10975 }
10976 }
10977 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
10978}
10979
10980fn empty_candidate_detail(state: &RunState, base: &str) -> String {
10983 let mut detail = format!(
10984 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
10985 nothing to open a pull request for"
10986 );
10987 match refs::describe(&state.seeds) {
10988 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
10989 None => detail.push_str(
10990 "\nThe task names no existing branch or commit; if it means to land work \
10991 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
10992 ),
10993 }
10994 detail
10995}
10996
10997#[derive(Debug, PartialEq, Eq)]
11000enum PrPlan {
11001 Create,
11002 Adopt { url: String, title: String },
11003 Stop(String),
11004}
11005
11006fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11009 match found {
11010 Ok(land::OpenPr::None) => PrPlan::Create,
11011 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11012 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11013 "several open pull requests exist for this branch, not picking one: {}",
11014 urls.join(" ")
11015 )),
11016 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11017 }
11018}
11019
11020pub(crate) async fn gh_pr_create(
11022 cwd: &Path,
11023 base: &str,
11024 head: &str,
11025 title: &str,
11026 body: &str,
11027) -> Result<String> {
11028 let out = tokio::process::Command::new("gh")
11029 .args([
11030 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11031 ])
11032 .current_dir(cwd)
11033 .quiet()
11034 .stdin(std::process::Stdio::null())
11035 .output()
11036 .await
11037 .context("spawn gh")?;
11038 if out.status.success() {
11039 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11040 } else {
11041 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11042 }
11043}
11044
11045pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11054 if crate::pr_recovery::sources_retained(home, &state.id) {
11055 return Ok(Vec::new());
11056 }
11057 let repo = state.repo.clone();
11058 let wait = repository_wait(state, None);
11061 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11062 Ok(lease) => lease,
11063 Err(e) => {
11064 state.save_under(home)?;
11065 return Err(e);
11066 }
11067 };
11068 let root = state.worktree_root();
11069 let winner = state.tally.as_ref().map(|t| t.winner);
11070 let mut removed = Vec::new();
11071
11072 for i in 0..state.candidates.len() {
11073 let c = state.candidates[i].clone();
11074 let is_winner = Some(c.label) == winner;
11075 if is_winner && !drop_winner {
11076 continue;
11077 }
11078 if c.worktree.exists() {
11079 git::worktree_remove(&repo, &c.worktree).await.ok();
11080 removed.push(c.worktree.to_string_lossy().into_owned());
11081 }
11082 let handed_over = state.released_branches.contains(&c.branch);
11085 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
11086 git::branch_delete(&repo, &c.branch).await.ok();
11087 removed.push(c.branch.clone());
11088 }
11089 state.candidates[i].folded = true;
11090 }
11091
11092 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11093 let path = name.path();
11094 let keep = !drop_winner
11095 && winner.is_some_and(|w| {
11096 path.file_name()
11097 .is_some_and(|n| n == format!("cand-{w}").as_str())
11098 });
11099 if keep {
11100 continue;
11101 }
11102 git::worktree_remove(&repo, &path).await.ok();
11103 removed.push(path.to_string_lossy().into_owned());
11104 }
11105
11106 remove_if_empty(&root);
11115
11116 if state.enabled_worktree_config && drop_winner {
11117 git::release_worktree_config(&repo).await.ok();
11121 state.enabled_worktree_config = false;
11122 }
11123 state.save_under(home)?;
11124 Ok(removed)
11125}
11126
11127fn remove_if_empty(dir: &Path) {
11138 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11139 std::fs::remove_dir(dir).ok();
11140 }
11141}
11142
11143pub fn worst_open(state: &RunState) -> Option<Severity> {
11145 state
11146 .reviews
11147 .last()?
11148 .reviews
11149 .iter()
11150 .flat_map(|r| r.findings.iter())
11151 .map(|f| f.severity)
11152 .max()
11153}
11154
11155async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11159 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11160 git::acquire_worktree_config(repo).await?;
11161 state.enabled_worktree_config = true;
11162 }
11163 Ok(())
11164}
11165
11166#[cfg(test)]
11167mod tests {
11168 #[test]
11169 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11170 let leaked = vec!["chore(deps): update a crate".to_owned()];
11171 let own = "fix(daemon): apply a chosen action";
11172 assert!(should_retitle("", own, &leaked));
11173 assert!(should_retitle(
11174 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11175 own,
11176 &leaked
11177 ));
11178 assert!(should_retitle(
11179 "chore: land candidate A of run 1",
11180 own,
11181 &leaked
11182 ));
11183 assert!(should_retitle(
11184 "magi: candidate A (uncommitted work)",
11185 own,
11186 &leaked
11187 ));
11188 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11189 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11190 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11191 }
11192
11193 #[test]
11194 fn pr_merge_plan_creates_adopts_or_stops() {
11195 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11196 assert_eq!(
11197 pr_merge_plan(Ok(land::OpenPr::One {
11198 url: "u".into(),
11199 title: "t".into()
11200 })),
11201 PrPlan::Adopt {
11202 url: "u".into(),
11203 title: "t".into()
11204 }
11205 );
11206 let PrPlan::Stop(many) =
11207 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11208 else {
11209 panic!("many must stop");
11210 };
11211 assert!(many.contains('a') && many.contains('b'));
11212 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11213 panic!("a failed lookup must stop");
11214 };
11215 assert!(err.contains("bad token"));
11216 }
11217
11218 use super::*;
11219 use crate::run::GateStatus;
11220 use std::collections::BTreeMap;
11221 use std::time::Duration;
11222
11223 fn conductor() -> AgentSpec {
11224 AgentSpec {
11225 id: "conductor".to_owned(),
11226 kind: crate::config::AgentKind::Command,
11227 model: None,
11228 command: vec!["true".to_owned()],
11229 extra_args: Vec::new(),
11230 env: BTreeMap::new(),
11231 prompt_delivery: None,
11232 }
11233 }
11234
11235 fn spec(id: &str) -> AgentSpec {
11236 AgentSpec {
11237 id: id.to_owned(),
11238 kind: crate::config::AgentKind::Command,
11239 model: None,
11240 command: vec!["true".to_owned()],
11241 extra_args: Vec::new(),
11242 env: BTreeMap::new(),
11243 prompt_delivery: None,
11244 }
11245 }
11246
11247 fn ids(xs: &[&str]) -> BTreeSet<String> {
11248 xs.iter().map(|s| (*s).to_owned()).collect()
11249 }
11250
11251 #[test]
11252 fn pick_successor_skips_an_agent_another_seat_holds() {
11253 let roster = [spec("a"), spec("b"), spec("c")];
11255 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11256 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11257 }
11258
11259 #[test]
11260 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11261 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11263 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11264 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11265 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11266 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11267 }
11268
11269 #[test]
11270 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11271 let roster = [spec("a"), spec("b")];
11272 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11273 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11274 let carried = ids(&["b"]);
11275 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11276 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11277 }
11278
11279 #[test]
11280 fn pick_successor_returns_none_without_any_untried_successor() {
11281 let roster = [spec("a"), spec("b")];
11282 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11283 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11284 let carried = ids(&["a"]);
11285 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11286 }
11287
11288 #[test]
11289 fn pick_successor_rescue_avoids_another_seats_occupant() {
11290 let roster = [spec("a"), spec("b"), spec("c")];
11292 let carried = ids(&["b", "c"]);
11293 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11294 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11295 }
11296
11297 #[test]
11298 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11299 let roster = [spec("a"), spec("b"), spec("c")];
11300 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11301 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11302 }
11303
11304 #[test]
11305 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11306 let roster = [spec("a"), spec("b"), spec("c")];
11307 let failed = ids(&["a", "b", "c"]);
11308 let mut tried = ids(&["b"]);
11310 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11311 assert_eq!(first.id, "a");
11312 tried.insert(first.id.clone());
11313 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11314 assert_eq!(second.id, "c");
11315 tried.insert(second.id.clone());
11316 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11317 }
11318
11319 #[test]
11320 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11321 let roster = [spec("a"), spec("b")];
11322 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11323 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11324 }
11325
11326 #[test]
11327 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11328 let roster = [spec("a"), spec("b"), spec("c")];
11329 let h = SeatHistory {
11330 failed: ids(&["a"]),
11331 last_ok: Some("b".to_owned()),
11332 last_fail: None,
11333 };
11334 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11335 let gone = SeatHistory {
11337 last_ok: Some("zzz".to_owned()),
11338 ..h.clone()
11339 };
11340 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11341 let failed = SeatHistory {
11342 failed: ids(&["a", "b"]),
11343 last_ok: Some("b".to_owned()),
11344 last_fail: None,
11345 };
11346 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11347 }
11348
11349 #[test]
11350 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11351 let first = SeatState::new("review-1", "alpha", 7);
11352 let next = handover_seat("review-1", "gamma", 7);
11353 assert_ne!(first.claude_session, next.claude_session);
11354 }
11355
11356 #[test]
11357 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11358 let mut state = state_with_summary("x", "y");
11359 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11360 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11361 assert_ne!(a.claude_session, b.claude_session);
11362 }
11363
11364 #[test]
11365 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11366 let roster = [spec("a"), spec("b")];
11367 let h = SeatHistory {
11368 failed: ids(&["a", "b"]),
11369 ..SeatHistory::default()
11370 };
11371 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11372 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11373 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11374 }
11375
11376 #[test]
11383 fn failure_signature_ignores_numbers_and_paths() {
11384 assert_eq!(
11385 failure_signature("exited with Some(2) and no usable output"),
11386 failure_signature("exited with Some(137) and no usable output")
11387 );
11388 assert_eq!(
11389 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11390 failure_signature("cannot open /var/x.txt: denied")
11391 );
11392 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11393 }
11394
11395 #[test]
11396 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11397 let other = FailClass::Other("x".into());
11398 assert!(should_hand_over(None, &FailClass::Quota));
11399 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11400 assert!(should_hand_over(
11401 Some(&FailClass::Timeout),
11402 &FailClass::Timeout
11403 ));
11404 assert!(should_hand_over(None, &other));
11405 assert!(!should_hand_over(Some(&other), &other));
11406 assert!(should_hand_over(
11407 Some(&other),
11408 &FailClass::Other("y".into())
11409 ));
11410 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11412 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11413 }
11414
11415 #[test]
11416 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11417 let a = SeatState::new("judge-1", "alpha", 7);
11418 let b = handover_seat("judge-1", "beta", 7);
11419 assert_ne!(a.claude_session, b.claude_session);
11420 assert_eq!(b.turns, 0);
11421 }
11422
11423 #[test]
11424 fn a_timeout_is_classified_apart_from_other_failures() {
11425 assert_eq!(
11426 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11427 Some(FailClass::Timeout)
11428 );
11429 assert!(matches!(
11430 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11431 Some(FailClass::Other(_))
11432 ));
11433 }
11434
11435 #[test]
11436 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11437 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11438 let tried = BTreeSet::from(["beta".to_owned()]);
11439 let next = next_untried_in_roster(&roster, 1, &tried);
11442 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11443 }
11444
11445 #[test]
11446 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11447 let roster = vec![spec("alpha"), spec("beta")];
11448 let tried = BTreeSet::from(["beta".to_owned()]);
11449 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11453 }
11454
11455 #[test]
11456 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11457 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11458 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11459 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11463 }
11464
11465 #[test]
11466 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11467 let roster = vec![spec("a"), spec("a"), spec("b")];
11468 let tried = BTreeSet::from(["a".to_owned()]);
11469 let next = next_untried_in_roster(&roster, 0, &tried);
11470 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11471 }
11472
11473 #[test]
11474 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11475 let roster = vec![spec("a"), spec("b")];
11476 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11477 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11478 }
11479
11480 #[test]
11481 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11482 let dir = tempfile::tempdir().unwrap();
11483 let bay = dir.path().join("ffff");
11484
11485 remove_if_empty(&bay);
11487 assert!(!bay.exists());
11488
11489 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11492 remove_if_empty(&bay);
11493 assert!(bay.exists(), "non-empty directory must survive");
11494
11495 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11497 remove_if_empty(&bay);
11498 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11499 }
11500
11501 #[test]
11510 fn a_full_panel_that_found_nothing_is_clean() {
11511 assert!(round_is_clean(
11512 0,
11513 true,
11514 2,
11515 2,
11516 0,
11517 IncompleteReviewPolicy::Block
11518 ));
11519 }
11520
11521 #[test]
11522 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11523 assert!(!round_is_clean(
11524 0,
11525 true,
11526 1,
11527 2,
11528 0,
11529 IncompleteReviewPolicy::Block
11530 ));
11531 }
11532
11533 #[test]
11534 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11535 assert!(!round_is_clean(
11536 1,
11537 true,
11538 1,
11539 2,
11540 0,
11541 IncompleteReviewPolicy::Warn
11542 ));
11543 }
11544
11545 #[test]
11546 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11547 assert!(round_is_clean(
11548 0,
11549 true,
11550 1,
11551 2,
11552 0,
11553 IncompleteReviewPolicy::Warn
11554 ));
11555 }
11556
11557 #[test]
11558 fn a_full_panel_with_an_open_finding_is_not_clean() {
11559 assert!(!round_is_clean(
11560 1,
11561 true,
11562 2,
11563 2,
11564 0,
11565 IncompleteReviewPolicy::Block
11566 ));
11567 }
11568
11569 #[test]
11570 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11571 assert!(!round_is_clean(
11572 0,
11573 false,
11574 2,
11575 2,
11576 0,
11577 IncompleteReviewPolicy::Block
11578 ));
11579 }
11580
11581 #[test]
11588 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11589 assert!(round_is_clean(
11592 0,
11593 true,
11594 1,
11595 2,
11596 1,
11597 IncompleteReviewPolicy::Block
11598 ));
11599 }
11600
11601 #[test]
11602 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11603 assert!(!round_is_clean(
11606 0,
11607 true,
11608 1,
11609 2,
11610 0,
11611 IncompleteReviewPolicy::Block
11612 ));
11613 }
11614
11615 #[test]
11616 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11617 assert!(!round_is_clean(
11618 1,
11619 true,
11620 1,
11621 2,
11622 1,
11623 IncompleteReviewPolicy::Block
11624 ));
11625 assert!(!round_is_clean(
11626 0,
11627 false,
11628 1,
11629 2,
11630 1,
11631 IncompleteReviewPolicy::Block
11632 ));
11633 }
11634
11635 #[test]
11636 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11637 assert!(!round_is_clean(
11641 0,
11642 true,
11643 0,
11644 2,
11645 2,
11646 IncompleteReviewPolicy::Block
11647 ));
11648 }
11649
11650 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11651 CommandOutcome {
11652 command: "test".to_owned(),
11653 code,
11654 output_tail: String::new(),
11655 duration_ms: 0,
11656 resource_blocked,
11657 }
11658 }
11659
11660 #[test]
11661 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11662 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11663 assert!(
11664 !verify_inconclusive(&[outcome(Some(1), false)]),
11665 "an ordinary failure is still evidence about the patch"
11666 );
11667 assert!(verify_inconclusive(&[outcome(None, true)]));
11668 assert!(
11669 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11670 "one inconclusive outcome taints the whole batch"
11671 );
11672 assert!(!verify_inconclusive(&[]));
11673 }
11674
11675 #[tokio::test]
11676 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11677 let calls = std::sync::atomic::AtomicUsize::new(0);
11681 let started = Instant::now();
11682 wait_for_pids_with(
11683 &[123],
11684 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11685 Duration::from_millis(5),
11686 Duration::from_secs(5),
11687 )
11688 .await;
11689 assert!(
11690 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11691 "must keep checking rather than deciding on the first answer"
11692 );
11693 assert!(
11694 started.elapsed() < Duration::from_secs(1),
11695 "must return the moment it is confirmed dead, not wait out the ceiling"
11696 );
11697 }
11698
11699 #[tokio::test]
11700 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11701 let started = Instant::now();
11702 wait_for_pids_with(
11703 &[123],
11704 |_| true, Duration::from_millis(5),
11706 Duration::from_millis(30),
11707 )
11708 .await;
11709 let elapsed = started.elapsed();
11710 assert!(
11711 elapsed >= Duration::from_millis(30),
11712 "must not give up before its own ceiling: {elapsed:?}"
11713 );
11714 assert!(
11715 elapsed < Duration::from_secs(1),
11716 "must not wait past its own ceiling either: {elapsed:?}"
11717 );
11718 }
11719
11720 #[tokio::test]
11721 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
11722 let started = Instant::now();
11723 wait_for_pids_with(
11724 &[],
11725 |_| true,
11726 Duration::from_secs(5),
11727 Duration::from_secs(5),
11728 )
11729 .await;
11730 assert!(
11731 started.elapsed() < Duration::from_millis(200),
11732 "an empty pid list has nothing to confirm"
11733 );
11734 }
11735
11736 fn review_round(
11742 clean: bool,
11743 blocking: usize,
11744 answered: usize,
11745 expected: usize,
11746 progressed: bool,
11747 e2e_ok: bool,
11748 ) -> ReviewRound {
11749 ReviewRound {
11750 round: 1,
11751 head: "h".to_owned(),
11752 verified_head: None,
11753 verified_at: None,
11754 reviews: Vec::new(),
11755 e2e: vec![CommandOutcome {
11756 command: "test".to_owned(),
11757 code: Some(if e2e_ok { 0 } else { 1 }),
11758 output_tail: String::new(),
11759 duration_ms: 0,
11760 resource_blocked: false,
11761 }],
11762 verify_retried: false,
11763 e2e_deferred: false,
11764 e2e_defer_reason: None,
11765 fix: None,
11766 blocking,
11767 answered,
11768 expected,
11769 clean,
11770 progressed,
11771 vote_split: false,
11772 reconsideration: Vec::new(),
11773 verdict: None,
11774 }
11775 }
11776
11777 #[test]
11778 fn review_conclusion_is_none_when_nothing_has_run() {
11779 assert_eq!(review_conclusion(&[], 3), None);
11780 }
11781
11782 #[test]
11783 fn review_conclusion_is_none_while_rounds_remain() {
11784 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
11785 assert_eq!(review_conclusion(&rounds, 3), None);
11786 }
11787
11788 #[test]
11789 fn review_conclusion_is_gating_once_a_round_is_clean() {
11790 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
11791 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
11792 }
11793
11794 #[test]
11795 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
11796 let rounds = vec![
11797 review_round(false, 1, 2, 2, true, true),
11798 review_round(false, 1, 2, 2, true, true),
11799 ];
11800 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
11801 }
11802
11803 #[test]
11804 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
11805 let rounds = vec![
11806 review_round(false, 1, 2, 2, true, true),
11807 review_round(false, 1, 2, 2, true, false),
11808 ];
11809 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
11810 }
11811
11812 #[test]
11813 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
11814 let mut blocked = review_round(false, 1, 2, 2, true, false);
11821 blocked.e2e[0].resource_blocked = true;
11822 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
11823 assert_eq!(review_conclusion(&rounds, 2), None);
11824 }
11825
11826 #[test]
11827 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
11828 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
11830 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
11831 }
11832
11833 #[test]
11834 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
11835 let rounds = vec![
11836 review_round(false, 1, 2, 2, false, true),
11837 review_round(false, 1, 2, 2, false, true),
11838 ];
11839 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
11840 }
11841
11842 fn secs(n: u64) -> Duration {
11843 Duration::from_secs(n)
11844 }
11845
11846 #[tokio::test]
11849 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
11850 ask_test_home();
11851 let tmp = tempfile::tempdir().unwrap();
11852 let repo = tmp.path().join("repo");
11853 std::fs::create_dir_all(&repo).unwrap();
11854 init_repo(&repo);
11855 let g = |args: &[&str]| {
11856 let out = std::process::Command::new("git")
11857 .args(args)
11858 .current_dir(&repo)
11859 .quiet()
11860 .output()
11861 .unwrap();
11862 assert!(
11863 out.status.success(),
11864 "{}",
11865 String::from_utf8_lossy(&out.stderr)
11866 );
11867 };
11868 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
11869 g(&["checkout", "-b", "historical"]);
11870 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
11871 g(&["add", "."]);
11872 g(&["commit", "-m", "old alternative"]);
11873 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
11874 g(&["checkout", "-b", "magi/task-selected", &base]);
11875 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
11876 g(&["add", "."]);
11877 g(&["commit", "-m", "measurement infrastructure"]);
11878 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
11879 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
11880 g(&["add", "."]);
11881 g(&["commit", "-m", "dependent implementation"]);
11882 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11883 g(&["checkout", "main"]);
11884 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
11885 g(&["add", "."]);
11886 g(&["commit", "-m", "unrelated base change"]);
11887 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
11888 let origin = tmp.path().join("origin.git");
11889 g(&[
11890 "clone",
11891 "--bare",
11892 repo.to_str().unwrap(),
11893 origin.to_str().unwrap(),
11894 ]);
11895 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
11896 g(&["fetch", "origin"]);
11897 let selection = crate::pr_recovery::Selection {
11898 url: "https://example.test/project/pull/1".into(),
11899 branch: "magi/task-selected".into(),
11900 head: head.clone(),
11901 instruction: "preserve measured behavior".into(),
11902 approved_at: jiff::Timestamp::now(),
11903 };
11904 let mut template = runner_at(RunStatus::Reviewing);
11905 template.state.repo = repo.clone();
11906 template.state.base_commit = current.clone();
11907 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
11908 template.state.origin = Some(crate::run::Origin::queue("selected"));
11909 template.state.pr_head = Some(selection.branch.clone());
11910 template.state.pr_recovery = Some(selection.clone());
11911 let mut runner =
11912 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
11913 .await
11914 .unwrap();
11915 let wt = runner.state.winner().unwrap().worktree.clone();
11916 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
11917 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
11918 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
11919 assert!(runner.state.seeds.is_empty());
11920 assert_eq!(runner.state.instruction, "preserve measured behavior");
11921 runner.sync_to_base().await.unwrap();
11922 assert_eq!(
11923 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
11924 "prerequisite"
11925 );
11926 assert_eq!(
11927 std::fs::read_to_string(wt.join("implementation")).unwrap(),
11928 "measured implementation"
11929 );
11930 assert_eq!(
11931 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
11932 "current main change"
11933 );
11934 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
11935 assert_eq!(
11936 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
11937 head
11938 );
11939
11940 let mut partial = runner_at(RunStatus::Reviewing);
11942 partial.state.repo = repo.clone();
11943 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
11944 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
11945 run: Some("exit 1".into()),
11946 ..Default::default()
11947 }];
11948 let selection = runner.state.pr_recovery.clone().unwrap();
11949 partial.state.pr_recovery = Some(selection.clone());
11950 partial.state.pr_head = Some(selection.branch.clone());
11951 partial.state.origin = Some(crate::run::Origin::queue("selected"));
11952 let id = partial.state.id.clone();
11953 let error = match Runner::open_selected(
11954 &repo,
11955 partial.state,
11956 partial.roles,
11957 1,
11958 runner.state.base_commit.clone(),
11959 selection,
11960 )
11961 .await
11962 {
11963 Ok(_) => panic!("setup must fail"),
11964 Err(e) => e,
11965 };
11966 assert_eq!(
11967 error
11968 .downcast_ref::<crate::pr_recovery::StartupFailed>()
11969 .unwrap()
11970 .run,
11971 id
11972 );
11973 let failed = RunState::load(&id).unwrap();
11974 assert_eq!(failed.status, RunStatus::Blocked);
11975 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
11976 assert!(Runner::resume(&id).is_err());
11977 runner.state.status = RunStatus::Ready;
11978 runner.state.driver_exited = true;
11979 runner.state.save().unwrap();
11980 let mut retry = runner_at(RunStatus::Reviewing);
11981 retry.state.repo = repo.clone();
11982 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
11983 let mut reselected = runner.state.pr_recovery.clone().unwrap();
11984 reselected.approved_at = jiff::Timestamp::now();
11985 retry.state.origin = Some(crate::run::Origin::queue("selected"));
11986 retry.state.pr_head = Some(reselected.branch.clone());
11987 retry.state.pr_recovery = Some(reselected.clone());
11988 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
11989 .await
11990 .unwrap();
11991 let restarted = Runner::open_selected(
11992 &repo,
11993 retry.state,
11994 retry.roles,
11995 1,
11996 runner.state.base_commit.clone(),
11997 reselected,
11998 )
11999 .await
12000 .unwrap();
12001 assert_eq!(
12002 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12003 .await
12004 .unwrap(),
12005 head
12006 );
12007 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12008 assert_eq!(
12009 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12010 head
12011 );
12012 }
12013
12014 #[tokio::test]
12015 #[cfg(unix)]
12016 async fn fallback_and_interrupted_restoration_require_coordination() {
12017 use std::os::fd::AsRawFd as _;
12018 ask_test_home();
12019 let tmp = tempfile::tempdir().unwrap();
12020 let mut runner = registered_runner(tmp.path()).await;
12021 runner.state.config.graph.timeout_resource_wait = 1;
12022 let winner = runner.state.candidates[0].clone();
12023 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12024 let leftover = winner.worktree.join("leftover");
12025 std::fs::write(&leftover, "untrusted edits").unwrap();
12026 let next = tmp.path().join("next-fixer");
12027 let mut failed = spec("first");
12028 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12029 let mut successor = spec("second");
12030 successor.command = vec![
12031 "sh".into(),
12032 "-c".into(),
12033 format!("touch '{}'; echo done", next.display()),
12034 ];
12035 let attempts = vec![
12036 (failed.clone(), "first".into()),
12037 (successor.clone(), "second".into()),
12038 ];
12039 let lock_path = runner.state.repo.join(".git/updater.lock");
12040 let external = std::fs::File::create(&lock_path).unwrap();
12041 assert_eq!(
12042 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12043 0
12044 );
12045 let artifacts = tmp.path().join("artifacts");
12046 let build = |spec, seat| SeatJob {
12047 spec,
12048 seat,
12049 cwd: winner.worktree.clone(),
12050 prompt: "fixture".into(),
12051 timeout: Duration::from_secs(10),
12052 allow_write: true,
12053 sessions: false,
12054 artifacts: artifacts.clone(),
12055 stem: "fix".into(),
12056 handover: None,
12057 };
12058 let (_, _, out) = runner
12059 .ask_fixer_among(
12060 attempts.clone(),
12061 "graft",
12062 None,
12063 Some((&winner.worktree, &head)),
12064 build,
12065 )
12066 .await;
12067 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12068 assert!(leftover.exists(), "contention must prevent clean");
12069 assert!(!next.exists());
12070 assert!(runner.state.handovers.is_empty());
12071 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12073 from_run: "earlier".into(),
12074 generation: 1,
12075 mode: crate::recovery::Mode::VerifyRepair,
12076 reasons: Vec::new(),
12077 reviewed_head: None,
12078 rounds: Vec::new(),
12079 open_findings: Vec::new(),
12080 evidence: None,
12081 criteria_inherited: true,
12082 repair: crate::recovery::Repair::Started {
12083 before: head.clone(),
12084 },
12085 });
12086 assert!(
12087 runner
12088 .recovery_repair(&winner)
12089 .await
12090 .unwrap_err()
12091 .is::<crate::repo_lock::ResourceBlocked>()
12092 );
12093 assert!(matches!(
12094 runner.state.recovery.as_ref().unwrap().repair,
12095 crate::recovery::Repair::Started { .. }
12096 ));
12097 assert!(leftover.exists());
12098 drop(external);
12099 runner.recovery_repair(&winner).await.unwrap();
12100 assert!(!leftover.exists());
12101 assert!(matches!(
12102 runner.state.recovery.as_ref().unwrap().repair,
12103 crate::recovery::Repair::Done { .. }
12104 ));
12105 std::fs::remove_file(&lock_path).unwrap();
12107 std::fs::create_dir(&lock_path).unwrap();
12108 std::fs::write(&leftover, "untrusted edits").unwrap();
12109 let (_, _, out) = runner
12110 .ask_fixer_among(
12111 attempts,
12112 "graft",
12113 None,
12114 Some((&winner.worktree, &head)),
12115 build,
12116 )
12117 .await;
12118 assert!(matches!(out, AgentOutcome::Failed(_)));
12119 assert!(leftover.exists());
12120 assert!(!next.exists());
12121 assert_eq!(
12122 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12123 head
12124 );
12125 }
12126
12127 #[cfg(unix)]
12128 async fn registered_runner(root: &Path) -> Runner {
12129 ask_test_home();
12130 let repo = root.join("repo");
12131 std::fs::create_dir_all(&repo).unwrap();
12132 init_repo(&repo);
12133 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12134 let mut runner = runner_at(RunStatus::Prep);
12135 runner.state.repo = repo.clone();
12136 runner.state.config.coordination = crate::config::test_coordination();
12137 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12138 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12139 runner.state.config.graph.advise = false;
12140 runner.roles.implementers = vec![spec("implement")];
12141 runner.roles.judges = vec![spec("judge")];
12142 runner.prep().await.unwrap();
12143 runner
12144 }
12145
12146 #[tokio::test]
12147 #[cfg(unix)]
12148 async fn parked_implementer_persists_completed_work() {
12149 for registered in [false, true] {
12150 let tmp = tempfile::tempdir().unwrap();
12151 let mut runner = registered_runner(tmp.path()).await;
12152 if !registered {
12153 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12154 }
12155 let marker = tmp.path().join("started");
12156 let release = tmp.path().join("release");
12157 let mut implementer = spec("implement");
12158 implementer.command = vec![
12159 "sh".into(),
12160 "-c".into(),
12161 format!(
12162 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12163 marker.display(),
12164 release.display()
12165 ),
12166 ];
12167 runner.roles.implementers = vec![implementer.clone()];
12168 runner.state.config.agents = vec![implementer];
12169 let pause = runner.pause.clone();
12170 let (result, ()) = tokio::join!(runner.implement(), async {
12171 tokio::time::timeout(Duration::from_secs(10), async {
12172 while !marker.exists() {
12173 tokio::time::sleep(Duration::from_millis(10)).await;
12174 }
12175 })
12176 .await
12177 .unwrap();
12178 pause.park();
12179 std::fs::write(&release, "go").unwrap();
12180 });
12181 result.unwrap();
12182 let candidate = &runner.state.candidates[0];
12183 assert!(candidate.commits > 0, "{candidate:?}");
12184 assert!(candidate.summary.contains("saved-result"));
12185 assert_eq!(candidate.files, 1);
12186 let before = candidate.commits;
12187 runner.implement().await.unwrap();
12188 assert_eq!(runner.state.candidates[0].commits, before);
12189 }
12190 }
12191
12192 #[tokio::test]
12193 #[cfg(unix)]
12194 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12195 let tmp = tempfile::tempdir().unwrap();
12196 let mut runner = registered_runner(tmp.path()).await;
12197 let winner = runner.state.candidates[0].clone();
12198 let cache = tmp.path().join("cache");
12199 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12200 runner.state.config.graph.timeout_resource_wait = 0;
12201 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12202 let crate::cache::AcquireOutcome::Acquired(held) =
12203 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12204 else {
12205 panic!("cache must be free")
12206 };
12207 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12208 from_run: "earlier".into(),
12209 generation: 1,
12210 mode: crate::recovery::Mode::VerifyRepair,
12211 reasons: Vec::new(),
12212 reviewed_head: None,
12213 rounds: Vec::new(),
12214 open_findings: Vec::new(),
12215 evidence: None,
12216 criteria_inherited: true,
12217 repair: crate::recovery::Repair::NotStarted,
12218 });
12219 runner.recovery_repair(&winner).await.unwrap();
12220 assert!(matches!(
12221 runner.state.recovery.as_ref().unwrap().repair,
12222 crate::recovery::Repair::WaitingForRestore { .. }
12223 ));
12224 drop(held);
12225 runner.state.resource_block = None;
12226 runner.state.config.graph.timeout_resource_wait = 1;
12227 runner.recovery_repair(&winner).await.unwrap();
12228 assert!(matches!(
12229 runner.state.recovery.as_ref().unwrap().repair,
12230 crate::recovery::Repair::Done { .. }
12231 ));
12232 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12233 }
12234
12235 #[tokio::test]
12236 #[cfg(unix)]
12237 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12238 use std::os::fd::AsRawFd as _;
12239 for graft in [false, true] {
12240 let tmp = tempfile::tempdir().unwrap();
12241 let mut runner = registered_runner(tmp.path()).await;
12242 runner.state.config.graph.timeout_resource_wait = 1;
12243 let winner = runner.state.candidates[0].clone();
12244 let marker = tmp.path().join("fixer-started");
12245 let mut fixer = spec("fixer");
12246 fixer.command = vec![
12247 "sh".into(),
12248 "-c".into(),
12249 format!("echo started >> '{}'; echo done", marker.display()),
12250 ];
12251 runner.roles.fixer = Some(vec![fixer]);
12252 if graft {
12253 let mut loser = winner.clone();
12254 loser.label = 'B';
12255 loser.index = 1;
12256 runner.state.candidates.push(loser);
12257 runner.state.config.graph.graft_runner_up = true;
12258 runner.state.tally = Some(Tally {
12259 first_choice: BTreeMap::new(),
12260 borda: BTreeMap::new(),
12261 winner: winner.label,
12262 rankings: 1,
12263 unanimous_initial: true,
12264 deliberated: false,
12265 changed_votes: 0,
12266 unanimous_final: true,
12267 tie_break: None,
12268 judges: 1,
12269 present: 1,
12270 quorum: 1,
12271 met_quorum: true,
12272 uncontested: None,
12273 });
12274 runner.state.judgements.push(crate::run::Judgement {
12275 judge: 1,
12276 seat: "judge-1".into(),
12277 agent: "judge".into(),
12278 ranking: vec![winner.label, 'B'],
12279 reasons: BTreeMap::new(),
12280 confidence: None,
12281 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12282 order: vec![0, 1],
12283 failed: None,
12284 duration_ms: 0,
12285 });
12286 } else {
12287 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12288 from_run: "earlier".into(),
12289 generation: 1,
12290 mode: crate::recovery::Mode::VerifyRepair,
12291 reasons: Vec::new(),
12292 reviewed_head: None,
12293 rounds: Vec::new(),
12294 open_findings: Vec::new(),
12295 evidence: None,
12296 criteria_inherited: true,
12297 repair: crate::recovery::Repair::NotStarted,
12298 });
12299 }
12300 let reader = crate::repo_lock::SeatLease::for_checkout(
12301 &winner.worktree,
12302 false,
12303 crate::config::test_coordination().as_ref(),
12304 )
12305 .await
12306 .unwrap();
12307 let pause = runner.pause.clone();
12308 let (result, ()) = tokio::join!(
12309 async {
12310 if graft {
12311 runner.graft().await
12312 } else {
12313 runner.recovery_repair(&winner).await
12314 }
12315 },
12316 async {
12317 tokio::time::sleep(Duration::from_millis(100)).await;
12318 pause.park();
12319 }
12320 );
12321 result.unwrap();
12322 assert!(!marker.exists(), "a blocked seat must not spawn");
12323 runner.state = RunState::load(&runner.state.id).unwrap();
12325 if graft {
12326 assert_eq!(
12327 runner.state.graft.as_ref().unwrap().outcome,
12328 GraftOutcome::WaitingForRestore
12329 );
12330 } else {
12331 assert!(matches!(
12332 runner.state.recovery.as_ref().unwrap().repair,
12333 crate::recovery::Repair::WaitingForRestore { .. }
12334 ));
12335 }
12336 drop(reader);
12337 runner.pause = Pause::new();
12338 runner.state.resource_block = None;
12339 let external =
12341 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12342 assert_eq!(
12343 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12344 0
12345 );
12346 let leftover = winner.worktree.join("leftover");
12347 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12348 let result = if graft {
12349 runner.graft().await
12350 } else {
12351 runner.recovery_repair(&winner).await
12352 };
12353 assert!(
12354 result
12355 .unwrap_err()
12356 .is::<crate::repo_lock::ResourceBlocked>()
12357 );
12358 assert!(leftover.exists());
12359 assert!(!marker.exists());
12360 drop(external);
12361 runner.state.resource_block = None;
12362 let result = if graft {
12363 runner.graft().await
12364 } else {
12365 runner.recovery_repair(&winner).await
12366 };
12367 result.unwrap();
12368 assert!(!leftover.exists(), "restore must precede retry");
12369 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12370 if graft {
12371 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12372 } else {
12373 assert!(matches!(
12374 runner.state.recovery.as_ref().unwrap().repair,
12375 crate::recovery::Repair::Done { .. }
12376 ));
12377 }
12378 }
12379 }
12380
12381 #[tokio::test]
12382 #[cfg(unix)]
12383 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12384 use std::os::fd::AsRawFd as _;
12385 let tmp = tempfile::tempdir().unwrap();
12386 let mut runner = registered_runner(tmp.path()).await;
12387 runner.state.config.graph.timeout_resource_wait = 1;
12388 let winner = runner.state.candidates[0].clone();
12389 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12390 runner.state.tally = Some(Tally {
12391 first_choice: BTreeMap::new(),
12392 borda: BTreeMap::new(),
12393 winner: winner.label,
12394 rankings: 0,
12395 unanimous_initial: false,
12396 deliberated: false,
12397 changed_votes: 0,
12398 unanimous_final: false,
12399 tie_break: None,
12400 judges: 0,
12401 present: 0,
12402 quorum: 0,
12403 met_quorum: true,
12404 uncontested: Some("fixture".into()),
12405 });
12406 runner.state.graft = Some(GraftRecord {
12407 pre: pre.clone(),
12408 post: None,
12409 lines: None,
12410 limit: None,
12411 agent: None,
12412 outcome: GraftOutcome::Running,
12413 requested: Vec::new(),
12414 adoption: None,
12415 adoption_error: None,
12416 });
12417 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12418 git::commit_all(&winner.worktree, "unfinished graft")
12419 .await
12420 .unwrap();
12421 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12422 let leftover = winner.worktree.join("leftover");
12423 std::fs::write(&leftover, "untrusted edits").unwrap();
12424 let path = runner.state.repo.join(".git/updater.lock");
12425 let external = std::fs::File::create(&path).unwrap();
12426 assert_eq!(
12427 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12428 0
12429 );
12430 assert!(
12431 runner
12432 .graft()
12433 .await
12434 .unwrap_err()
12435 .is::<crate::repo_lock::ResourceBlocked>()
12436 );
12437 assert!(matches!(
12438 runner.state.graft.as_ref().unwrap().outcome,
12439 GraftOutcome::Running
12440 ));
12441 assert_eq!(
12442 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12443 unfinished
12444 );
12445 assert!(leftover.exists());
12446 drop(external);
12447 std::fs::remove_file(&path).unwrap();
12448 std::fs::create_dir(&path).unwrap();
12449 assert!(runner.graft().await.is_err());
12450 assert!(matches!(
12451 runner.state.graft.as_ref().unwrap().outcome,
12452 GraftOutcome::Running
12453 ));
12454 assert_eq!(
12455 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12456 unfinished
12457 );
12458 assert!(leftover.exists());
12459 std::fs::remove_dir(&path).unwrap();
12460 runner.graft().await.unwrap();
12461 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12462 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12463 assert!(!leftover.exists());
12464 }
12465
12466 #[tokio::test]
12467 #[cfg(unix)]
12468 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12469 use std::os::fd::AsRawFd as _;
12470 let tmp = tempfile::tempdir().unwrap();
12471 let mut runner = registered_runner(tmp.path()).await;
12472 runner.state.prep_complete = Some(false);
12473 runner.state.config.graph.timeout_resource_wait = 1;
12474 runner.state.save().unwrap();
12475 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12476 assert_eq!(
12477 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12478 0
12479 );
12480 runner.execute().await.unwrap();
12481 let disk = RunState::load(&runner.state.id).unwrap();
12482 assert!(disk.driver_exited);
12483 assert_eq!(disk.driver_pid, None);
12484 assert_eq!(runner.state.status, RunStatus::Blocked);
12485 assert_eq!(
12486 runner.state.resource_block.as_ref().unwrap().kind,
12487 "repository coordination"
12488 );
12489 assert!(runner.state.prep_failure.is_none());
12490 assert!(runner.state.seats.is_empty());
12491 runner.pause.park();
12492 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12495 .await
12496 .unwrap()
12497 .unwrap();
12498 assert!(runner.state.parked);
12499 assert!(runner.state.prep_failure.is_none());
12500 let disk = RunState::load(&runner.state.id).unwrap();
12501 assert!(disk.driver_exited);
12502 assert_eq!(disk.driver_pid, None);
12503 }
12504
12505 #[tokio::test]
12506 #[cfg(unix)]
12507 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12508 use std::os::fd::AsRawFd as _;
12509 let tmp = tempfile::tempdir().unwrap();
12510 let mut runner = registered_runner(tmp.path()).await;
12511 runner.state.prep_complete = Some(false);
12512 runner.state.config.graph.timeout_resource_wait = 30;
12513 runner.state.save().unwrap();
12514 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12515 assert_eq!(
12516 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12517 0
12518 );
12519 assert!(
12520 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12521 .await
12522 .is_err()
12523 );
12524 let disk = RunState::load(&runner.state.id).unwrap();
12525 assert!(disk.driver_exited);
12526 assert_eq!(disk.driver_pid, None);
12527 assert!(disk.status.resumable());
12528 assert!(runner.state.driver_exited);
12529 assert_eq!(runner.state.driver_pid, None);
12530 runner.state.save().unwrap();
12531 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12532 }
12533
12534 #[tokio::test]
12535 #[cfg(unix)]
12536 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12537 let tmp = tempfile::tempdir().unwrap();
12538 let mut runner = registered_runner(tmp.path()).await;
12539 let winner = runner.state.candidates[0].clone();
12540 let cache = tmp.path().join("cache");
12541 let started = tmp.path().join("pre-gate-started");
12542 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12543 runner.state.config.verify.gate =
12544 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12545 let home = crate::run::home();
12546 let owner =
12547 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12548 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12549 crate::cache::AcquireOutcome::Acquired(g) => g,
12550 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12551 };
12552 let mut gate = Box::pin(runner.run_gate(&winner));
12553 assert!(
12554 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12555 .await
12556 .is_err()
12557 );
12558 assert!(!started.exists(), "pre-gate must wait for the cache too");
12559 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12563 let editor = crate::repo_lock::SeatLease::checkout_with(
12564 &winner.worktree,
12565 true,
12566 crate::config::test_coordination().as_ref(),
12567 &wait,
12568 )
12569 .await
12570 .unwrap();
12571 drop(editor);
12572 guard.release();
12573 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12574 .await
12575 .unwrap()
12576 .unwrap();
12577 drop(gate);
12578 assert!(outcomes.iter().all(CommandOutcome::ok));
12579 assert!(started.exists());
12580 }
12581
12582 #[tokio::test]
12583 #[cfg(unix)]
12584 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12585 use std::os::fd::AsRawFd as _;
12586
12587 ask_test_home();
12588 let tmp = tempfile::tempdir().unwrap();
12589 let repo = tmp.path().join("repo");
12590 std::fs::create_dir_all(&repo).unwrap();
12591 init_repo(&repo);
12592 let common = repo.join(".git");
12593 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12594 let mut runner = runner_at(RunStatus::Prep);
12595 runner.state.repo = repo.clone();
12596 runner.state.config.coordination = crate::config::test_coordination();
12597 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12598 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12599 runner.state.config.graph.advise = false;
12600 runner.roles.implementers = vec![spec("implement")];
12601 runner.roles.judges = vec![spec("judge")];
12602 runner.prep().await.unwrap();
12603 let winner = runner.state.candidates[0].clone();
12604 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12607 let completed = tmp.path().join("seat-completed");
12608 runner.roles.implementers[0].command = vec![
12609 "sh".into(),
12610 "-c".into(),
12611 format!("touch '{}'; echo completed", completed.display()),
12612 ];
12613 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12614 assert_eq!(
12615 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12616 0
12617 );
12618 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12619 let mut implement = Box::pin(runner.implement());
12620 tokio::select! {
12621 result = &mut implement => panic!("rescue must wait: {result:?}"),
12622 result = tokio::time::timeout(Duration::from_secs(10), async {
12623 while !completed.exists() {
12624 tokio::time::sleep(Duration::from_millis(20)).await;
12625 }
12626 }) => result.unwrap(),
12627 }
12628 assert!(
12629 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12630 .await
12631 .is_err()
12632 );
12633 assert_eq!(
12634 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12635 before
12636 );
12637 assert_eq!(
12638 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12639 0
12640 );
12641 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12642 .await
12643 .unwrap()
12644 .unwrap();
12645 drop(implement);
12646 assert_ne!(
12647 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12648 before
12649 );
12650 assert!(git::is_clean(&winner.worktree).await.unwrap());
12651
12652 let started = tmp.path().join("pre-gate-started");
12653 let proceed = tmp.path().join("pre-gate-proceed");
12654 let checked = tmp.path().join("gate-ran");
12655 runner.state.config.verify.pre_gate = vec![format!(
12656 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12657 started.display(),
12658 proceed.display()
12659 )];
12660 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12661 assert_eq!(
12662 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12663 0
12664 );
12665 let mut gate = Box::pin(runner.run_gate(&winner));
12666 assert!(
12667 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12668 .await
12669 .is_err()
12670 );
12671 assert!(!started.exists());
12672 assert!(!checked.exists());
12673 assert!(git::is_clean(&winner.worktree).await.unwrap());
12674 assert_eq!(
12675 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12676 0
12677 );
12678 tokio::select! {
12679 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12680 result = tokio::time::timeout(Duration::from_secs(10), async {
12681 while !started.exists() {
12682 tokio::time::sleep(Duration::from_millis(20)).await;
12683 }
12684 }) => result.unwrap(),
12685 }
12686 assert_eq!(
12687 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12688 -1,
12689 "mechanical edits must hold the real lock"
12690 );
12691 std::fs::write(proceed, "ready").unwrap();
12692 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12693 .await
12694 .unwrap()
12695 .unwrap();
12696 drop(gate);
12697 assert!(outcomes.iter().all(CommandOutcome::ok));
12698 assert!(checked.exists());
12699 assert!(runner.state.pre_gate_commit.is_some());
12700 assert!(git::is_clean(&winner.worktree).await.unwrap());
12701 assert_eq!(
12702 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12703 0,
12704 "gate completion must release coordination"
12705 );
12706 drop(outsider);
12707 std::fs::remove_file(common.join("updater.lock")).unwrap();
12708 std::fs::create_dir(common.join("updater.lock")).unwrap();
12709 std::fs::remove_file(&started).unwrap();
12710 std::fs::remove_file(&checked).unwrap();
12711 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12712 let error = runner.run_gate(&winner).await.unwrap_err();
12713 assert!(
12714 error
12715 .to_string()
12716 .contains("acquire mandatory repository lock")
12717 );
12718 assert!(
12719 !started.exists(),
12720 "failed acquisition must prevent pre-gate"
12721 );
12722 assert!(!checked.exists(), "failed acquisition must prevent gate");
12723 assert!(git::is_clean(&winner.worktree).await.unwrap());
12724 assert_eq!(
12725 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12726 head
12727 );
12728 }
12729
12730 #[tokio::test]
12731 #[cfg(unix)]
12732 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
12733 use std::os::fd::AsRawFd as _;
12734 use std::time::Duration;
12735
12736 ask_test_home();
12737 let tmp = tempfile::tempdir().unwrap();
12738 let repo = tmp.path().join("repo");
12739 std::fs::create_dir_all(&repo).unwrap();
12740 init_repo(&repo);
12741 let common = repo.join(".git");
12742 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12743 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
12744 assert_eq!(
12745 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12746 0
12747 );
12748 let mut runner = runner_at(RunStatus::Prep);
12749 runner.state.repo = repo.clone();
12750 runner.state.config.coordination = crate::config::test_coordination();
12751 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12752 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12753 runner.state.config.graph.advise = true;
12754 runner.state.config.graph.advisors = 1;
12755 runner.roles.implementers = vec![spec("implement")];
12756 runner.roles.judges = vec![spec("judge")];
12757 let root = runner.state.worktree_root();
12758 let started = tmp.path().join("setup-started");
12759 let proceed = tmp.path().join("setup-proceed");
12760 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
12761 run: Some(format!(
12762 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12763 started.display(),
12764 proceed.display()
12765 )),
12766 ..Default::default()
12767 }];
12768 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
12769 let config = git::git(&repo, &["config", "--local", "--list"])
12770 .await
12771 .unwrap();
12772 let mut prep = Box::pin(runner.prep());
12773 assert!(
12774 tokio::time::timeout(Duration::from_millis(200), &mut prep)
12775 .await
12776 .is_err()
12777 );
12778 assert!(
12779 !root.exists(),
12780 "contention must prevent even checkout creation"
12781 );
12782 assert!(!started.exists(), "contention must prevent setup");
12783 assert_eq!(
12784 git::git(&repo, &["branch", "--list"]).await.unwrap(),
12785 branches
12786 );
12787 assert_eq!(
12788 git::git(&repo, &["config", "--local", "--list"])
12789 .await
12790 .unwrap(),
12791 config
12792 );
12793 drop(lock);
12794 tokio::select! {
12795 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
12796 result = tokio::time::timeout(Duration::from_secs(10), async {
12797 while !started.exists() {
12798 tokio::time::sleep(Duration::from_millis(20)).await;
12799 }
12800 }) => result.unwrap(),
12801 }
12802 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
12803 assert_eq!(
12804 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12805 -1
12806 );
12807 assert!(
12808 tokio::time::timeout(
12809 Duration::from_millis(100),
12810 crate::repo_lock::SeatLease::for_checkout(
12811 &repo,
12812 true,
12813 crate::config::test_coordination().as_ref()
12814 )
12815 )
12816 .await
12817 .is_err(),
12818 "preparation must also exclude local editors"
12819 );
12820 std::fs::write(proceed, "ready").unwrap();
12821 tokio::time::timeout(Duration::from_secs(10), &mut prep)
12822 .await
12823 .unwrap()
12824 .unwrap();
12825 drop(prep);
12826 assert_eq!(runner.state.prep_complete, Some(true));
12827 assert!(root.join("judge-1").exists());
12828 assert!(root.join("advisor-1").exists());
12829 assert_eq!(
12830 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12831 0
12832 );
12833 }
12834
12835 #[tokio::test]
12836 #[cfg(unix)]
12837 async fn preparation_lock_failure_prevents_repository_changes() {
12838 ask_test_home();
12839 let tmp = tempfile::tempdir().unwrap();
12840 let repo = tmp.path().join("repo");
12841 std::fs::create_dir_all(&repo).unwrap();
12842 init_repo(&repo);
12843 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12844 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
12847 let mut runner = runner_at(RunStatus::Prep);
12848 runner.state.repo = repo.clone();
12849 runner.state.config.coordination = crate::config::test_coordination();
12850 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12851 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12852 runner.roles.implementers = vec![spec("implement")];
12853 let error = runner.prep().await.unwrap_err();
12854 assert!(
12855 error
12856 .to_string()
12857 .contains("acquire mandatory repository lock")
12858 );
12859 assert!(!runner.state.worktree_root().exists());
12860 assert!(runner.state.candidates.is_empty());
12861 assert!(!repo.join(".git/config.worktree").exists());
12862 assert_eq!(
12863 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
12864 "* main"
12865 );
12866 }
12867
12868 #[tokio::test]
12869 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
12870 ask_test_home();
12871 let tmp = tempfile::tempdir().unwrap();
12872 let repo = tmp.path().join("repo");
12873 std::fs::create_dir_all(&repo).unwrap();
12874 init_repo(&repo);
12875 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12876 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
12877 std::fs::write(repo.join("referenced"), "required commit").unwrap();
12878 git::git(&repo, &["add", "."]).await.unwrap();
12879 git::git(&repo, &["commit", "-m", "required seed"])
12880 .await
12881 .unwrap();
12882 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
12883 git::git(&repo, &["checkout", "main"]).await.unwrap();
12884 let mut runner = runner_at(RunStatus::Prep);
12885 runner.state.repo = repo.clone();
12886 runner.state.base_commit = base.clone();
12887 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12888 runner.state.config.graph.advise = false;
12889 runner.state.config.blind.commit_msg_hook = false;
12890 runner.state.seeds = vec![refs::Seed {
12891 token: seed.clone(),
12892 sha: seed.clone(),
12893 kind: refs::SeedKind::Unmerged,
12894 branch: false,
12895 detail: String::new(),
12896 }];
12897 runner.roles.implementers = vec![spec("one"), spec("two")];
12898 runner.roles.judges = vec![spec("judge")];
12899 let label = blind::assign_labels(2, runner.state.seed)[0];
12900 let branch = runner.state.branch_for(label);
12901 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
12902 git::worktree_add_branch(&repo, &worktree, &branch, &base)
12903 .await
12904 .unwrap();
12905 runner.state.candidates.push(Candidate {
12906 index: 0,
12907 label,
12908 agent: "one".into(),
12909 branch,
12910 worktree,
12911 summary: String::new(),
12912 stat: String::new(),
12913 files: 0,
12914 commits: 0,
12915 empty: false,
12916 failed: None,
12917 verified_noop: None,
12918 duration_ms: 0,
12919 folded: false,
12920 });
12921 runner.state.save().unwrap();
12922 runner.state = RunState::load(&runner.state.id).unwrap();
12924 assert_eq!(runner.state.prep_complete, Some(false));
12925 runner.prep().await.unwrap();
12926 assert_eq!(runner.state.prep_complete, Some(true));
12927 assert_eq!(runner.state.candidates.len(), 2);
12928 for candidate in &runner.state.candidates {
12929 assert_eq!(
12930 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
12931 git::tree_of(&repo, &seed).await.unwrap()
12932 );
12933 assert_eq!(
12934 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
12935 "required commit"
12936 );
12937 }
12938 assert!(runner.state.worktree_root().join("judge-1").exists());
12939 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
12940 runner.prep().await.unwrap();
12941 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
12942
12943 runner.state.prep_complete = Some(false);
12944 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
12945 run: Some(format!(
12946 "test -f '{}'",
12947 tmp.path().join("dependency-ready").display()
12948 )),
12949 ..Default::default()
12950 }];
12951 assert!(runner.prep().await.is_err());
12952 assert!(runner.state.prep_failure.is_some());
12953 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
12954 runner.state = RunState::load(&runner.state.id).unwrap();
12955 assert!(
12956 runner.prep().await.is_err(),
12957 "correcting the cause alone must not auto-retry"
12958 );
12959 let failed_at = runner.state.events.last().unwrap().at;
12960 assert!(
12961 runner.retry_preparation(failed_at).is_err(),
12962 "old approvals cannot unlock the hold"
12963 );
12964 let mut task = crate::queue::Task::new(
12965 "retry".into(),
12966 "retry setup".into(),
12967 repo.clone(),
12968 crate::queue::Source::Human,
12969 );
12970 task.start(runner.state.id.clone());
12971 task.hold_machine(Some("setup failed".into()));
12972 task.release_operator("cli", None);
12973 runner
12974 .retry_preparation(task.resumption.as_ref().unwrap().at)
12975 .unwrap();
12976 runner.prep().await.unwrap();
12977 assert!(runner.state.prep_failure.is_none());
12978 assert_eq!(runner.state.prep_complete, Some(true));
12979 assert_eq!(runner.state.candidates.len(), 2);
12980 assert!(runner.state.worktree_root().join("judge-1").exists());
12981 for candidate in &runner.state.candidates {
12982 assert_eq!(
12983 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
12984 git::tree_of(&repo, &seed).await.unwrap()
12985 );
12986 }
12987 }
12988
12989 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
12990 let mut heads = Vec::new();
12991 for candidate in candidates {
12992 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
12993 }
12994 heads
12995 }
12996
12997 fn init_repo(dir: &Path) {
12998 let run = |args: &[&str]| {
12999 let out = std::process::Command::new("git")
13000 .args(args)
13001 .current_dir(dir)
13002 .quiet()
13003 .output()
13004 .expect("spawn git");
13005 assert!(
13006 out.status.success(),
13007 "git {args:?} failed: {}",
13008 String::from_utf8_lossy(&out.stderr)
13009 );
13010 };
13011 run(&["init", "-b", "main"]);
13012 run(&["config", "user.name", "magi test"]);
13013 run(&["config", "user.email", "magi@example.com"]);
13014 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13015 run(&["add", "-A"]);
13016 run(&["commit", "-m", "init"]);
13017 }
13018
13019 fn ask_test_home() {
13027 crate::run::pin_test_home();
13028 }
13029
13030 fn runner_at(status: RunStatus) -> Runner {
13033 let mut state = RunState::new(
13034 PathBuf::from("/nonexistent/repo"),
13035 "main".to_owned(),
13036 "deadbeef".to_owned(),
13037 "task".to_owned(),
13038 Config::default(),
13039 );
13040 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13043 state.id = format!(
13044 "20000101-000000-{:04x}",
13045 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13046 );
13047 state.status = status;
13048 Runner {
13049 state,
13050 roles: ResolvedRoles {
13051 implementers: Vec::new(),
13052 judges: Vec::new(),
13053 reviewers: Vec::new(),
13054 fixer: None,
13055 conductor: conductor(),
13056 implementer_roster: Vec::new(),
13057 judge_roster: Vec::new(),
13058 reviewer_roster: Vec::new(),
13059 },
13060 sem: Arc::new(Semaphore::new(1)),
13061 pause: Pause::new(),
13062 interrupt: Pause::new(),
13063 }
13064 }
13065
13066 #[test]
13067 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13068 crate::run::pin_test_home();
13069 let mut runner = runner_at(RunStatus::Gating);
13070 runner.state.save().unwrap();
13071 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13074 assert!(runner.claim_driver().is_err());
13075 assert!(runner.state.driver_exited);
13076 assert_eq!(runner.state.driver_pid, None);
13077 let disk = RunState::load(&runner.state.id).unwrap();
13078 assert!(disk.driver_exited);
13079 assert_eq!(disk.driver_pid, None);
13080 assert_eq!(disk.status, RunStatus::Gating);
13081 }
13082
13083 #[tokio::test]
13084 async fn an_early_graph_error_releases_its_driver() {
13085 crate::run::pin_test_home();
13086 let mut runner = runner_at(RunStatus::Blocked);
13087 runner.state.config.graph.selected_implementer = Some("selected".into());
13088 runner.state.save().unwrap();
13089 assert!(runner.execute().await.is_err());
13090 let disk = RunState::load(&runner.state.id).unwrap();
13091 assert!(disk.driver_exited);
13092 assert_eq!(disk.driver_pid, None);
13093 assert_eq!(disk.status, RunStatus::Blocked);
13094 }
13095
13096 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13097 crate::run::MergeRepair {
13098 question: "repair-question".into(),
13099 head: "old-head".into(),
13100 pr: "https://github.com/o/r/pull/42".into(),
13101 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13102 gate: vec![outcome(Some(0), false)],
13103 gate_head: Some("old-head".into()),
13104 gate_fixes: Vec::new(),
13105 pre_gate: Vec::new(),
13106 phase,
13107 failure: None,
13108 }
13109 }
13110
13111 #[tokio::test]
13112 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13113 crate::run::pin_test_home();
13114 let mut runner = runner_at(RunStatus::Landing);
13115 runner.state.merge_repairs.push(merge_repair_request(
13116 crate::run::MergeRepairPhase::Requested,
13117 ));
13118 runner.state.save().unwrap();
13119 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13120 assert!(
13121 runner
13122 .execute()
13123 .await
13124 .unwrap_err()
13125 .to_string()
13126 .contains("already running")
13127 );
13128 assert_eq!(
13129 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13130 crate::run::MergeRepairPhase::Requested
13131 );
13132 drop(claim);
13133 let mut disk = runner.state.clone();
13134 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13135 disk.save().unwrap();
13136 assert!(
13137 runner
13138 .execute()
13139 .await
13140 .unwrap_err()
13141 .to_string()
13142 .contains("decision changed")
13143 );
13144 assert_eq!(
13145 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13146 crate::run::MergeRepairPhase::Complete
13147 );
13148 }
13149
13150 #[test]
13151 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13152 crate::run::pin_test_home();
13153 let mut runner = runner_at(RunStatus::Landing);
13154 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13155 runner.state.reviews = request.reviews.clone();
13156 runner.state.gate = request.gate.clone();
13157 runner.state.gate_ran = true;
13158 runner.state.land_approval = Some(crate::run::LandApproval {
13159 question: "old".into(),
13160 head: "old-head".into(),
13161 });
13162 runner.state.merge_repairs.push(request);
13163 runner.begin_merge_repair_review(0).unwrap();
13164 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13165 assert!(!runner.state.gate_status().ok());
13166 assert!(runner.state.land_approval.is_none());
13167 let disk = RunState::load(&runner.state.id).unwrap();
13168 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13169 assert!(disk.merge_repairs[0].gate[0].ok());
13170 assert_eq!(
13171 disk.merge_repairs[0].phase,
13172 crate::run::MergeRepairPhase::Reviewing
13173 );
13174 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13175 assert!(disk.reviews.is_empty());
13176 }
13177
13178 #[tokio::test]
13179 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13180 crate::run::pin_test_home();
13181 for phase in [
13182 crate::run::MergeRepairPhase::Fixing,
13183 crate::run::MergeRepairPhase::Requested,
13184 ] {
13185 let mut runner = runner_at(RunStatus::Landing);
13186 runner.state.config.graph.land_rounds = 0;
13187 runner.state.merge_repairs.push(merge_repair_request(phase));
13188 runner.state.save().unwrap();
13189 assert!(!runner.repair_merge_review().await.unwrap());
13190 assert_eq!(runner.state.status, RunStatus::Blocked);
13191 assert_eq!(
13192 runner.state.merge_repairs[0].phase,
13193 crate::run::MergeRepairPhase::Failed
13194 );
13195 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
13196 assert!(
13197 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
13198 "interrupted"
13199 } else {
13200 "budget exhausted"
13201 })
13202 );
13203 assert!(runner.state.merge.is_none());
13204 }
13205 }
13206
13207 #[test]
13211 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
13212 crate::run::pin_test_home();
13213 let mut runner = runner_at(RunStatus::Implementing);
13214 let interrupt = Pause::new();
13215 runner.watch_interrupt(interrupt.clone());
13216
13217 interrupt.park_because("task a1b2 asked to run first");
13218
13219 assert!(runner.park_here().expect("park_here"));
13220 assert!(runner.state.parked);
13221 let last = runner.state.events.last().expect("a park event");
13222 assert_eq!(last.node, "park");
13223 assert!(
13224 last.message.contains("task a1b2 asked to run first"),
13225 "expected the interrupt reason in {:?}",
13226 last.message
13227 );
13228 }
13229
13230 #[test]
13233 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
13234 crate::run::pin_test_home();
13235 let mut runner = runner_at(RunStatus::Implementing);
13236 runner.state.begin_pass();
13237 let shutdown = Pause::new();
13238 runner.on_pause(shutdown.clone());
13239 let upgrade = ParkCause::Upgrade {
13240 from: "0.121.0".to_owned(),
13241 to: Some("0.121.1".to_owned()),
13242 };
13243 shutdown.park_for(upgrade.clone());
13244 shutdown.park_for(ParkCause::Operator);
13245
13246 assert!(runner.park_here().expect("park_here"));
13247 assert_eq!(
13248 runner.state.passes[0].end,
13249 Some(crate::run::PassEnd::Parked { cause: upgrade })
13250 );
13251 let last = runner.state.events.last().expect("a park event");
13252 assert!(last.message.contains("0.121.0"), "{}", last.message);
13253 }
13254
13255 #[test]
13263 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
13264 crate::run::pin_test_home();
13265 let mut runner = runner_at(RunStatus::Implementing);
13266 let shutdown = Pause::new();
13267 runner.on_pause(shutdown.clone());
13268 let interrupt = Pause::new();
13269 runner.watch_interrupt(interrupt.clone());
13270
13271 assert!(!runner.park_here().expect("park_here"));
13273 assert!(!runner.state.parked);
13274
13275 interrupt.park_because("test");
13277 assert!(!shutdown.parked());
13278 assert!(runner.park_here().expect("park_here"));
13279 }
13280
13281 #[tokio::test]
13295 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
13296 crate::run::pin_test_home();
13297 let mut runner = runner_at(RunStatus::Implementing);
13298 let interrupt = Pause::new();
13299 runner.watch_interrupt(interrupt.clone());
13300
13301 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
13302 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
13303
13304 let node = async move {
13308 started_tx.send(()).expect("send started");
13309 finish_rx.await.expect("recv finish");
13310 "node finished"
13311 };
13312
13313 let interrupter = async move {
13314 started_rx.await.expect("recv started");
13315 interrupt.park_because("higher-priority task waiting");
13317 tokio::task::yield_now().await;
13321 finish_tx.send(()).expect("send finish");
13322 };
13323
13324 let (node_result, ()) = tokio::join!(node, interrupter);
13325 assert_eq!(
13326 node_result, "node finished",
13327 "the in-flight call ran to completion"
13328 );
13329
13330 assert!(runner.park_here().expect("park_here"));
13333 assert!(runner.state.parked);
13334 }
13335
13336 #[test]
13342 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
13343 crate::run::pin_test_home();
13344 let mut runner = runner_at(RunStatus::Judging);
13345 runner.state.config.agents = vec![conductor()];
13349 runner.state.candidates = vec![Candidate {
13350 index: 0,
13351 label: 'A',
13352 agent: "alpha".to_owned(),
13353 branch: "magi/x/A".to_owned(),
13354 worktree: PathBuf::from("/nonexistent/worktree"),
13355 summary: "did the thing".to_owned(),
13356 stat: "1 file changed".to_owned(),
13357 files: 1,
13358 commits: 1,
13359 empty: false,
13360 failed: None,
13361 verified_noop: None,
13362 duration_ms: 1234,
13363 folded: false,
13364 }];
13365 let run_id = runner.state.id.clone();
13366
13367 let interrupt = Pause::new();
13368 runner.watch_interrupt(interrupt.clone());
13369 interrupt.park_because("task c3d4 asked to run first");
13370 assert!(runner.park_here().expect("park_here"));
13371
13372 let resumed = Runner::resume(&run_id).expect("resume");
13373 assert_eq!(resumed.state.candidates.len(), 1);
13374 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
13375 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
13376 assert_eq!(resumed.state.status, runner.state.status);
13377 assert!(
13378 resumed.state.parked,
13379 "still parked until `execute` actually walks the graph again"
13380 );
13381 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
13382 }
13383
13384 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
13386 let mut q = ask::Question::new(
13387 run.to_owned(),
13388 "implement".to_owned(),
13389 "impl-A".to_owned(),
13390 "Which storage backend should the cache use?".to_owned(),
13391 String::new(),
13392 vec!["SQLite".to_owned(), "Redis".to_owned()],
13393 );
13394 store.put(&mut q).unwrap();
13395 q
13396 }
13397
13398 #[test]
13399 fn a_failed_runs_open_question_is_abandoned() {
13400 ask_test_home();
13401 let store = ask::Questions::open();
13402 let mut runner = runner_at(RunStatus::Failed);
13403 let run = runner.state.id.clone();
13404 let q = ask_open_question(&store, &run);
13405
13406 runner.settle_questions();
13407
13408 let back = store.get(&q.id).unwrap();
13409 assert!(
13410 !back.status.open(),
13411 "the seat that asked died with the run; nobody is left to read an answer"
13412 );
13413 assert!(
13414 back.detail.contains(&run) && back.detail.contains("failed"),
13415 "the reason names what the run became, not just that it is gone: {}",
13416 back.detail
13417 );
13418 }
13419
13420 #[test]
13421 fn a_merged_runs_open_question_is_abandoned_too() {
13422 ask_test_home();
13423 let store = ask::Questions::open();
13424 for status in [RunStatus::Merged, RunStatus::Ready] {
13427 let mut runner = runner_at(status);
13428 let run = runner.state.id.clone();
13429 let q = ask_open_question(&store, &run);
13430
13431 runner.settle_questions();
13432
13433 let back = store.get(&q.id).unwrap();
13434 assert!(
13435 !back.status.open(),
13436 "{status:?} run's question must not outlive the run"
13437 );
13438 }
13439 }
13440
13441 #[test]
13442 fn a_still_resumable_runs_open_question_is_left_alone() {
13443 ask_test_home();
13444 let store = ask::Questions::open();
13445 for status in [RunStatus::Blocked, RunStatus::Stalled] {
13451 let mut runner = runner_at(status);
13452 let run = runner.state.id.clone();
13453 let q = ask_open_question(&store, &run);
13454
13455 runner.settle_questions();
13456
13457 let back = store.get(&q.id).unwrap();
13458 assert!(
13459 back.status.open(),
13460 "{status:?} is still alive; the question must still be waiting"
13461 );
13462 }
13463 }
13464
13465 #[test]
13466 fn settle_questions_never_touches_an_already_answered_question() {
13467 ask_test_home();
13468 let store = ask::Questions::open();
13469 let mut runner = runner_at(RunStatus::Failed);
13470 let run = runner.state.id.clone();
13471 let mut q = ask_open_question(&store, &run);
13472 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
13473 .unwrap();
13474 store.put(&mut q).unwrap();
13475
13476 runner.settle_questions();
13481 runner.settle_questions();
13482
13483 let back = store.get(&q.id).unwrap();
13484 assert_eq!(
13485 back.status,
13486 ask::QuestionStatus::Answered,
13487 "a real answer is a decision on record, never overwritten by a sweep"
13488 );
13489 }
13490
13491 #[tokio::test]
13502 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
13503 crate::run::pin_test_home();
13504 let tmp = tempfile::tempdir().expect("tempdir");
13505 let repo = tmp.path().join("repo");
13506 std::fs::create_dir_all(&repo).unwrap();
13507 init_repo(&repo);
13508
13509 let mut config = Config::default();
13510 config.graph.worktree_root = Some(tmp.path().join("wt"));
13511
13512 let mut state = RunState::new(
13513 repo.clone(),
13514 "main".to_owned(),
13515 "deadbeef".to_owned(),
13516 "task".to_owned(),
13517 config,
13518 );
13519 let root = state.worktree_root();
13520 let wt_a = root.join("cand-A");
13521 let wt_b = root.join("cand-B");
13522 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
13523 .await
13524 .expect("worktree A");
13525 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
13526 .await
13527 .expect("worktree B");
13528
13529 state.candidates = vec![
13530 Candidate {
13531 index: 0,
13532 label: 'A',
13533 agent: "alpha".to_owned(),
13534 branch: "magi/x/A".to_owned(),
13535 worktree: wt_a.clone(),
13536 summary: String::new(),
13537 stat: String::new(),
13538 files: 0,
13539 commits: 0,
13540 empty: false,
13541 failed: None,
13542 verified_noop: None,
13543 duration_ms: 0,
13544 folded: false,
13545 },
13546 Candidate {
13547 index: 1,
13548 label: 'B',
13549 agent: "beta".to_owned(),
13550 branch: "magi/x/B".to_owned(),
13551 worktree: wt_b.clone(),
13552 summary: String::new(),
13553 stat: String::new(),
13554 files: 0,
13555 commits: 0,
13556 empty: false,
13557 failed: None,
13558 verified_noop: None,
13559 duration_ms: 0,
13560 folded: false,
13561 },
13562 ];
13563 state.tally = Some(Tally {
13564 first_choice: BTreeMap::from([('A', 1)]),
13565 borda: BTreeMap::new(),
13566 winner: 'A',
13567 rankings: 1,
13568 unanimous_initial: true,
13569 deliberated: false,
13570 changed_votes: 0,
13571 unanimous_final: true,
13572 tie_break: None,
13573 judges: 1,
13574 present: 1,
13575 quorum: 1,
13576 met_quorum: true,
13577 uncontested: None,
13578 });
13579 state.status = RunStatus::Ready;
13580
13581 fold_run(&mut state, false, &crate::run::home())
13582 .await
13583 .expect("fold_run");
13584
13585 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
13586 assert!(
13587 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
13588 "the unmerged winner's branch survives"
13589 );
13590 assert!(
13591 !state.candidates[0].folded,
13592 "the winner is not marked folded"
13593 );
13594
13595 assert!(!wt_b.exists(), "the loser's worktree is removed");
13596 assert!(
13597 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
13598 "the loser's branch is removed"
13599 );
13600 assert!(state.candidates[1].folded, "the loser is marked folded");
13601 }
13602
13603 #[tokio::test]
13604 #[cfg(unix)]
13605 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
13606 use std::os::fd::AsRawFd as _;
13607 use std::time::Duration;
13608
13609 let tmp = tempfile::tempdir().unwrap();
13610 let repo = tmp.path().join("repo");
13611 std::fs::create_dir_all(&repo).unwrap();
13612 init_repo(&repo);
13613 let common = repo.join(".git");
13614 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13615 let home = tmp.path().join("home");
13616 let mut config = Config::default();
13617 config.coordination = crate::config::test_coordination();
13618 config.graph.worktree_root = Some(tmp.path().join("wt"));
13619 config.graph.timeout_resource_wait = 1;
13620 let mut state = RunState::new(
13621 repo.clone(),
13622 "main".into(),
13623 "deadbeef".into(),
13624 "task".into(),
13625 config,
13626 );
13627 let root = state.worktree_root();
13628 let candidate = root.join("cand-A");
13629 let reviewer = root.join("review-1");
13630 let branch = state.branch_for('A');
13631 git::worktree_add_branch(&repo, &candidate, &branch, "main")
13632 .await
13633 .unwrap();
13634 git::worktree_add_detached(&repo, &reviewer, "main")
13635 .await
13636 .unwrap();
13637 git::acquire_worktree_config(&repo).await.unwrap();
13638 state.enabled_worktree_config = true;
13639 state.status = RunStatus::Ready;
13640 state.candidates.push(Candidate {
13641 index: 0,
13642 label: 'A',
13643 agent: "implement".into(),
13644 branch: branch.clone(),
13645 worktree: candidate.clone(),
13646 summary: String::new(),
13647 stat: String::new(),
13648 files: 0,
13649 commits: 0,
13650 empty: false,
13651 failed: None,
13652 verified_noop: None,
13653 duration_ms: 0,
13654 folded: false,
13655 });
13656 state.save_under(&home).unwrap();
13657 let config_before = std::fs::read(common.join("config")).unwrap();
13658 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
13659 .await
13660 .unwrap();
13661 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
13664 assert_eq!(
13665 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13666 0
13667 );
13668 let mut fold = Box::pin(fold_run(&mut state, true, &home));
13669 assert!(
13670 tokio::time::timeout(Duration::from_millis(200), &mut fold)
13671 .await
13672 .is_err()
13673 );
13674 assert!(candidate.exists());
13675 assert!(reviewer.exists());
13676 assert!(git::branch_exists(&repo, &branch).await.unwrap());
13677 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
13678 assert_eq!(
13679 git::git(&repo, &["worktree", "list", "--porcelain"])
13680 .await
13681 .unwrap(),
13682 worktrees_before
13683 );
13684 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
13685 .await
13686 .unwrap()
13687 .unwrap_err();
13688 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
13689 drop(fold);
13690 assert!(candidate.exists() && reviewer.exists());
13691 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
13692 assert_eq!(
13693 state.resource_block.as_ref().unwrap().kind,
13694 "repository coordination"
13695 );
13696 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
13697 drop(external);
13698 let mut fold = Box::pin(fold_run(&mut state, true, &home));
13699 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
13700 .await
13701 .unwrap()
13702 .unwrap();
13703 drop(fold);
13704 assert!(removed.contains(&branch));
13705 assert!(!root.exists());
13706 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
13707 assert!(state.candidates[0].folded);
13708 assert!(!state.enabled_worktree_config);
13709 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
13710 let enabled = git::git_raw(
13711 &repo,
13712 &["config", "--local", "--get", "extensions.worktreeConfig"],
13713 )
13714 .await
13715 .unwrap();
13716 assert!(!enabled.ok(), "the last config reference was released");
13717 }
13718
13719 #[tokio::test]
13722 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
13723 let tmp = tempfile::tempdir().expect("tempdir");
13724 let repo = tmp.path().join("repo");
13725 std::fs::create_dir_all(&repo).unwrap();
13726 init_repo(&repo);
13727 let home = tmp.path().join("home");
13728
13729 let mut config = Config::default();
13730 config.graph.worktree_root = Some(tmp.path().join("wt"));
13731 let mut state = RunState::new(
13732 repo.clone(),
13733 "main".to_owned(),
13734 "deadbeef".to_owned(),
13735 "task".to_owned(),
13736 config,
13737 );
13738 git::git(&repo, &["branch", "magi/x/A", "main"])
13740 .await
13741 .expect("branch");
13742 state.candidates = vec![Candidate {
13743 index: 0,
13744 label: 'A',
13745 agent: "alpha".to_owned(),
13746 branch: "magi/x/A".to_owned(),
13747 worktree: state.worktree_root().join("cand-A"),
13748 summary: String::new(),
13749 stat: String::new(),
13750 files: 0,
13751 commits: 0,
13752 empty: false,
13753 failed: None,
13754 verified_noop: None,
13755 duration_ms: 0,
13756 folded: true,
13757 }];
13758 state.released_to = Some("20260901-000000-new1".to_owned());
13759 state.released_branches = vec!["magi/x/A".to_owned()];
13760
13761 fold_run(&mut state, true, &home).await.expect("fold_run");
13762
13763 assert!(
13764 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
13765 "the handed-over branch survives a fold"
13766 );
13767 }
13768
13769 #[tokio::test]
13773 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
13774 let tmp = tempfile::tempdir().expect("tempdir");
13775 let repo = tmp.path().join("repo");
13776 std::fs::create_dir_all(&repo).unwrap();
13777 init_repo(&repo);
13778 let run = |args: &[&str]| {
13779 let out = std::process::Command::new("git")
13780 .quiet()
13781 .args(args)
13782 .current_dir(&repo)
13783 .output()
13784 .expect("spawn git");
13785 assert!(out.status.success(), "git {args:?}");
13786 };
13787 run(&["branch", "magi/x/A"]);
13788 run(&["checkout", "-q", "-b", "magi/x/B"]);
13789 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
13790 run(&["add", "-A"]);
13791 run(&["commit", "-q", "-m", "work"]);
13792 run(&["checkout", "-q", "main"]);
13793
13794 let probe = RunState::new(
13798 repo.clone(),
13799 "main".to_owned(),
13800 "deadbeef".to_owned(),
13801 "task".to_owned(),
13802 Config::default(),
13803 );
13804 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
13805 let bare = tmp.path().join("origin.git");
13806 let out = std::process::Command::new("git")
13807 .quiet()
13808 .args(["init", "-q", "--bare"])
13809 .arg(&bare)
13810 .output()
13811 .expect("spawn git");
13812 assert!(out.status.success(), "git init --bare");
13813 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
13814 run(&["push", "-q", "origin", "main"]);
13815
13816 let mut state = RunState::new(
13817 repo.clone(),
13818 "main".to_owned(),
13819 "deadbeef".to_owned(),
13820 "task".to_owned(),
13821 Config::default(),
13822 );
13823 state.seeds = vec![refs::Seed {
13824 token: "magi/27b2/A".to_owned(),
13825 kind: refs::SeedKind::Unresolved,
13826 sha: String::new(),
13827 branch: true,
13828 detail: "no branch or commit named magi/27b2/A".to_owned(),
13829 }];
13830
13831 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
13832 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
13833 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
13834 let detail = empty_candidate_detail(&state, "main");
13835 assert!(detail.starts_with("empty candidate"), "{detail}");
13836 assert!(detail.contains("magi/27b2/A"), "{detail}");
13837 }
13838
13839 #[tokio::test]
13848 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
13849 let tmp = tempfile::tempdir().expect("tempdir");
13850 let repo = tmp.path().join("repo");
13851 std::fs::create_dir_all(&repo).unwrap();
13852 init_repo(&repo);
13853
13854 let mut config = Config::default();
13855 config.merge.mode = MergeMode::Local;
13856
13857 let mut state = RunState::new(
13858 repo.clone(),
13859 "main".to_owned(),
13860 "deadbeef".to_owned(),
13861 "task".to_owned(),
13862 config,
13863 );
13864 state.candidates = vec![Candidate {
13865 index: 0,
13866 label: 'A',
13867 agent: "alpha".to_owned(),
13868 branch: "does-not-exist".to_owned(),
13869 worktree: repo.clone(),
13870 summary: String::new(),
13871 stat: String::new(),
13872 files: 0,
13873 commits: 0,
13874 empty: false,
13875 failed: None,
13876 verified_noop: None,
13877 duration_ms: 0,
13878 folded: false,
13879 }];
13880 state.tally = Some(Tally {
13881 first_choice: BTreeMap::from([('A', 1)]),
13882 borda: BTreeMap::new(),
13883 winner: 'A',
13884 rankings: 1,
13885 unanimous_initial: true,
13886 deliberated: false,
13887 changed_votes: 0,
13888 unanimous_final: true,
13889 tie_break: None,
13890 judges: 0,
13891 present: 0,
13892 quorum: 0,
13893 met_quorum: true,
13894 uncontested: Some("only candidate A produced a change".to_owned()),
13895 });
13896 state.reviews = vec![ReviewRound {
13897 round: 1,
13898 head: "deadbeef".to_owned(),
13899 verified_head: None,
13900 verified_at: None,
13901 reviews: Vec::new(),
13902 e2e: Vec::new(),
13903 fix: None,
13904 blocking: 0,
13905 answered: 0,
13906 expected: 0,
13907 clean: true,
13908 verify_retried: false,
13909 e2e_deferred: false,
13910 e2e_defer_reason: None,
13911 progressed: false,
13912 vote_split: false,
13913 reconsideration: Vec::new(),
13914 verdict: None,
13915 }];
13916 state.gate = vec![CommandOutcome {
13917 command: "test".to_owned(),
13918 code: Some(0),
13919 output_tail: String::new(),
13920 duration_ms: 0,
13921 resource_blocked: false,
13922 }];
13923 state.gate_ran = true;
13924 state.status = RunStatus::Ready;
13929 state.merge = Some(MergeOutcome {
13930 mode: MergeMode::Local,
13931 ok: false,
13932 detail: "already concluded".to_owned(),
13933 empty: false,
13934 });
13935
13936 let mut runner = Runner {
13937 state,
13938 roles: ResolvedRoles {
13939 implementers: Vec::new(),
13940 judges: Vec::new(),
13941 reviewers: Vec::new(),
13942 fixer: None,
13943 conductor: conductor(),
13944 implementer_roster: Vec::new(),
13945 judge_roster: Vec::new(),
13946 reviewer_roster: Vec::new(),
13947 },
13948 sem: Arc::new(Semaphore::new(1)),
13949 pause: Pause::new(),
13950 interrupt: Pause::new(),
13951 };
13952
13953 runner.merge().await.expect("merge");
13954
13955 assert_eq!(
13956 runner.state.status,
13957 RunStatus::Ready,
13958 "a concluded run's status must not change on reentry"
13959 );
13960 assert_eq!(
13961 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
13962 Some("already concluded"),
13963 "merge must not run again once the node already recorded an outcome"
13964 );
13965 }
13966
13967 #[tokio::test]
13976 async fn merge_refuses_a_gate_that_has_not_actually_run() {
13977 let tmp = tempfile::tempdir().expect("tempdir");
13978 let repo = tmp.path().join("repo");
13979 std::fs::create_dir_all(&repo).unwrap();
13980 init_repo(&repo);
13981
13982 let mut config = Config::default();
13983 config.merge.mode = MergeMode::Local;
13984
13985 let mut state = RunState::new(
13986 repo.clone(),
13987 "main".to_owned(),
13988 "deadbeef".to_owned(),
13989 "task".to_owned(),
13990 config,
13991 );
13992 state.candidates = vec![Candidate {
13993 index: 0,
13994 label: 'A',
13995 agent: "alpha".to_owned(),
13996 branch: "does-not-exist".to_owned(),
13997 worktree: repo.clone(),
13998 summary: String::new(),
13999 stat: String::new(),
14000 files: 0,
14001 commits: 0,
14002 empty: false,
14003 failed: None,
14004 verified_noop: None,
14005 duration_ms: 0,
14006 folded: false,
14007 }];
14008 state.tally = Some(Tally {
14009 first_choice: BTreeMap::from([('A', 1)]),
14010 borda: BTreeMap::new(),
14011 winner: 'A',
14012 rankings: 1,
14013 unanimous_initial: true,
14014 deliberated: false,
14015 changed_votes: 0,
14016 unanimous_final: true,
14017 tie_break: None,
14018 judges: 0,
14019 present: 0,
14020 quorum: 0,
14021 met_quorum: true,
14022 uncontested: Some("only candidate A produced a change".to_owned()),
14023 });
14024 state.reviews = vec![ReviewRound {
14025 round: 1,
14026 head: "deadbeef".to_owned(),
14027 verified_head: None,
14028 verified_at: None,
14029 reviews: Vec::new(),
14030 e2e: Vec::new(),
14031 fix: None,
14032 blocking: 0,
14033 answered: 0,
14034 expected: 0,
14035 clean: true,
14036 verify_retried: false,
14037 e2e_deferred: false,
14038 e2e_defer_reason: None,
14039 progressed: false,
14040 vote_split: false,
14041 reconsideration: Vec::new(),
14042 verdict: None,
14043 }];
14044 state.gate = Vec::new();
14046 state.gate_ran = false;
14047 state.status = RunStatus::Gating;
14048
14049 let mut runner = Runner {
14050 state,
14051 roles: ResolvedRoles {
14052 implementers: Vec::new(),
14053 judges: Vec::new(),
14054 reviewers: Vec::new(),
14055 fixer: None,
14056 conductor: conductor(),
14057 implementer_roster: Vec::new(),
14058 judge_roster: Vec::new(),
14059 reviewer_roster: Vec::new(),
14060 },
14061 sem: Arc::new(Semaphore::new(1)),
14062 pause: Pause::new(),
14063 interrupt: Pause::new(),
14064 };
14065
14066 runner.merge().await.expect("merge");
14067
14068 assert!(
14069 runner.state.merge.is_none(),
14070 "an empty gate must never be read as a passing one: {:?}",
14071 runner.state.merge
14072 );
14073 }
14074
14075 #[tokio::test]
14082 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14083 let tmp = tempfile::tempdir().expect("tempdir");
14084 let repo = tmp.path().join("repo");
14085 std::fs::create_dir_all(&repo).unwrap();
14086 init_repo(&repo);
14087
14088 let config = Config::default();
14090
14091 let mut state = RunState::new(
14092 repo.clone(),
14093 "main".to_owned(),
14094 "deadbeef".to_owned(),
14095 "task".to_owned(),
14096 config,
14097 );
14098 state.candidates = vec![Candidate {
14099 index: 0,
14100 label: 'A',
14101 agent: "alpha".to_owned(),
14102 branch: "does-not-exist".to_owned(),
14103 worktree: repo.clone(),
14104 summary: String::new(),
14105 stat: String::new(),
14106 files: 0,
14107 commits: 0,
14108 empty: false,
14109 failed: None,
14110 verified_noop: None,
14111 duration_ms: 0,
14112 folded: false,
14113 }];
14114 state.tally = Some(Tally {
14115 first_choice: BTreeMap::from([('A', 1)]),
14116 borda: BTreeMap::new(),
14117 winner: 'A',
14118 rankings: 1,
14119 unanimous_initial: true,
14120 deliberated: false,
14121 changed_votes: 0,
14122 unanimous_final: true,
14123 tie_break: None,
14124 judges: 0,
14125 present: 0,
14126 quorum: 0,
14127 met_quorum: true,
14128 uncontested: Some("only candidate A produced a change".to_owned()),
14129 });
14130 state.reviews = vec![ReviewRound {
14131 round: 1,
14132 head: "deadbeef".to_owned(),
14133 verified_head: None,
14134 verified_at: None,
14135 reviews: Vec::new(),
14136 e2e: Vec::new(),
14137 fix: None,
14138 blocking: 0,
14139 answered: 0,
14140 expected: 0,
14141 clean: true,
14142 verify_retried: false,
14143 e2e_deferred: false,
14144 e2e_defer_reason: None,
14145 progressed: false,
14146 vote_split: false,
14147 reconsideration: Vec::new(),
14148 verdict: None,
14149 }];
14150
14151 let mut runner = Runner {
14152 state,
14153 roles: ResolvedRoles {
14154 implementers: Vec::new(),
14155 judges: Vec::new(),
14156 reviewers: Vec::new(),
14157 fixer: None,
14158 conductor: conductor(),
14159 implementer_roster: Vec::new(),
14160 judge_roster: Vec::new(),
14161 reviewer_roster: Vec::new(),
14162 },
14163 sem: Arc::new(Semaphore::new(1)),
14164 pause: Pause::new(),
14165 interrupt: Pause::new(),
14166 };
14167
14168 runner.gate().await.expect("gate");
14169 assert!(
14170 runner.state.gate_ran,
14171 "zero configured commands is still a real attempt, not an unrun gate"
14172 );
14173 assert!(runner.state.gate.is_empty());
14174 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
14175 assert_ne!(
14176 runner.state.status,
14177 RunStatus::Blocked,
14178 "a gate with nothing to check must not read as failed"
14179 );
14180
14181 runner.merge().await.expect("merge");
14182 assert_eq!(
14183 runner.state.status,
14184 RunStatus::Ready,
14185 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14186 );
14187 }
14188
14189 #[tokio::test]
14200 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
14201 crate::run::pin_test_home();
14202 let home = crate::run::home();
14203
14204 let tmp = tempfile::tempdir().expect("tempdir");
14205 let repo = tmp.path().join("repo");
14206 std::fs::create_dir_all(&repo).unwrap();
14207 init_repo(&repo);
14208 let cache_dir = tmp.path().join("target");
14211
14212 let mut config = Config::default();
14213 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
14214 config.graph.timeout_verify = Some(2);
14217 config.graph.timeout_resource_wait = 2;
14218
14219 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14220 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14221 .expect("no io error acquiring directly")
14222 {
14223 crate::cache::AcquireOutcome::Acquired(g) => g,
14224 crate::cache::AcquireOutcome::Busy(b) => {
14225 panic!("expected the direct acquire to win the lease first: {b:?}")
14226 }
14227 };
14228
14229 let mut state = RunState::new(
14230 repo.clone(),
14231 "main".to_owned(),
14232 "deadbeef".to_owned(),
14233 "task".to_owned(),
14234 config,
14235 );
14236 state.candidates = vec![Candidate {
14237 index: 0,
14238 label: 'A',
14239 agent: "alpha".to_owned(),
14240 branch: "does-not-exist".to_owned(),
14241 worktree: repo.clone(),
14242 summary: String::new(),
14243 stat: String::new(),
14244 files: 0,
14245 commits: 0,
14246 empty: false,
14247 failed: None,
14248 verified_noop: None,
14249 duration_ms: 0,
14250 folded: false,
14251 }];
14252 state.tally = Some(Tally {
14253 first_choice: BTreeMap::from([('A', 1)]),
14254 borda: BTreeMap::new(),
14255 winner: 'A',
14256 rankings: 1,
14257 unanimous_initial: true,
14258 deliberated: false,
14259 changed_votes: 0,
14260 unanimous_final: true,
14261 tie_break: None,
14262 judges: 0,
14263 present: 0,
14264 quorum: 0,
14265 met_quorum: true,
14266 uncontested: Some("only candidate A produced a change".to_owned()),
14267 });
14268 state.reviews = vec![ReviewRound {
14269 round: 1,
14270 head: "deadbeef".to_owned(),
14271 verified_head: None,
14272 verified_at: None,
14273 reviews: Vec::new(),
14274 e2e: Vec::new(),
14275 fix: None,
14276 blocking: 0,
14277 answered: 0,
14278 expected: 0,
14279 clean: true,
14280 verify_retried: false,
14281 e2e_deferred: false,
14282 e2e_defer_reason: None,
14283 progressed: false,
14284 vote_split: false,
14285 reconsideration: Vec::new(),
14286 verdict: None,
14287 }];
14288
14289 let mut runner = Runner {
14290 state,
14291 roles: ResolvedRoles {
14292 implementers: Vec::new(),
14293 judges: Vec::new(),
14294 reviewers: Vec::new(),
14295 fixer: None,
14296 conductor: conductor(),
14297 implementer_roster: Vec::new(),
14298 judge_roster: Vec::new(),
14299 reviewer_roster: Vec::new(),
14300 },
14301 sem: Arc::new(Semaphore::new(1)),
14302 pause: Pause::new(),
14303 interrupt: Pause::new(),
14304 };
14305
14306 let started = std::time::Instant::now();
14307 runner.gate().await.expect("gate");
14308 assert!(
14309 started.elapsed() < Duration::from_secs(1),
14310 "a gate with nothing to run must never wait on a lease it never needed"
14311 );
14312 assert!(
14313 runner.state.gate_ran,
14314 "zero commands is still a real, immediate attempt"
14315 );
14316 assert!(runner.state.gate.is_empty());
14317 assert_ne!(
14318 runner.state.status,
14319 RunStatus::Blocked,
14320 "must not read as resource-blocked on a lease it never asked for"
14321 );
14322 }
14323
14324 #[tokio::test]
14334 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
14335 crate::run::pin_test_home();
14336
14337 let tmp = tempfile::tempdir().expect("tempdir");
14338 let repo = tmp.path().join("repo");
14339 std::fs::create_dir_all(&repo).unwrap();
14340 init_repo(&repo);
14341
14342 let mut config = Config::default();
14343 config.verify.gate = vec![
14344 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
14345 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
14346 .to_owned(),
14347 ];
14348
14349 let mut state = RunState::new(
14350 repo.clone(),
14351 "main".to_owned(),
14352 "deadbeef".to_owned(),
14353 "task".to_owned(),
14354 config,
14355 );
14356 let run_id = state.id.clone();
14357 state.candidates = vec![Candidate {
14358 index: 0,
14359 label: 'A',
14360 agent: "alpha".to_owned(),
14361 branch: "does-not-exist".to_owned(),
14362 worktree: repo.clone(),
14363 summary: String::new(),
14364 stat: String::new(),
14365 files: 0,
14366 commits: 0,
14367 empty: false,
14368 failed: None,
14369 verified_noop: None,
14370 duration_ms: 0,
14371 folded: false,
14372 }];
14373 state.tally = Some(Tally {
14374 first_choice: BTreeMap::from([('A', 1)]),
14375 borda: BTreeMap::new(),
14376 winner: 'A',
14377 rankings: 1,
14378 unanimous_initial: true,
14379 deliberated: false,
14380 changed_votes: 0,
14381 unanimous_final: true,
14382 tie_break: None,
14383 judges: 0,
14384 present: 0,
14385 quorum: 0,
14386 met_quorum: true,
14387 uncontested: Some("only candidate A produced a change".to_owned()),
14388 });
14389 state.reviews = vec![ReviewRound {
14390 round: 1,
14391 head: "deadbeef".to_owned(),
14392 verified_head: None,
14393 verified_at: None,
14394 reviews: Vec::new(),
14395 e2e: Vec::new(),
14396 fix: None,
14397 blocking: 0,
14398 answered: 0,
14399 expected: 0,
14400 clean: true,
14401 verify_retried: false,
14402 e2e_deferred: false,
14403 e2e_defer_reason: None,
14404 progressed: false,
14405 vote_split: false,
14406 reconsideration: Vec::new(),
14407 verdict: None,
14408 }];
14409
14410 let mut runner = Runner {
14411 state,
14412 roles: ResolvedRoles {
14413 implementers: Vec::new(),
14414 judges: Vec::new(),
14415 reviewers: Vec::new(),
14416 fixer: None,
14417 conductor: conductor(),
14418 implementer_roster: Vec::new(),
14419 judge_roster: Vec::new(),
14420 reviewer_roster: Vec::new(),
14421 },
14422 sem: Arc::new(Semaphore::new(1)),
14423 pause: Pause::new(),
14424 interrupt: Pause::new(),
14425 };
14426
14427 let started_marker = repo.join("started.marker");
14428 let release_marker = repo.join("release.marker");
14429 let poller = tokio::spawn(async move {
14430 for _ in 0..100 {
14435 if started_marker.exists()
14436 && let Ok(s) = crate::run::RunState::load(&run_id)
14437 && let Some(a) = s.active.get("gate")
14438 {
14439 std::fs::write(&release_marker, b"go").expect("release marker");
14440 return Some(a.clone());
14441 }
14442 tokio::time::sleep(Duration::from_millis(50)).await;
14443 }
14444 None
14445 });
14446
14447 runner.gate().await.expect("gate");
14448 let captured = poller.await.expect("poller task");
14449 let captured = captured.expect(
14450 "the poller never saw a `gate` task entry in run.json while the command was \
14451 still blocked on its own release marker",
14452 );
14453
14454 assert_eq!(captured.task.as_deref(), Some("gate"));
14455 assert_eq!(captured.node, "gate");
14456 assert_eq!(captured.index, Some(1));
14457 assert_eq!(captured.total, Some(1));
14458 assert!(
14459 captured
14460 .command
14461 .as_deref()
14462 .is_some_and(|c| c.contains("started.marker")),
14463 "{captured:?}"
14464 );
14465
14466 assert!(
14467 runner.state.active.is_empty(),
14468 "the entry must be cleared once the command actually finished: {:?}",
14469 runner.state.active
14470 );
14471 assert!(runner.state.gate_ran);
14472 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
14473 }
14474
14475 #[tokio::test]
14478 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
14479 use crate::verdict::{Finding, ReviewVote, Severity};
14480 crate::run::pin_test_home();
14481 let tmp = tempfile::tempdir().expect("tempdir");
14482 let repo = tmp.path().join("repo");
14483 std::fs::create_dir_all(&repo).unwrap();
14484 init_repo(&repo);
14485
14486 for (severity, vote, expect) in [
14487 (Severity::Major, ReviewVote::Reject, true),
14488 (Severity::Minor, ReviewVote::Reject, false),
14489 (Severity::Major, ReviewVote::Approve, false),
14490 ] {
14491 let mut round = review_round(false, 1, 1, 1, false, true);
14492 round.reviews = vec![ReviewRecord {
14493 reviewer: 1,
14494 agent: "alpha".to_owned(),
14495 summary: String::new(),
14496 findings: vec![Finding {
14497 id: "R1-1-1".to_owned(),
14498 severity,
14499 file: None,
14500 line: None,
14501 title: "t".to_owned(),
14502 detail: String::new(),
14503 }],
14504 vote: Some(vote),
14505 failed: None,
14506 duration_ms: 0,
14507 attempts: 0,
14508 }];
14509 let mut state = RunState::new(
14510 repo.clone(),
14511 "main".to_owned(),
14512 "deadbeef".to_owned(),
14513 "task".to_owned(),
14514 Config::default(),
14515 );
14516 state.reviews = vec![round];
14517 let mut runner = Runner {
14518 state,
14519 roles: ResolvedRoles {
14520 implementers: Vec::new(),
14521 judges: Vec::new(),
14522 reviewers: Vec::new(),
14523 fixer: None,
14524 conductor: conductor(),
14525 implementer_roster: Vec::new(),
14526 judge_roster: Vec::new(),
14527 reviewer_roster: Vec::new(),
14528 },
14529 sem: Arc::new(Semaphore::new(1)),
14530 pause: Pause::new(),
14531 interrupt: Pause::new(),
14532 };
14533 let shell = runner.state.config.shell();
14534 runner
14535 .stop_reviewing("round budget spent", &shell, &repo)
14536 .await
14537 .expect("stop_reviewing");
14538 assert_eq!(runner.state.status, RunStatus::Gating);
14539 assert_eq!(
14540 runner.state.contested_handoff.is_some(),
14541 expect,
14542 "{severity:?} + {vote:?}"
14543 );
14544 }
14545 }
14546
14547 #[tokio::test]
14560 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
14561 crate::run::pin_test_home();
14562 let home = crate::run::home();
14563
14564 let tmp = tempfile::tempdir().expect("tempdir");
14565 let repo = tmp.path().join("repo");
14566 std::fs::create_dir_all(&repo).unwrap();
14567 init_repo(&repo);
14568 let head = crate::git::rev_parse(&repo, "HEAD")
14569 .await
14570 .expect("rev-parse");
14571 let cache_dir = tmp.path().join("target");
14574
14575 let mut config = Config::default();
14576 config.verify.e2e = vec![format!(
14577 "CARGO_TARGET_DIR='{}' test -f README.md",
14578 cache_dir.display()
14579 )];
14580 config.graph.review_rounds = 1;
14581 config.graph.timeout_verify = Some(2);
14584 config.graph.timeout_resource_wait = 2;
14585
14586 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14587 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14588 .expect("no io error acquiring directly")
14589 {
14590 crate::cache::AcquireOutcome::Acquired(g) => g,
14591 crate::cache::AcquireOutcome::Busy(b) => {
14592 panic!("expected the direct acquire to win the lease first: {b:?}")
14593 }
14594 };
14595
14596 let mut state = RunState::new(
14597 repo.clone(),
14598 "main".to_owned(),
14599 head.clone(),
14600 "task".to_owned(),
14601 config,
14602 );
14603 state.candidates = vec![Candidate {
14604 index: 0,
14605 label: 'A',
14606 agent: "alpha".to_owned(),
14607 branch: "does-not-exist".to_owned(),
14608 worktree: repo.clone(),
14609 summary: String::new(),
14610 stat: String::new(),
14611 files: 0,
14612 commits: 0,
14613 empty: false,
14614 failed: None,
14615 verified_noop: None,
14616 duration_ms: 0,
14617 folded: false,
14618 }];
14619 state.tally = Some(Tally {
14620 first_choice: BTreeMap::from([('A', 1)]),
14621 borda: BTreeMap::new(),
14622 winner: 'A',
14623 rankings: 1,
14624 unanimous_initial: true,
14625 deliberated: false,
14626 changed_votes: 0,
14627 unanimous_final: true,
14628 tie_break: None,
14629 judges: 0,
14630 present: 0,
14631 quorum: 0,
14632 met_quorum: true,
14633 uncontested: Some("only candidate A produced a change".to_owned()),
14634 });
14635 state.reviews = vec![ReviewRound {
14639 round: 1,
14640 head: head.clone(),
14641 verified_head: None,
14642 verified_at: None,
14643 reviews: Vec::new(),
14644 e2e: Vec::new(),
14645 fix: None,
14646 blocking: 1,
14647 answered: 1,
14648 expected: 1,
14649 clean: false,
14650 verify_retried: false,
14651 e2e_deferred: true,
14652 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
14653 progressed: false,
14654 vote_split: false,
14655 reconsideration: Vec::new(),
14656 verdict: None,
14657 }];
14658
14659 let mut runner = Runner {
14660 state,
14661 roles: ResolvedRoles {
14662 implementers: Vec::new(),
14663 judges: Vec::new(),
14664 reviewers: Vec::new(),
14665 fixer: None,
14666 conductor: conductor(),
14667 implementer_roster: Vec::new(),
14668 judge_roster: Vec::new(),
14669 reviewer_roster: Vec::new(),
14670 },
14671 sem: Arc::new(Semaphore::new(1)),
14672 pause: Pause::new(),
14673 interrupt: Pause::new(),
14674 };
14675
14676 let shell = runner.state.config.shell();
14677 runner
14678 .stop_reviewing("round budget spent", &shell, &repo)
14679 .await
14680 .expect("stop_reviewing");
14681
14682 let last = runner.state.reviews.last().expect("round record");
14683 assert_eq!(
14684 last.e2e_status(),
14685 E2eStatus::ResourceBlocked,
14686 "the shared cache is still held; the attempt must read as blocked, not deferred or \
14687 failed: {last:?}"
14688 );
14689 assert_eq!(
14690 last.verified_head.as_deref(),
14691 Some(head.as_str()),
14692 "which commit this attempt targeted is known even though nothing finished checking \
14693 it"
14694 );
14695 let first_attempt_at = last
14696 .verified_at
14697 .expect("when this attempt ran is known too");
14698 assert_ne!(
14699 runner.state.status,
14700 RunStatus::Blocked,
14701 "contention is evidence about the machine, not the patch — it must not settle the \
14702 run as blocked: {:?}",
14703 runner.state.status
14704 );
14705 assert!(
14706 !runner
14707 .state
14708 .events
14709 .iter()
14710 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
14711 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
14712 runner.state.events
14713 );
14714
14715 runner
14720 .stop_reviewing("round budget spent", &shell, &repo)
14721 .await
14722 .expect("stop_reviewing retry");
14723 assert_eq!(
14724 runner.state.reviews.len(),
14725 1,
14726 "no new round was started: {:?}",
14727 runner.state.reviews
14728 );
14729 let last = runner.state.reviews.last().expect("round record");
14730 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
14731 assert!(
14732 last.verified_at.expect("still known") > first_attempt_at,
14733 "a second reentry must be a fresh attempt, not a stale copy of the first"
14734 );
14735 assert_ne!(runner.state.status, RunStatus::Blocked);
14736
14737 held.release();
14738 }
14739
14740 #[tokio::test]
14752 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
14753 crate::run::pin_test_home();
14754 let home = crate::run::home();
14755
14756 let tmp = tempfile::tempdir().expect("tempdir");
14757 let repo = tmp.path().join("repo");
14758 std::fs::create_dir_all(&repo).unwrap();
14759 init_repo(&repo);
14760 let head = crate::git::rev_parse(&repo, "HEAD")
14761 .await
14762 .expect("rev-parse");
14763 let cache_dir = tmp.path().join("target");
14764
14765 let mut config = Config::default();
14766 config.verify.e2e = vec![format!(
14767 "CARGO_TARGET_DIR='{}' test -f README.md",
14768 cache_dir.display()
14769 )];
14770 config.graph.review_rounds = 1;
14771 config.graph.timeout_verify = Some(2);
14772 config.graph.timeout_resource_wait = 2;
14773
14774 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14775 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14776 .expect("no io error acquiring directly")
14777 {
14778 crate::cache::AcquireOutcome::Acquired(g) => g,
14779 crate::cache::AcquireOutcome::Busy(b) => {
14780 panic!("expected the direct acquire to win the lease first: {b:?}")
14781 }
14782 };
14783
14784 let mut state = RunState::new(
14785 repo.clone(),
14786 "main".to_owned(),
14787 head.clone(),
14788 "task".to_owned(),
14789 config,
14790 );
14791 state.candidates = vec![Candidate {
14792 index: 0,
14793 label: 'A',
14794 agent: "alpha".to_owned(),
14795 branch: "does-not-exist".to_owned(),
14796 worktree: repo.clone(),
14797 summary: String::new(),
14798 stat: String::new(),
14799 files: 0,
14800 commits: 0,
14801 empty: false,
14802 failed: None,
14803 verified_noop: None,
14804 duration_ms: 0,
14805 folded: false,
14806 }];
14807 state.tally = Some(Tally {
14808 first_choice: BTreeMap::from([('A', 1)]),
14809 borda: BTreeMap::new(),
14810 winner: 'A',
14811 rankings: 1,
14812 unanimous_initial: true,
14813 deliberated: false,
14814 changed_votes: 0,
14815 unanimous_final: true,
14816 tie_break: None,
14817 judges: 0,
14818 present: 0,
14819 quorum: 0,
14820 met_quorum: true,
14821 uncontested: Some("only candidate A produced a change".to_owned()),
14822 });
14823 state.reviews = vec![ReviewRound {
14827 round: 1,
14828 head: head.clone(),
14829 verified_head: Some(head.clone()),
14830 verified_at: Some(jiff::Timestamp::now()),
14831 reviews: Vec::new(),
14832 e2e: vec![CommandOutcome {
14833 command: format!(
14834 "CARGO_TARGET_DIR='{}' test -f README.md",
14835 cache_dir.display()
14836 ),
14837 code: None,
14838 output_tail: "waiting for the shared build cache".to_owned(),
14839 duration_ms: 0,
14840 resource_blocked: true,
14841 }],
14842 fix: None,
14843 blocking: 1,
14844 answered: 1,
14845 expected: 1,
14846 clean: false,
14847 verify_retried: false,
14848 e2e_deferred: false,
14849 e2e_defer_reason: None,
14850 progressed: false,
14851 vote_split: false,
14852 reconsideration: Vec::new(),
14853 verdict: None,
14854 }];
14855
14856 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
14857 let mut runner = Runner {
14858 state,
14859 roles: ResolvedRoles {
14860 implementers: Vec::new(),
14861 judges: Vec::new(),
14862 reviewers: Vec::new(),
14863 fixer: None,
14864 conductor: conductor(),
14865 implementer_roster: Vec::new(),
14866 judge_roster: Vec::new(),
14867 reviewer_roster: Vec::new(),
14868 },
14869 sem: Arc::new(Semaphore::new(1)),
14870 pause: Pause::new(),
14871 interrupt: Pause::new(),
14872 };
14873
14874 runner.review_loop().await.expect("review_loop");
14879
14880 assert_eq!(
14881 runner.state.reviews.len(),
14882 1,
14883 "no new round was started on top of the unresolved one: {:?}",
14884 runner.state.reviews
14885 );
14886 let last = &runner.state.reviews[0];
14887 assert_eq!(
14888 last.e2e_status(),
14889 E2eStatus::ResourceBlocked,
14890 "still contended: {last:?}"
14891 );
14892 assert!(
14893 last.verified_at.expect("still known") > first_attempt_at,
14894 "review_loop must have actually retried the check, not left it exactly as found"
14895 );
14896 assert_ne!(
14897 runner.state.status,
14898 RunStatus::Blocked,
14899 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
14900 runner.state.status
14901 );
14902
14903 held.release();
14904 }
14905
14906 #[tokio::test]
14907 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
14908 crate::run::pin_test_home();
14909 let tmp = tempfile::tempdir().expect("tempdir");
14910 let repo = tmp.path().join("repo");
14911 std::fs::create_dir_all(&repo).unwrap();
14912 init_repo(&repo);
14913
14914 let mut config = Config::default();
14915 config.merge.mode = MergeMode::Pr;
14916 config.graph.land = true;
14917 config.graph.land_approval = false;
14918
14919 let mut state = RunState::new(
14920 repo.clone(),
14921 "main".to_owned(),
14922 "deadbeef".to_owned(),
14923 "task".to_owned(),
14924 config,
14925 );
14926 state.candidates = vec![Candidate {
14927 index: 0,
14928 label: 'A',
14929 agent: "alpha".to_owned(),
14930 branch: "does-not-exist".to_owned(),
14931 worktree: repo.clone(),
14932 summary: String::new(),
14933 stat: String::new(),
14934 files: 0,
14935 commits: 0,
14936 empty: false,
14937 failed: None,
14938 verified_noop: None,
14939 duration_ms: 0,
14940 folded: false,
14941 }];
14942 state.tally = Some(Tally {
14943 first_choice: BTreeMap::from([('A', 1)]),
14944 borda: BTreeMap::new(),
14945 winner: 'A',
14946 rankings: 1,
14947 unanimous_initial: true,
14948 deliberated: false,
14949 changed_votes: 0,
14950 unanimous_final: true,
14951 tie_break: None,
14952 judges: 0,
14953 present: 0,
14954 quorum: 0,
14955 met_quorum: true,
14956 uncontested: Some("only candidate A produced a change".to_owned()),
14957 });
14958 state.reviews = vec![ReviewRound {
14959 round: 1,
14960 head: "deadbeef".to_owned(),
14961 verified_head: None,
14962 verified_at: None,
14963 reviews: Vec::new(),
14964 e2e: Vec::new(),
14965 fix: None,
14966 blocking: 0,
14967 answered: 0,
14968 expected: 0,
14969 clean: true,
14970 verify_retried: false,
14971 e2e_deferred: false,
14972 e2e_defer_reason: None,
14973 progressed: false,
14974 vote_split: false,
14975 reconsideration: Vec::new(),
14976 verdict: None,
14977 }];
14978 state.gate = vec![CommandOutcome {
14979 command: "test".to_owned(),
14980 code: Some(0),
14981 output_tail: String::new(),
14982 duration_ms: 0,
14983 resource_blocked: false,
14984 }];
14985 state.gate_ran = true;
14986 state.status = RunStatus::Landing;
14990 state.merge = Some(MergeOutcome {
14991 mode: MergeMode::Pr,
14992 ok: true,
14993 detail: "https://example.invalid/x/y/pull/1".to_owned(),
14994 empty: false,
14995 });
14996
14997 ask_test_home();
15001 state.save().expect("save landing run");
15004 let store = ask::Questions::open();
15005 let q = ask_open_question(&store, &state.id);
15006
15007 let mut runner = Runner {
15008 state,
15009 roles: ResolvedRoles {
15010 implementers: Vec::new(),
15011 judges: Vec::new(),
15012 reviewers: Vec::new(),
15013 fixer: None,
15014 conductor: conductor(),
15015 implementer_roster: Vec::new(),
15016 judge_roster: Vec::new(),
15017 reviewer_roster: Vec::new(),
15018 },
15019 sem: Arc::new(Semaphore::new(1)),
15020 pause: Pause::new(),
15021 interrupt: Pause::new(),
15022 };
15023
15024 runner.execute().await.expect("execute");
15029
15030 assert_eq!(
15031 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15032 Some("https://example.invalid/x/y/pull/1"),
15033 "reentry must not push again or open a second pull request over the \
15034 one `land` is already watching"
15035 );
15036 assert_ne!(
15037 runner.state.status,
15038 RunStatus::Landing,
15039 "land could not actually reach the fake pull request, so it must \
15040 have given up rather than left the run silently parked forever"
15041 );
15042 assert_eq!(runner.state.status, RunStatus::Blocked);
15046 assert!(
15047 store.get(&q.id).unwrap().status.open(),
15048 "Blocked is still alive; settle_questions must have been a no-op here"
15049 );
15050 }
15051
15052 fn state_with_round(round: ReviewRound) -> RunState {
15053 let mut s = RunState::new(
15054 PathBuf::from("/repo"),
15055 "main".to_owned(),
15056 "abc1234".to_owned(),
15057 "add retries".to_owned(),
15058 Config::default(),
15059 );
15060 s.reviews = vec![round];
15061 s
15062 }
15063
15064 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15065 crate::verdict::Finding {
15066 id: id.to_owned(),
15067 severity,
15068 file: None,
15069 line: None,
15070 title: title.to_owned(),
15071 detail: String::new(),
15072 }
15073 }
15074
15075 #[test]
15076 fn pr_body_names_open_findings_and_declined_ones() {
15077 let round = ReviewRound {
15078 round: 2,
15079 head: "deadbee".to_owned(),
15080 verified_head: None,
15081 verified_at: None,
15082 reviews: vec![ReviewRecord {
15083 attempts: 0,
15084 reviewer: 1,
15085 agent: "alpha".to_owned(),
15086 summary: String::new(),
15087 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15088 vote: None,
15089 failed: None,
15090 duration_ms: 0,
15091 }],
15092 e2e: vec![CommandOutcome {
15093 command: "cargo test".to_owned(),
15094 code: Some(0),
15095 output_tail: String::new(),
15096 duration_ms: 0,
15097 resource_blocked: false,
15098 }],
15099 verify_retried: false,
15100 e2e_deferred: false,
15101 e2e_defer_reason: None,
15102 fix: Some(FixRecord {
15103 agent: "alpha".to_owned(),
15104 addressed: Vec::new(),
15105 rejected: vec![crate::verdict::Rejection {
15106 id: "R1-1-1".to_owned(),
15107 why: "not reachable from any caller".to_owned(),
15108 }],
15109 notes: String::new(),
15110 committed: true,
15111 failed: None,
15112 duration_ms: 0,
15113 continuation: None,
15114 }),
15115 blocking: 0,
15116 answered: 1,
15117 expected: 1,
15118 clean: false,
15119 progressed: true,
15120 vote_split: false,
15121 reconsideration: Vec::new(),
15122 verdict: None,
15123 };
15124 let state = state_with_round(round);
15125 let body = pr_message(&state, 'A').body;
15126
15127 assert!(body.contains("add retries"), "the task must still be there");
15128 assert!(body.contains("R2-1-1"), "{body}");
15129 assert!(!body.contains("unused import"), "{body}");
15130 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15131 assert!(
15132 !body.contains("not reachable from any caller"),
15133 "the reason it was declined: {body}"
15134 );
15135 }
15136
15137 #[test]
15138 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15139 let round = ReviewRound {
15140 round: 1,
15141 head: "deadbee".to_owned(),
15142 verified_head: None,
15143 verified_at: None,
15144 reviews: vec![ReviewRecord {
15145 attempts: 0,
15146 reviewer: 1,
15147 agent: "alpha".to_owned(),
15148 summary: String::new(),
15149 findings: Vec::new(),
15150 vote: None,
15151 failed: None,
15152 duration_ms: 0,
15153 }],
15154 e2e: Vec::new(),
15155 verify_retried: false,
15156 e2e_deferred: false,
15157 e2e_defer_reason: None,
15158 fix: None,
15159 blocking: 0,
15160 answered: 1,
15161 expected: 1,
15162 clean: true,
15163 progressed: false,
15164 vote_split: false,
15165 reconsideration: Vec::new(),
15166 verdict: None,
15167 };
15168 let state = state_with_round(round);
15169 let body = pr_message(&state, 'A').body;
15170 assert!(!body.contains("Open review findings"), "{body}");
15171 assert!(!body.contains("Declined"), "{body}");
15172 }
15173
15174 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15175 let mut state = RunState::new(
15176 PathBuf::from("/repo"),
15177 "main".to_owned(),
15178 "abc1234".to_owned(),
15179 instruction.to_owned(),
15180 Config::default(),
15181 );
15182 state.candidates.push(Candidate {
15183 index: 0,
15184 label: 'A',
15185 agent: "alpha".to_owned(),
15186 branch: "magi/x/A".to_owned(),
15187 worktree: PathBuf::from("/wt"),
15188 summary: summary.to_owned(),
15189 stat: String::new(),
15190 files: 1,
15191 commits: 1,
15192 empty: false,
15193 failed: None,
15194 verified_noop: None,
15195 folded: false,
15196 duration_ms: 0,
15197 });
15198 state
15199 }
15200
15201 fn review_state(subjects: &[&str]) -> RunState {
15202 let mut state = state_with_summary(
15203 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
15204 "",
15205 );
15206 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
15207 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
15208 state
15209 }
15210
15211 #[tokio::test]
15214 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
15215 ask_test_home();
15216 let tmp = tempfile::tempdir().unwrap();
15217 let repo = tmp.path().join("repo");
15218 std::fs::create_dir_all(&repo).unwrap();
15219 init_repo(&repo);
15220 let origin = tmp.path().join("origin.git");
15221 let g = |dir: &Path, args: &[&str]| {
15222 let out = std::process::Command::new("git")
15223 .args(args)
15224 .current_dir(dir)
15225 .quiet()
15226 .output()
15227 .expect("spawn git");
15228 assert!(
15229 out.status.success(),
15230 "git {args:?}: {}",
15231 String::from_utf8_lossy(&out.stderr)
15232 );
15233 };
15234 g(
15235 tmp.path(),
15236 &[
15237 "clone",
15238 "--bare",
15239 "-q",
15240 repo.to_str().unwrap(),
15241 origin.to_str().unwrap(),
15242 ],
15243 );
15244 g(
15245 &repo,
15246 &["remote", "add", "origin", origin.to_str().unwrap()],
15247 );
15248 let c1 = git::rev_parse(&repo, "main").await.unwrap();
15249
15250 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
15252 g(&repo, &["add", "-A"]);
15253 g(
15254 &repo,
15255 &["commit", "-q", "-m", "chore(deps): update a crate"],
15256 );
15257 g(&repo, &["push", "-q", "origin", "main"]);
15258 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
15259 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
15260 g(&repo, &["add", "-A"]);
15261 g(
15262 &repo,
15263 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
15264 );
15265 g(&repo, &["checkout", "-q", "main"]);
15266
15267 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
15268 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
15269 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
15271 assert_eq!(fallback, c1);
15272
15273 let mut state = review_state(&[
15274 "chore(deps): update a crate",
15275 "fix(daemon): apply a chosen action",
15276 ]);
15277 state.repo = repo.clone();
15278 state.base_branch = "main".to_owned();
15279 state.base_commit = c1;
15280 refresh_reviewed_commits(&mut state, "feat/own").await;
15281 assert_eq!(
15282 state.reviewed_commits,
15283 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
15284 );
15285 assert_eq!(
15286 review_title(&state).as_deref(),
15287 Some("fix(daemon): apply a chosen action")
15288 );
15289 assert_eq!(
15290 leaked_subjects(&state, "feat/own").await,
15291 Some(vec!["chore(deps): update a crate".to_owned()])
15292 );
15293 state.config.merge.remote = "nowhere".to_owned();
15296 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
15297 }
15298
15299 struct SummaryFixture {
15303 _tmp: tempfile::TempDir,
15304 repo: PathBuf,
15305 reply: PathBuf,
15306 runner: Runner,
15307 }
15308
15309 fn summary_fixture() -> SummaryFixture {
15310 ask_test_home();
15311 let tmp = tempfile::tempdir().unwrap();
15312 let repo = tmp.path().join("repo");
15313 std::fs::create_dir_all(&repo).unwrap();
15314 init_repo(&repo);
15315 let origin = tmp.path().join("origin.git");
15316 let g = |args: &[&str]| {
15317 let out = std::process::Command::new("git")
15318 .args(args)
15319 .current_dir(&repo)
15320 .quiet()
15321 .output()
15322 .expect("spawn git");
15323 assert!(
15324 out.status.success(),
15325 "git {args:?}: {}",
15326 String::from_utf8_lossy(&out.stderr)
15327 );
15328 };
15329 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
15330 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
15331 g(&["push", "-q", "origin", "main"]);
15332 g(&["checkout", "-q", "-b", "feat/own"]);
15333 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
15334 g(&["add", "-A"]);
15335 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
15336 let reply = tmp.path().join("reply.txt");
15337 std::fs::write(
15338 &reply,
15339 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
15340 )
15341 .unwrap();
15342 let mut runner = runner_at(RunStatus::Gating);
15343 let mut state = review_state(&["feat(own): add the own file"]);
15344 state.repo = repo.clone();
15345 state.base_branch = "main".to_owned();
15346 state.candidates[0].branch = "feat/own".to_owned();
15347 state.candidates[0].worktree = repo.clone();
15348 let mut author = spec("alpha");
15349 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
15350 author
15351 .env
15352 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
15353 state.config.agents = vec![author];
15354 runner.state = state;
15355 SummaryFixture {
15356 _tmp: tmp,
15357 repo,
15358 reply,
15359 runner,
15360 }
15361 }
15362
15363 async fn posted(f: &mut SummaryFixture) -> PrMessage {
15364 let winner = f.runner.state.candidates[0].clone();
15365 f.runner
15366 .guarded_pr_message(&winner, None, true)
15367 .await
15368 .unwrap()
15369 .expect("an English description never parks")
15370 }
15371
15372 fn clean_round_with_minor_findings() -> ReviewRound {
15373 let mut round = passing_round(2, None);
15374 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
15375 round.reviews = vec![ReviewRecord {
15376 attempts: 0,
15377 reviewer: 1,
15378 agent: "alpha".into(),
15379 summary: "内部レビュー".into(),
15380 findings: vec![
15381 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
15382 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
15383 ],
15384 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
15385 failed: None,
15386 duration_ms: 0,
15387 }];
15388 round
15389 }
15390
15391 #[tokio::test]
15392 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
15393 for review_only in [false, true] {
15394 let mut f = summary_fixture();
15395 if !review_only {
15396 f.runner.state.candidates[0].agent = "alpha".into();
15397 }
15398 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
15399 assert!(f.runner.state.reviews[0].clean);
15400 assert!(f.runner.state.reviews[0].fix.is_none());
15401 assert!(f.runner.state.open_findings().is_empty());
15402 assert_eq!(
15403 crate::pr_outcome::required_finding_ids(&f.runner.state),
15404 ["R2-1-1", "R2-1-2"]
15405 );
15406 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15407 let evidence = crate::pr_outcome::summary_evidence(
15408 &f.runner.state,
15409 crate::pr_outcome::FinalHead::At(&head),
15410 );
15411 assert!(
15412 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
15413 );
15414 let omitted = posted(&mut f).await;
15416 assert!(omitted.body.contains("No description"));
15417 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
15418 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
15419 assert!(f.runner.state.change_summary.is_none());
15420 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
15421 let described = posted(&mut f).await;
15422 assert!(described.body.contains("improve documentation for callers"));
15423 assert!(described.body.contains("remaining edge case"));
15424 assert!(described.body.contains("## Open review findings"));
15425 assert!(!described.body.contains("未修正"));
15426 }
15427 }
15428
15429 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
15430 let mut f = summary_fixture();
15431 if !review_only {
15432 f.runner.state.candidates[0].agent = "alpha".into();
15433 }
15434 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15435 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
15436 fixed
15437 .fix
15438 .as_mut()
15439 .unwrap()
15440 .rejected
15441 .push(crate::verdict::Rejection {
15442 id: "R1-1-1".into(),
15443 why: "呼び出されないコードのため辞退".into(),
15444 });
15445 let mut clean = clean_round_with_minor_findings();
15446 clean.verified_head = Some(head.clone());
15447 f.runner.state.reviews = vec![fixed, clean];
15448 ok_gate(&mut f.runner.state, Some(&head));
15449 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
15450 f.runner.state.config.agents.push(scripted("beta", reply));
15451 f.runner.state.config.roles.language_judge =
15452 Some(crate::config::AgentChoice::One("beta".into()));
15453 f
15454 }
15455
15456 #[tokio::test]
15457 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
15458 for review_only in [false, true] {
15459 let mut f = rewrite_fixture_with_findings(review_only,
15460 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
15461 let m = posted(&mut f).await;
15462 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15463 let expected =
15464 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
15465 assert!(m.body.contains(&expected));
15466 assert!(m.body.contains("## Review rounds"));
15467 assert!(
15468 m.body
15469 .contains("Gate (`verify.gate`): passed on the final commit")
15470 );
15471 assert!(
15472 m.body
15473 .contains("Tests (`verify.e2e`): passed on the final commit")
15474 );
15475 let c = f.runner.state.change_summary.as_ref().unwrap();
15476 assert!(
15477 m.body
15478 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
15479 );
15480 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15481 assert!(crate::pr_outcome::covers_required_findings(
15482 &f.runner.state,
15483 &m.body
15484 ));
15485 assert!(m.body.contains("documentation remains incomplete"));
15486 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
15487 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
15488 assert!(f.runner.state.github_text.is_none());
15489 }
15490 }
15491
15492 #[tokio::test]
15493 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
15494 for review_only in [false, true] {
15495 for reply in [
15496 "TITLE: fix: public description\nA short English description.",
15497 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
15498 ] {
15499 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
15500 let winner = f.runner.state.candidates[0].clone();
15501 let posted = f
15502 .runner
15503 .guarded_pr_message(&winner, None, true)
15504 .await
15505 .unwrap();
15506 assert!(posted.is_none());
15507 assert!(f.runner.state.parked);
15508 let withheld = f.runner.state.github_text.as_ref().unwrap();
15509 assert!(
15510 withheld.rewrite.is_none(),
15511 "an incomplete rewrite must not be selectable"
15512 );
15513 let q = ask::Questions::open()
15514 .get(withheld.question.as_ref().unwrap())
15515 .unwrap();
15516 assert!(
15517 !q.choices
15518 .contains(&crate::github_text::USE_REWRITE.to_owned())
15519 );
15520 assert!(
15521 q.detail.contains("R1-1-1")
15522 && q.detail.contains("R2-1-1")
15523 && q.detail.contains("R2-1-2")
15524 );
15525 }
15526 }
15527 }
15528
15529 #[tokio::test]
15530 async fn japanese_reports_stay_local_in_both_public_run_modes() {
15531 for review_only in [false, true] {
15532 let mut f = summary_fixture();
15533 if !review_only {
15534 f.runner.state.candidates[0].agent = "alpha".into();
15535 }
15536 f.runner.state.instruction = "内部タスクの説明".into();
15537 f.runner.state.candidates[0].summary = "内部の候補報告".into();
15538 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15539 f.runner.state.reviews = vec![
15540 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
15541 round_with_notes(2, Some("二回目の内部修正報告です。")),
15542 passing_round(3, Some(&head)),
15543 ];
15544 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
15545 ok_gate(&mut f.runner.state, Some(&head));
15546 let captured = f.repo.join("public-prompt.txt");
15547 let author = &mut f.runner.state.config.agents[0];
15548 author.command = vec![
15549 "sh".into(),
15550 "-c".into(),
15551 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
15552 ];
15553 author.env.insert(
15554 "PROMPT_FILE".into(),
15555 captured.to_string_lossy().into_owned(),
15556 );
15557 let m = posted(&mut f).await;
15558 assert!(m.body.contains("Adds `own.txt`"));
15559 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
15560 assert!(m.body.contains("R1-1-1"));
15561 assert!(
15562 m.body
15563 .contains("Gate (`verify.gate`): passed on the final commit")
15564 );
15565 assert!(
15566 m.body
15567 .contains("Tests (`verify.e2e`): passed on the final commit")
15568 );
15569 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15570 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
15571 let prompt = std::fs::read_to_string(captured).unwrap();
15572 assert!(prompt.contains(&format!(
15573 "git diff {} {head}",
15574 f.runner.state.change_summary.as_ref().unwrap().base
15575 )));
15576 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
15577 assert_eq!(
15578 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
15579 "二回目の内部修正報告です。"
15580 );
15581 }
15582 }
15583
15584 #[tokio::test]
15585 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
15586 for review_only in [false, true] {
15587 let mut f = summary_fixture();
15588 if !review_only {
15589 f.runner.state.candidates[0].agent = "alpha".into();
15590 }
15591 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
15592 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
15593 std::fs::write(&f.reply, "no usable description").unwrap();
15594 let failed = posted(&mut f).await;
15595 assert!(
15596 failed
15597 .body
15598 .contains("No description of what this change does could be generated")
15599 );
15600 assert!(failed.body.is_ascii());
15601 assert!(f.runner.state.github_text.is_none());
15602 assert!(f.runner.state.change_summary.is_none());
15603 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15604 let evidence = crate::pr_outcome::summary_evidence(
15605 &f.runner.state,
15606 crate::pr_outcome::FinalHead::At(&head),
15607 );
15608 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
15609 assert!(
15610 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
15611 .exists()
15612 );
15613 f.runner.state =
15614 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
15615 std::fs::write(
15616 &f.reply,
15617 "TITLE: feat(own): add the own file\nAdds the committed own file.",
15618 )
15619 .unwrap();
15620 let retried = posted(&mut f).await;
15621 assert!(retried.body.contains("Adds the committed own file."));
15622 assert_eq!(
15623 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
15624 "日本語の修正報告"
15625 );
15626 }
15627 }
15628
15629 #[tokio::test]
15630 async fn same_head_verification_and_findings_invalidate_public_prose() {
15631 let mut f = summary_fixture();
15632 posted(&mut f).await;
15633 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15634 let original = f
15635 .runner
15636 .state
15637 .change_summary
15638 .as_ref()
15639 .unwrap()
15640 .evidence
15641 .clone();
15642 ok_gate(&mut f.runner.state, Some(&head));
15643 std::fs::write(
15644 &f.reply,
15645 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
15646 )
15647 .unwrap();
15648 let verified = posted(&mut f).await;
15649 assert!(verified.body.contains("now verified"));
15650 assert_ne!(
15651 f.runner.state.change_summary.as_ref().unwrap().evidence,
15652 original
15653 );
15654 let mut round = round_with_notes(1, Some("内部報告"));
15655 round.reviews.push(ReviewRecord {
15656 attempts: 0,
15657 reviewer: 1,
15658 agent: "alpha".into(),
15659 summary: "内部レビュー".into(),
15660 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
15661 vote: None,
15662 failed: None,
15663 duration_ms: 0,
15664 });
15665 round
15666 .fix
15667 .as_mut()
15668 .unwrap()
15669 .rejected
15670 .push(crate::verdict::Rejection {
15671 id: "R1-1-1".into(),
15672 why: "呼び出し元から到達できない".into(),
15673 });
15674 f.runner.state.reviews = vec![round];
15675 let omitted = posted(&mut f).await;
15677 assert!(omitted.body.contains("No description"));
15678 assert!(omitted.body.contains("R1-1-1"));
15679 assert!(!omitted.body.contains("到達"));
15680 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
15681 let described = posted(&mut f).await;
15682 assert!(described.body.contains("unreachable from callers"));
15683 assert!(described.body.contains("R1-1-1"));
15684 assert!(!described.body.contains("到達"));
15685 let evidence = f
15686 .runner
15687 .state
15688 .change_summary
15689 .as_ref()
15690 .unwrap()
15691 .evidence
15692 .clone();
15693 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
15694 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
15695 assert!(posted(&mut f).await.body.contains("rationale updated"));
15696 assert_ne!(
15697 f.runner.state.change_summary.as_ref().unwrap().evidence,
15698 evidence
15699 );
15700 f.runner
15702 .state
15703 .change_summary
15704 .as_mut()
15705 .unwrap()
15706 .evidence
15707 .clear();
15708 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
15709 assert!(posted(&mut f).await.body.contains("current specification"));
15710 }
15711
15712 #[tokio::test]
15713 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
15714 let mut f = summary_fixture();
15715 let m = posted(&mut f).await;
15716 assert_eq!(m.title, "feat(own): add the own file");
15717 assert!(
15718 m.body
15719 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
15720 "{}",
15721 m.body
15722 );
15723 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
15724 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
15725 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15726 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15727 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
15728 let kept = f.runner.state.change_summary.as_ref().unwrap();
15729 assert_eq!(
15730 (kept.head.as_str(), kept.base.as_str()),
15731 (head.as_str(), base.as_str())
15732 );
15733 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
15734 }
15735
15736 #[tokio::test]
15737 async fn review_only_summary_receives_the_configured_github_language() {
15738 let mut f = summary_fixture();
15739 f.runner.state.config.graph.github_language = "ja".into();
15740 let captured = f.repo.join("summary-prompt.txt");
15741 let author = &mut f.runner.state.config.agents[0];
15742 author.command = vec![
15743 "sh".into(),
15744 "-c".into(),
15745 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
15746 ];
15747 author.env.insert(
15748 "PROMPT_FILE".into(),
15749 captured.to_string_lossy().into_owned(),
15750 );
15751 std::fs::write(
15752 &f.reply,
15753 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
15754 )
15755 .unwrap();
15756 let m = posted(&mut f).await;
15757 let prompt = std::fs::read_to_string(captured).unwrap();
15758 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
15759 assert!(!prompt.contains("in English"));
15760 assert_eq!(m.title, "feat(own): ファイルを追加");
15761 assert!(m.body.contains("own.txt を追加します。"));
15762 }
15763
15764 #[tokio::test]
15765 async fn review_fixes_come_after_the_whole_change_summary() {
15766 let mut f = summary_fixture();
15767 f.runner.state.reviews = vec![
15768 round_with_notes(1, Some("handled the PATH case")),
15769 round_with_notes(2, None),
15770 ];
15771 let m = posted(&mut f).await;
15772 let summary = m.body.find("## Summary").expect("summary");
15773 let fixes = m.body.find("## Review rounds").expect("review rounds");
15774 assert!(summary < fixes, "{}", m.body);
15775 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
15776 assert!(m.body.contains("## Final verification"), "{}", m.body);
15777 }
15778
15779 #[tokio::test]
15780 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
15781 let mut f = summary_fixture();
15782 let first = posted(&mut f).await;
15783 let first_head = f.runner.state.change_summary.clone().unwrap().head;
15784 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
15786 let again = posted(&mut f).await;
15787 assert_eq!((again.title, again.body), (first.title, first.body));
15788
15789 let g = |args: &[&str]| {
15790 let out = std::process::Command::new("git")
15791 .args(args)
15792 .current_dir(&f.repo)
15793 .quiet()
15794 .output()
15795 .expect("spawn git");
15796 assert!(out.status.success(), "git {args:?}");
15797 };
15798 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
15799 g(&["add", "-A"]);
15800 g(&["commit", "-q", "-m", "fix(own): handle more"]);
15801 let second = posted(&mut f).await;
15802 let kept = f.runner.state.change_summary.clone().unwrap();
15803 assert_ne!(kept.head, first_head);
15804 assert!(second.body.contains("Not read yet."), "{}", second.body);
15805 assert!(!second.body.contains("so the first summary is read"));
15806 assert_eq!(second.title, "feat(own): other");
15807 }
15808
15809 #[tokio::test]
15810 async fn an_unavailable_summary_is_stated_and_never_invented() {
15811 let mut f = summary_fixture();
15812 std::fs::write(&f.reply, "no title line here").unwrap();
15813 let m = posted(&mut f).await;
15814 assert!(
15815 m.body
15816 .contains("No description of what this change does could be generated"),
15817 "{}",
15818 m.body
15819 );
15820 assert!(m.body.contains("internal reports remain in local records"));
15821 assert!(f.runner.state.change_summary.is_none());
15822 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
15823 let winner = f.runner.state.candidates[0].clone();
15825 let preview = f
15826 .runner
15827 .guarded_pr_message(&winner, None, false)
15828 .await
15829 .unwrap()
15830 .unwrap();
15831 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
15832 }
15833
15834 #[test]
15835 fn pr_message_review_single_commit_uses_its_subject() {
15836 let state = review_state(&["feat(nats): per-role user"]);
15837 let m = pr_message(&state, 'A');
15838 assert_eq!(m.title, "feat(nats): per-role user");
15839 assert!(!m.body.contains("Review the work already"), "{}", m.body);
15840 assert!(m.body.contains("## Commits under review"), "{}", m.body);
15841 }
15842
15843 #[test]
15844 fn pr_message_review_multi_commit_takes_the_oldest() {
15845 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
15846 let m = pr_message(&state, 'A');
15847 assert_eq!(m.title, "feat: the change");
15848 for s in ["feat: the change", "fix: typo", "fix: again"] {
15849 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
15850 }
15851 }
15852
15853 #[test]
15854 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
15855 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
15856 let mut state = review_state(&[first, "fix: later fixup"]);
15857 state.candidates[0].branch = "機能/ログイン".to_owned();
15858 let m = pr_message(&state, 'A');
15859 assert!(
15860 m.title.starts_with("chore: land candidate A of run"),
15861 "{}",
15862 m.title
15863 );
15864 }
15865 }
15866
15867 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
15868 BranchFacts {
15869 commits: commits
15870 .iter()
15871 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
15872 .collect(),
15873 stat: stat.to_owned(),
15874 }
15875 }
15876
15877 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
15878 let mut r = review_round(true, 0, 1, 1, true, true);
15879 r.round = round;
15880 r.fix = notes.map(|n| FixRecord {
15881 agent: "fixer".to_owned(),
15882 addressed: Vec::new(),
15883 rejected: Vec::new(),
15884 notes: n.to_owned(),
15885 committed: true,
15886 failed: None,
15887 duration_ms: 0,
15888 continuation: None,
15889 });
15890 r
15891 }
15892
15893 #[test]
15894 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
15895 let state = review_state(&["ignored"]);
15896 let f = facts(
15897 &[
15898 (
15899 "fix(login): resolve PATH on macOS",
15900 "Login shells skip rc files.",
15901 ),
15902 ("fix: address review", ""),
15903 ],
15904 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
15905 );
15906 let m = pr_message_with(&state, 'A', Some(&f));
15907 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
15908 assert!(
15909 m.body
15910 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
15911 "{}",
15912 m.body
15913 );
15914 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
15915 assert!(m.body.contains("## Diff stat"), "{}", m.body);
15916 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
15917 for banned in [
15918 "Review the work already",
15919 "no task statement",
15920 "Original task",
15921 ] {
15922 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
15923 }
15924 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
15925 }
15926
15927 #[test]
15928 fn pr_message_review_truncates_a_huge_first_commit_body() {
15929 let state = review_state(&["ignored"]);
15930 let f = facts(
15931 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
15932 "s",
15933 );
15934 let m = pr_message_with(&state, 'A', Some(&f));
15935 assert!(m.body.len() < 30_000, "{}", m.body.len());
15936 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
15937 assert!(
15938 m.body.contains("- ... 1 more commit(s)"),
15939 "{}",
15940 m.body.len()
15941 );
15942 assert!(m.body.ends_with("magi:candidate-a\n"));
15943 }
15944
15945 #[test]
15946 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
15947 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
15948 assert_eq!(m.title, "feat: x");
15949 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
15950 assert!(!m.body.contains("Diff stat"), "{}", m.body);
15951 assert!(!m.body.contains("no task statement"), "{}", m.body);
15952 }
15953
15954 #[test]
15955 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
15956 let mut state = review_state(&["日本語の件名"]);
15957 state.candidates[0].branch = "fix/macos-login-path".to_owned();
15958 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
15959 state.candidates[0].branch = "機能/ログイン".to_owned();
15960 assert!(
15961 pr_message(&state, 'A')
15962 .title
15963 .starts_with("chore: land candidate A")
15964 );
15965 }
15966
15967 #[test]
15968 fn pr_message_review_fixes_survive_a_clean_final_round() {
15969 let mut state = review_state(&["feat: x"]);
15970 state.reviews = vec![
15971 round_with_notes(1, Some("handled the PATH case")),
15972 round_with_notes(2, None),
15973 ];
15974 let body = pr_message(&state, 'A').body;
15975 assert!(
15976 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
15977 "{body}"
15978 );
15979 assert!(body.contains("### Round 1"), "{body}");
15980
15981 state.reviews = vec![
15982 round_with_notes(1, Some("first fix")),
15983 round_with_notes(2, Some("")),
15984 round_with_notes(3, Some("second fix")),
15985 round_with_notes(4, None),
15986 ];
15987 let body = pr_message(&state, 'A').body;
15988 assert!(
15989 body.contains("### Round 1") && !body.contains("first fix"),
15990 "{body}"
15991 );
15992 assert!(
15993 body.contains("### Round 3") && !body.contains("second fix"),
15994 "{body}"
15995 );
15996 assert!(body.contains("### Round 2"), "{body}");
15997 }
15998
15999 const FINAL: &str = "abcdef1234567890";
16000
16001 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16002 let mut r = round_with_notes(round, None);
16003 r.verified_head = verified.map(str::to_owned);
16004 r
16005 }
16006
16007 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16008 state.gate = vec![CommandOutcome {
16009 command: "gate".to_owned(),
16010 code: Some(0),
16011 output_tail: String::new(),
16012 duration_ms: 0,
16013 resource_blocked: false,
16014 }];
16015 state.gate_ran = true;
16016 state.gate_head = head.map(str::to_owned);
16017 }
16018
16019 fn body_at(state: &RunState) -> String {
16020 pr_message_raw(
16021 state,
16022 'A',
16023 None,
16024 crate::pr_outcome::FinalHead::At(FINAL),
16025 ChangeNote::NotAsked,
16026 )
16027 .body
16028 }
16029
16030 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16031
16032 #[test]
16033 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16034 let mut state = review_state(&["feat: x"]);
16035 state.reviews = vec![
16036 round_with_notes(1, Some(SANDBOXED)),
16037 passing_round(2, Some(FINAL)),
16038 ];
16039 ok_gate(&mut state, Some(FINAL));
16040 let body = body_at(&state);
16041 assert!(!body.contains("Handled the PATH case."), "{body}");
16042 assert!(!body.contains("could not commit"), "{body}");
16043 assert!(!body.contains("could not run"), "{body}");
16044 assert!(
16045 body.contains("The fix is committed on the branch"),
16046 "{body}"
16047 );
16048 assert!(!body.contains("sandbox"), "{body}");
16049 assert!(
16050 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16051 "{body}"
16052 );
16053 assert!(
16054 body.contains("Gate (`verify.gate`): passed on the final commit"),
16055 "{body}"
16056 );
16057 assert!(
16059 state.reviews[0]
16060 .fix
16061 .as_ref()
16062 .unwrap()
16063 .notes
16064 .contains("could not commit")
16065 );
16066 }
16067
16068 #[test]
16069 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16070 let mut state = review_state(&["feat: x"]);
16071 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16072 r1.fix.as_mut().unwrap().committed = false;
16073 let mut r2 = passing_round(2, Some(FINAL));
16074 r2.e2e[0].code = Some(1);
16075 state.reviews = vec![r1, r2];
16076 let body = body_at(&state);
16077 assert!(!body.contains("could not commit"), "{body}");
16078 assert!(!body.contains("could not run"), "{body}");
16079 assert!(body.contains("This round produced no commit"), "{body}");
16080 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16081 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16082 assert!(!body.contains("passed"), "{body}");
16083 }
16084
16085 #[test]
16086 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16087 let mut state = review_state(&["feat: x"]);
16088 state.reviews = vec![
16089 round_with_notes(1, Some(SANDBOXED)),
16090 passing_round(2, Some("0123456789abcdef")),
16091 ];
16092 ok_gate(&mut state, Some("0123456789abcdef"));
16093 let body = body_at(&state);
16094 assert!(!body.contains("HTTP tests could not run"), "{body}");
16095 assert!(!body.contains("could not commit"), "{body}");
16097 assert!(
16098 body.contains("passed on an earlier commit (`0123456`)"),
16099 "{body}"
16100 );
16101 assert!(!body.contains("passed on the final commit"), "{body}");
16102
16103 ok_gate(&mut state, None);
16105 let body = body_at(&state);
16106 assert!(
16107 body.contains("passed on a commit that was not recorded"),
16108 "{body}"
16109 );
16110 }
16111
16112 #[test]
16113 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16114 let state = state_with_summary(
16115 "add retries to the client",
16116 "TITLE: feat: retries\n\nDid it.",
16117 );
16118 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16119 assert_eq!(m.title, "feat: retries");
16120 assert!(
16121 m.body.contains("<summary>Original task</summary>"),
16122 "{}",
16123 m.body
16124 );
16125 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16126 assert!(
16127 m.body
16128 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16129 "{}",
16130 m.body
16131 );
16132 }
16133
16134 #[test]
16135 fn pr_message_review_bounds_a_long_english_subject() {
16136 let long = format!("feat: {}", "word ".repeat(100));
16137 let m = pr_message(&review_state(&[&long]), 'A');
16138 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16139 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16140 }
16141
16142 #[test]
16143 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16144 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16145 let m = pr_message(&state, 'A');
16146 assert_eq!(m.title, "add retries");
16147 assert!(m.body.contains("<summary>Original task</summary>"));
16148 assert!(!m.body.contains("Commits under review"));
16149 assert_eq!(landing_subject_source(&state), state.instruction);
16150 }
16151
16152 #[test]
16153 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16154 let state = review_state(&["feat: the change", "fix: typo"]);
16155 let source = landing_subject_source(&state);
16156 assert_eq!(land::merge_subject("", &source), "feat: the change");
16157 assert_eq!(
16158 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16159 "feat: the change"
16160 );
16161 assert_eq!(
16163 land::merge_subject("feat: renamed by hand", &source),
16164 "feat: renamed by hand"
16165 );
16166 let blank = review_state(&["日本語"]);
16167 assert!(
16168 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16169 );
16170 }
16171
16172 #[test]
16173 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16174 let state = review_state(&["feat: the change"]);
16175 let source = landing_subject_source(&state);
16176 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16177 assert_eq!(
16178 land::merge_subject(landing_title(&state, old), &source),
16179 "feat: the change"
16180 );
16181 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16182 let task = state_with_summary("add retries", "");
16183 assert_eq!(landing_title(&task, old), old);
16184 }
16185
16186 #[test]
16187 fn pr_message_describes_the_change_not_the_task() {
16188 let state = state_with_summary(
16189 "今回やってほしいこと: results projector を直す",
16190 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
16191 );
16192 let m = pr_message(&state, 'A');
16193 assert_eq!(m.title, "fix(web): batch the runs list reads");
16194 assert!(
16195 m.body.starts_with("## Summary\n\n- reads run.json once"),
16196 "{}",
16197 m.body
16198 );
16199 assert!(!m.body.contains("TITLE:"), "{}", m.body);
16200 let task_at = m.body.find("今回やってほしいこと").unwrap();
16201 let details_at = m.body.find("<details>").unwrap();
16202 assert!(
16203 details_at < task_at,
16204 "the task lives inside <details>: {}",
16205 m.body
16206 );
16207 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
16208 assert!(m.body.contains("magi:candidate-a"));
16209 }
16210
16211 #[test]
16212 fn pr_message_falls_back_to_the_task_without_a_title_line() {
16213 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
16214 let m = pr_message(&state, 'A');
16215 assert_eq!(m.title, "add retries");
16216 assert!(
16217 m.body.contains("## Summary\n\n- did some things"),
16218 "{}",
16219 m.body
16220 );
16221
16222 let none = RunState::new(
16223 PathBuf::from("/repo"),
16224 "main".to_owned(),
16225 "abc1234".to_owned(),
16226 "add retries".to_owned(),
16227 Config::default(),
16228 );
16229 let m = pr_message(&none, 'A');
16230 assert_eq!(m.title, "add retries");
16231 assert!(!m.body.contains("## Summary"), "{}", m.body);
16232 }
16233
16234 #[test]
16235 fn pr_message_refuses_the_candidate_commit_subject() {
16236 for bad in [
16237 "TITLE: magi: candidate A (uncommitted work)",
16238 "TITLE: chore: stuff (uncommitted work)",
16239 "TITLE: ",
16240 ] {
16241 let state = state_with_summary("add retries", bad);
16242 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
16243 }
16244 }
16245
16246 #[test]
16247 fn pr_message_bounds_a_very_long_task_and_title() {
16248 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
16249 let state = state_with_summary(&long, "- nothing");
16250 let m = pr_message(&state, 'A');
16251 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16252 assert!(!m.title.contains('\n'));
16253
16254 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
16255 let m = pr_message(&state, 'A');
16256 assert!(m.title.starts_with("feat: "));
16257 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16258 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
16259 }
16260
16261 fn long_title_of(instruction: &str) -> String {
16262 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
16263 }
16264
16265 #[test]
16266 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
16267 let first = "Make the landing path keep a readable title for long tasks";
16268 let line = format!(
16269 "{first}. {}",
16270 "And then keep going with more words ".repeat(20)
16271 );
16272 let t = long_title_of(&line);
16273 assert_eq!(t, first);
16274 assert!(!t.starts_with("chore: land"));
16275 }
16276
16277 #[test]
16278 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
16279 let line = "word ".repeat(200);
16280 let t = long_title_of(&line);
16281 assert!(t.ends_with("word..."), "{t}");
16282 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
16283 }
16284
16285 #[test]
16286 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
16287 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
16288 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
16289 }
16290 }
16291
16292 #[test]
16293 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
16294 let mut state = state_with_summary("再試行を追加する", "- nothing");
16295 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
16296 state.config.graph.github_language = "ja".into();
16297 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
16298 }
16299
16300 #[test]
16301 fn pr_message_title_limit_is_exact() {
16302 let at = "a".repeat(PR_TITLE_MAX);
16303 assert_eq!(long_title_of(&at), at);
16304 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
16305 assert!(over.ends_with("..."), "{over}");
16306 assert_eq!(over.chars().count(), PR_TITLE_MAX);
16307 }
16308
16309 #[test]
16310 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
16311 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
16312 let t = long_title_of(&line);
16313 assert!(t.ends_with("..."), "{t}");
16314 assert!(t.is_ascii(), "{t}");
16315 }
16316
16317 #[test]
16318 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
16319 let line = format!(
16320 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
16321 "plus more filler words ".repeat(20)
16322 );
16323 let t = long_title_of(&line);
16324 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
16325 }
16326
16327 #[test]
16328 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
16329 let line = format!("\n\n# {}", "title words ".repeat(40));
16330 let state = state_with_summary(&line, "- nothing");
16331 let m = pr_message(&state, 'A');
16332 assert!(m.title.starts_with("title words"), "{}", m.title);
16333 assert_eq!(
16334 land::merge_subject(&m.title, &landing_subject_source(&state)),
16335 m.title
16336 );
16337 assert_eq!(
16339 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
16340 "feat: renamed by hand"
16341 );
16342 }
16343
16344 #[tokio::test]
16345 async fn github_text_rewrite_is_bounded_and_falls_back() {
16346 crate::run::pin_test_home();
16347 for (reply, accepted) in [
16348 (
16349 "TITLE: fix: retries\nAdd retries for failed requests.",
16350 true,
16351 ),
16352 (
16353 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
16354 false,
16355 ),
16356 (
16357 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
16358 true,
16359 ),
16360 (
16361 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
16362 true,
16363 ),
16364 (
16365 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
16366 true,
16367 ),
16368 (
16369 "TITLE: fix: retries\n日本語の説明です。 token=secret",
16370 false,
16371 ),
16372 ("TITLE: fix: retries\n", false),
16373 ] {
16374 let dir = tempfile::tempdir().unwrap();
16375 let mut runner = runner_at(RunStatus::Gating);
16376 runner.state = state_with_summary(
16377 "add retries",
16378 "TITLE: fix: retries\n日本語の説明を書きます。",
16379 );
16380 runner.state.repo = dir.path().to_owned();
16381 runner.state.candidates[0].worktree = dir.path().to_owned();
16382 let mut author = spec("alpha");
16383 author.command = vec![
16384 "sh".into(),
16385 "-c".into(),
16386 "printf '%s' \"$REWRITE_REPLY\"".into(),
16387 ];
16388 author.env.insert("REWRITE_REPLY".into(), reply.into());
16389 runner.state.config.agents = vec![author];
16390 seed_public_summary(&mut runner);
16391 let winner = runner.state.candidates[0].clone();
16392 let first = runner
16393 .guarded_pr_message(&winner, None, true)
16394 .await
16395 .unwrap();
16396 let message = if accepted {
16397 first.expect("a passing rewrite never asks")
16398 } else {
16399 assert!(first.is_none());
16401 assert!(runner.state.parked);
16402 let store = ask::Questions::open();
16403 let asked: Vec<_> = store
16404 .list()
16405 .into_iter()
16406 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16407 .collect();
16408 assert_eq!(asked.len(), 1);
16409 assert!(!asked[0].detail.contains("secret"));
16410 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
16411 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
16412 assert_eq!(
16413 asked[0]
16414 .choices
16415 .contains(&crate::github_text::USE_REWRITE.to_owned()),
16416 rewrite.is_some()
16417 );
16418 store
16421 .update(&asked[0].id, |q| {
16422 q.abandon("test");
16423 Ok(())
16424 })
16425 .unwrap();
16426 let again = runner
16427 .guarded_pr_message(&winner, None, true)
16428 .await
16429 .unwrap()
16430 .expect("an abandoned question falls back");
16431 let n = store
16432 .list()
16433 .into_iter()
16434 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16435 .count();
16436 assert_eq!(n, 1, "asked once per rejected text");
16437 again
16438 };
16439 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
16440 assert!(!message.body.contains("secret"));
16441 assert!(!message.body.contains("prefix"));
16442 assert!(!message.title.contains("secret"));
16443 assert_eq!(
16444 message.body.contains("Add retries for failed requests."),
16445 accepted
16446 );
16447 assert_eq!(
16448 runner.state.seats["github-text-rewrite"].turns, 1,
16449 "only one rewrite invocation"
16450 );
16451 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
16452 if accepted {
16453 assert!(runner.state.github_text.is_none());
16454 assert!(!runner.state.parked);
16455 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
16456 assert!(message.body.contains("## Final verification"));
16457 assert!(message.body.contains("Evidence: final diff"));
16458 assert!(
16459 message
16460 .body
16461 .contains(&format!("magi:run/{}", runner.state.id))
16462 );
16463 }
16464 if !accepted {
16465 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
16466 assert!(
16467 message
16468 .body
16469 .contains(&format!("magi:run/{}", runner.state.id))
16470 );
16471 }
16472 }
16473 }
16474
16475 fn scripted(id: &str, reply: &str) -> AgentSpec {
16476 let mut a = spec(id);
16477 a.command = vec![
16478 "sh".into(),
16479 "-c".into(),
16480 "printf '%s' \"$REWRITE_REPLY\"".into(),
16481 ];
16482 a.env.insert("REWRITE_REPLY".into(), reply.into());
16483 a
16484 }
16485
16486 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
16487 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
16488
16489 fn seed_public_summary(runner: &mut Runner) {
16491 let repo = &runner.state.repo;
16492 let g = |args: &[&str]| {
16493 let out = std::process::Command::new("git")
16494 .args(args)
16495 .current_dir(repo)
16496 .quiet()
16497 .output()
16498 .unwrap();
16499 assert!(out.status.success(), "git {args:?}");
16500 String::from_utf8_lossy(&out.stdout).trim().to_owned()
16501 };
16502 g(&["init", "-q", "-b", "main"]);
16503 g(&[
16504 "-c",
16505 "user.name=Test",
16506 "-c",
16507 "user.email=test@example.invalid",
16508 "commit",
16509 "-q",
16510 "--allow-empty",
16511 "-m",
16512 "feat: fixture",
16513 ]);
16514 let head = g(&["rev-parse", "HEAD"]);
16515 g(&["update-ref", "refs/remotes/origin/main", &head]);
16516 let evidence = crate::pr_outcome::summary_evidence(
16517 &runner.state,
16518 crate::pr_outcome::FinalHead::At(&head),
16519 );
16520 let generated = &runner.state.candidates[0].summary;
16521 runner.state.change_summary = Some(crate::run::ChangeSummary {
16522 base: head.clone(),
16523 head,
16524 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
16525 title: summary_title(generated).unwrap(),
16526 body: summary_without_title(generated),
16527 });
16528 }
16529
16530 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
16531 let mut runner = runner_at(RunStatus::Gating);
16532 runner.state = state_with_summary("add retries", JA_SUMMARY);
16533 if review_only {
16534 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16535 }
16536 runner.state.repo = dir.to_owned();
16537 runner.state.candidates[0].worktree = dir.to_owned();
16538 runner.state.config.agents = agents;
16539 seed_public_summary(&mut runner);
16540 runner
16541 }
16542
16543 fn gate_questions(runner: &Runner) -> usize {
16544 ask::Questions::open()
16545 .list()
16546 .into_iter()
16547 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16548 .count()
16549 }
16550
16551 #[tokio::test]
16552 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
16553 crate::run::pin_test_home();
16554 let dir = tempfile::tempdir().unwrap();
16555 let mut runner = rewrite_runner(
16556 dir.path(),
16557 true,
16558 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
16559 );
16560 runner.state.config.roles.language_judge =
16561 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
16562 let winner = runner.state.candidates[0].clone();
16563 let msg = runner
16564 .guarded_pr_message(&winner, None, true)
16565 .await
16566 .unwrap()
16567 .expect("an English rewrite posts without asking");
16568 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
16569 assert!(msg.body.contains("<details>kept</details>"));
16570 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
16571 assert_eq!(gate_questions(&runner), 0);
16572 assert!(runner.state.github_text.is_none());
16573 assert!(!runner.state.seats.contains_key("impl-A"));
16574 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16575 }
16576
16577 #[tokio::test]
16578 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
16579 crate::run::pin_test_home();
16580 let dir = tempfile::tempdir().unwrap();
16581 let mut runner = rewrite_runner(
16582 dir.path(),
16583 false,
16584 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
16585 );
16586 runner.state.config.roles.language_judge =
16587 Some(crate::config::AgentChoice::One("beta".into()));
16588 runner.state.seats.insert(
16589 "impl-A".into(),
16590 SeatState::new("impl-A", "alpha", runner.state.seed),
16591 );
16592 let winner = runner.state.candidates[0].clone();
16593 let msg = runner
16594 .guarded_pr_message(&winner, None, true)
16595 .await
16596 .unwrap()
16597 .expect("rewritten");
16598 assert!(msg.body.contains("<details>kept</details>"));
16599 assert_eq!(runner.state.seats["impl-A"].turns, 0);
16600 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16601 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
16602 }
16603
16604 #[tokio::test]
16605 async fn rewrite_reuses_a_resumable_implementer_session() {
16606 crate::run::pin_test_home();
16607 let dir = tempfile::tempdir().unwrap();
16608 let mut runner = rewrite_runner(
16609 dir.path(),
16610 false,
16611 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
16612 );
16613 runner.state.config.roles.language_judge =
16614 Some(crate::config::AgentChoice::One("beta".into()));
16615 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
16616 seat.turns = 1;
16617 runner.state.seats.insert("impl-A".into(), seat);
16618 let winner = runner.state.candidates[0].clone();
16619 runner
16620 .guarded_pr_message(&winner, None, true)
16621 .await
16622 .unwrap()
16623 .expect("rewritten");
16624 assert_eq!(runner.state.seats["impl-A"].turns, 2);
16625 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
16626 }
16627
16628 #[tokio::test]
16629 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
16630 crate::run::pin_test_home();
16631 let dir = tempfile::tempdir().unwrap();
16632 let mut broken = spec("bad");
16633 broken.command = vec!["false".into()];
16634 let mut runner = rewrite_runner(
16635 dir.path(),
16636 true,
16637 vec![
16638 broken,
16639 scripted("noformat", "Add retries."),
16640 scripted("good", EN_REWRITE),
16641 ],
16642 );
16643 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
16644 "bad".into(),
16645 "noformat".into(),
16646 "good".into(),
16647 ]));
16648 let winner = runner.state.candidates[0].clone();
16649 let msg = runner
16650 .guarded_pr_message(&winner, None, true)
16651 .await
16652 .unwrap()
16653 .expect("the third agent rewrites");
16654 assert!(msg.body.contains("`src/net.rs`"));
16655 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
16656 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
16657 }
16658
16659 #[tokio::test]
16660 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
16661 crate::run::pin_test_home();
16662 for (agents, role, saved) in [
16664 (
16665 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
16666 Some("beta"),
16667 true,
16668 ),
16669 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
16670 (Vec::new(), None, false),
16671 ] {
16672 let dir = tempfile::tempdir().unwrap();
16673 let mut runner = rewrite_runner(dir.path(), true, agents);
16674 runner.state.config.roles.language_judge =
16675 role.map(|r| crate::config::AgentChoice::One(r.into()));
16676 let winner = runner.state.candidates[0].clone();
16677 let first = runner
16678 .guarded_pr_message(&winner, None, true)
16679 .await
16680 .unwrap();
16681 assert!(first.is_none(), "withheld text parks the run");
16682 assert!(runner.state.parked);
16683 assert_eq!(gate_questions(&runner), 1);
16684 let g = runner.state.github_text.as_ref().unwrap();
16685 assert_eq!(g.rewrite.is_some(), saved);
16686 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
16687 }
16688 }
16689
16690 #[tokio::test]
16691 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
16692 crate::run::pin_test_home();
16693 let dir = tempfile::tempdir().unwrap();
16694 let mut runner = runner_at(RunStatus::Gating);
16695 runner.state =
16696 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
16697 runner.state.repo = dir.path().to_owned();
16698 runner.state.candidates[0].worktree = dir.path().to_owned();
16699 let mut author = spec("alpha");
16700 author.command = vec![
16701 "sh".into(),
16702 "-c".into(),
16703 "printf '%s' \"$REWRITE_REPLY\"".into(),
16704 ];
16705 author.env.insert(
16706 "REWRITE_REPLY".into(),
16707 format!(
16708 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
16709 "English sentence. ".repeat(600)
16710 ),
16711 );
16712 runner.state.config.agents = vec![author];
16713 seed_public_summary(&mut runner);
16714 let winner = runner.state.candidates[0].clone();
16715 assert!(
16716 runner
16717 .guarded_pr_message(&winner, None, true)
16718 .await
16719 .unwrap()
16720 .is_none()
16721 );
16722 let record = runner.state.github_text.clone().unwrap();
16723 let rewrite = record.rewrite.clone().unwrap();
16724 assert_eq!(rewrite.failed, ["body-language"]);
16725 assert!(
16726 rewrite
16727 .body
16728 .ends_with(&format!("magi:run/{}", runner.state.id))
16729 );
16730 let store = ask::Questions::open();
16731 let q = store.get(record.question.as_ref().unwrap()).unwrap();
16732 assert!(
16733 q.choices
16734 .contains(&crate::github_text::USE_REWRITE.to_owned())
16735 );
16736 for label in [
16737 "ORIGINAL",
16738 "FALLBACK",
16739 "REWRITE",
16740 "Rules still failed: body-language",
16741 "truncated",
16742 ] {
16743 assert!(q.detail.contains(label), "{label}");
16744 }
16745 let artifact = std::fs::read_to_string(crate::run::artifact_path(
16746 &runner.state,
16747 &crate::github_text::artifact_name(&record.fingerprint),
16748 ))
16749 .unwrap();
16750 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
16751 runner.state =
16753 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
16754 store
16755 .update(&q.id, |q| {
16756 q.say("まだ日本語")?;
16757 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
16758 })
16759 .unwrap();
16760 assert!(
16761 runner
16762 .guarded_pr_message(&winner, None, true)
16763 .await
16764 .unwrap()
16765 .is_none()
16766 );
16767 let retry = runner.state.github_text.as_ref().unwrap();
16768 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
16769 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
16770 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
16771 assert!(
16772 q.posting_comparison
16773 .as_ref()
16774 .unwrap()
16775 .matches_detail(&q.detail)
16776 );
16777 store
16778 .update(&q.id, |q| {
16779 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
16780 })
16781 .unwrap();
16782 let message = runner
16783 .guarded_pr_message(&winner, None, true)
16784 .await
16785 .unwrap()
16786 .unwrap();
16787 assert_eq!(message.title, rewrite.title);
16788 assert_eq!(message.body, rewrite.body);
16789 assert_eq!(
16790 runner.state.github_text.as_ref().unwrap().choice,
16791 Some(GithubTextChoice::Rewrite)
16792 );
16793 let again = runner
16794 .guarded_pr_message(&winner, None, true)
16795 .await
16796 .unwrap()
16797 .unwrap();
16798 assert_eq!(again.title, message.title);
16799 assert_eq!(again.body, message.body);
16800 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16801 assert!(
16802 !runner
16803 .state
16804 .events
16805 .iter()
16806 .any(|e| e.message.contains(&rewrite.title)
16807 || e.message.contains("English sentence."))
16808 );
16809 }
16810
16811 #[tokio::test]
16812 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
16813 crate::run::pin_test_home();
16814 let dir = tempfile::tempdir().unwrap();
16815 let mut runner = runner_at(RunStatus::Gating);
16816 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
16817 runner.state.repo = dir.path().to_owned();
16818 runner.state.candidates[0].worktree = dir.path().to_owned();
16819 seed_public_summary(&mut runner);
16820 let winner = runner.state.candidates[0].clone();
16821 assert!(
16822 runner
16823 .guarded_pr_message(&winner, None, true)
16824 .await
16825 .unwrap()
16826 .is_none()
16827 );
16828 let record = runner.state.github_text.as_ref().unwrap();
16829 assert!(record.rewrite.is_none());
16830 let mut old = serde_json::to_value(record).unwrap();
16832 old.as_object_mut().unwrap().remove("rewrite");
16833 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
16834 assert!(old.rewrite.is_none());
16835 let id = record.question.clone().unwrap();
16836 ask::Questions::open()
16837 .update(&id, |q| {
16838 q.choices.push(crate::github_text::USE_REWRITE.into());
16839 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
16840 })
16841 .unwrap();
16842 let message = runner
16843 .guarded_pr_message(&winner, None, true)
16844 .await
16845 .unwrap()
16846 .unwrap();
16847 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
16848 assert_eq!(
16849 runner.state.github_text.as_ref().unwrap().choice,
16850 Some(GithubTextChoice::Fallback)
16851 );
16852 }
16853
16854 #[tokio::test]
16855 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
16856 crate::run::pin_test_home();
16857 let dir = tempfile::tempdir().unwrap();
16858 let mut runner = runner_at(RunStatus::Gating);
16859 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
16860 runner.state.repo = dir.path().to_owned();
16861 runner.state.candidates[0].worktree = dir.path().to_owned();
16862 seed_public_summary(&mut runner);
16863 let winner = runner.state.candidates[0].clone();
16864 assert!(
16865 runner
16866 .guarded_pr_message(&winner, None, true)
16867 .await
16868 .unwrap()
16869 .is_none()
16870 );
16871 let store = ask::Questions::open();
16872 let find = |run: &str| {
16873 store
16874 .list()
16875 .into_iter()
16876 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
16877 .unwrap()
16878 };
16879 let run = runner.state.id.clone();
16880 let q = find(&run);
16881 let fp = runner
16882 .state
16883 .github_text
16884 .as_ref()
16885 .unwrap()
16886 .fingerprint
16887 .clone();
16888 let name = crate::github_text::artifact_name(&fp);
16889 let path = crate::run::artifact_path(&runner.state, &name);
16890 assert!(q.detail.contains(&path.display().to_string()));
16891 assert!(q.detail.contains("title-language: title"));
16892 let saved = std::fs::read_to_string(&path).expect("artifact written");
16893 assert!(saved.contains("日本語のタイトル"));
16894 assert!(q.cwd.is_none());
16895 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
16896 let say = |id: &str, text: &str, choice: &str| {
16898 store
16899 .update(id, |q| {
16900 q.thread.push(ask::Turn {
16901 who: ask::Who::Operator,
16902 body: text.to_owned(),
16903 at: jiff::Timestamp::now(),
16904 note: None,
16905 });
16906 q.answer(ask::Answer::Choice(choice.to_owned()))
16907 })
16908 .unwrap();
16909 };
16910 say(&q.id, "まだ日本語", "use my text");
16911 assert!(
16912 runner
16913 .guarded_pr_message(&winner, None, true)
16914 .await
16915 .unwrap()
16916 .is_none()
16917 );
16918 let second = find(&run);
16919 assert_ne!(second.id, q.id);
16920 assert!(second.detail.contains("did not pass"));
16921 assert!(second.detail.contains("日本語のタイトル"));
16924 assert!(!second.detail.contains("まだ日本語"));
16925 assert!(second.detail.contains(&path.display().to_string()));
16926 say(&second.id, "fix: retry failed requests", "use my text");
16927 let message = runner
16928 .guarded_pr_message(&winner, None, true)
16929 .await
16930 .unwrap()
16931 .expect("a vetted replacement is posted");
16932 assert_eq!(message.title, "fix: retry failed requests");
16933 assert_eq!(
16934 runner
16935 .state
16936 .github_text
16937 .as_ref()
16938 .unwrap()
16939 .chosen_title
16940 .as_deref(),
16941 Some("fix: retry failed requests")
16942 );
16943 assert!(runner.state.github_text.as_ref().unwrap().resolved);
16944 let again = runner
16946 .guarded_pr_message(&winner, None, true)
16947 .await
16948 .unwrap()
16949 .unwrap();
16950 assert_eq!(again.title, message.title);
16951 assert_eq!(
16952 store
16953 .list()
16954 .into_iter()
16955 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
16956 .count(),
16957 2
16958 );
16959 }
16960
16961 #[test]
16962 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
16963 let mut state = state_with_summary(
16967 "add retries",
16968 "TITLE: fix(web): batch reads\n- reads run.json once",
16969 );
16970 state.config.graph.language = "ja".to_owned();
16971 let m = pr_message(&state, 'A');
16972 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
16973 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
16974
16975 let task = "今回やってほしいこと: results projector を直す";
16978 let mut state = state_with_summary(task, "- no title line");
16979 state.config.graph.language = "ja".to_owned();
16980 let m = pr_message(&state, 'A');
16981 assert_eq!(
16982 m.title,
16983 format!("chore: land candidate A of run {}", state.id)
16984 );
16985 assert!(
16986 m.body.contains(&format!(
16987 "<summary>Original task</summary>\n\n{task}\n\n</details>"
16988 )),
16989 "{}",
16990 m.body
16991 );
16992 }
16993
16994 #[test]
16995 fn pr_message_scrubs_home_paths_and_addresses() {
16996 let state = state_with_summary(
16997 "fix it in /Users/someone/src/x",
16998 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
16999 );
17000 let m = pr_message(&state, 'A');
17001 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17002 assert!(!m.body.contains(leak), "{}", m.body);
17003 }
17004 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17005 }
17006
17007 #[test]
17008 fn pr_message_survives_a_task_that_closes_details() {
17009 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17010 let m = pr_message(&state, 'A');
17011 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17012 }
17013
17014 #[test]
17015 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17016 let cmd = manual_merge_command(
17017 MergeStyle::Squash,
17018 Path::new("/repo"),
17019 "b",
17020 "fix: \"quoted\" $(x) `y`\n\nbody",
17021 );
17022 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17023 }
17024
17025 #[test]
17026 fn manual_merge_command_matches_the_configured_style() {
17027 let repo = Path::new("/repo");
17028 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17029
17030 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17031 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17032
17033 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17034 assert_eq!(
17035 squash,
17036 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17037 \"Merge magi run 0832 (candidate A)\""
17038 );
17039
17040 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17041 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17042 }
17043
17044 #[test]
17045 fn a_nudge_gets_a_quarter_of_the_budget() {
17046 assert_eq!(retry_budget(secs(1200), true), secs(300));
17048 assert_eq!(retry_budget(secs(3600), true), secs(900));
17049 }
17050
17051 #[test]
17052 fn a_resent_prompt_keeps_the_whole_budget() {
17053 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17056 assert_eq!(retry_budget(secs(60), false), secs(60));
17057 }
17058
17059 #[test]
17060 fn the_floor_never_exceeds_the_original_budget() {
17061 assert_eq!(retry_budget(secs(60), true), secs(60));
17065 assert_eq!(retry_budget(secs(480), true), secs(120));
17066 assert_eq!(retry_budget(secs(0), true), secs(0));
17067 }
17068
17069 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17070 agent::CommandEvidence {
17071 id: "item1".to_owned(),
17072 description: "cargo test".to_owned(),
17073 exit_code,
17074 result_summary: String::new(),
17075 source: "codex".to_owned(),
17076 }
17077 }
17078
17079 #[test]
17080 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17081 assert!(!has_unconfirmed_command(&[]));
17085 }
17086
17087 #[test]
17088 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17089 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17093 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17094 assert!(!has_unconfirmed_command(&[
17095 evidence(Some(0)),
17096 evidence(Some(101))
17097 ]));
17098 }
17099
17100 #[test]
17101 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17102 assert!(has_unconfirmed_command(&[
17103 evidence(Some(0)),
17104 evidence(None)
17105 ]));
17106 }
17107
17108 #[test]
17109 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17110 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17111 assert_eq!(
17112 verified_noop_claim(true, &[], text).as_deref(),
17113 Some("already fixed by b32cfc4, on main.")
17114 );
17115 }
17116
17117 #[test]
17118 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17119 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17122 assert!(verified_noop_claim(false, &[], text).is_none());
17123 }
17124
17125 #[test]
17126 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17127 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17128 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17129 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17131 }
17132
17133 #[test]
17134 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17135 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17136 }
17137
17138 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17141 runner.state.candidates = shape
17142 .iter()
17143 .enumerate()
17144 .map(|(i, &(empty, verified))| Candidate {
17145 index: i,
17146 label: (b'A' + i as u8) as char,
17147 agent: "sonnet".to_owned(),
17148 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17149 worktree: PathBuf::from(format!("/wt/{i}")),
17150 summary: String::new(),
17151 stat: String::new(),
17152 files: 0,
17153 commits: 0,
17154 empty,
17155 failed: None,
17156 verified_noop: verified.map(str::to_owned),
17157 duration_ms: 0,
17158 folded: false,
17159 })
17160 .collect();
17161 }
17162
17163 #[test]
17164 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17165 ask_test_home();
17166 let mut runner = runner_at(RunStatus::Implementing);
17167 set_candidates(
17168 &mut runner,
17169 &[
17170 (true, Some("already on main at b32cfc4")),
17171 (true, Some("same fix, see the existing test")),
17172 ],
17173 );
17174
17175 runner
17176 .after_implement()
17177 .expect("a verified no-op is not an error");
17178
17179 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17180 }
17181
17182 #[test]
17183 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17184 ask_test_home();
17185 let mut runner = runner_at(RunStatus::Implementing);
17186 set_candidates(
17190 &mut runner,
17191 &[(true, Some("already on main at b32cfc4")), (true, None)],
17192 );
17193
17194 let err = runner
17195 .after_implement()
17196 .expect_err("an unverified empty candidate must still fail the run");
17197
17198 assert!(
17199 err.to_string().contains("no candidate produced a change"),
17200 "{err}"
17201 );
17202 assert_eq!(runner.state.status, RunStatus::Failed);
17203 }
17204
17205 #[test]
17206 fn after_implement_still_fails_an_ordinary_all_empty_run() {
17207 ask_test_home();
17208 let mut runner = runner_at(RunStatus::Implementing);
17209 set_candidates(&mut runner, &[(true, None), (true, None)]);
17210
17211 let err = runner
17212 .after_implement()
17213 .expect_err("no candidate declared anything; this is an ordinary failure");
17214
17215 assert!(
17216 err.to_string().contains("no candidate produced a change"),
17217 "{err}"
17218 );
17219 assert_eq!(runner.state.status, RunStatus::Failed);
17220 }
17221}
17222
17223#[derive(Debug)]
17226struct GraftRestoreFailed(String);
17227
17228impl std::fmt::Display for GraftRestoreFailed {
17229 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
17230 write!(
17231 f,
17232 "could not restore the winner to its pre-graft commit: {}",
17233 self.0
17234 )
17235 }
17236}
17237
17238impl std::error::Error for GraftRestoreFailed {}
17239
17240#[cfg(test)]
17244mod resource_wait_tests {
17245 use super::*;
17246 use std::collections::BTreeMap;
17247
17248 fn sh(id: &str, script: &str) -> AgentSpec {
17249 AgentSpec {
17250 id: id.to_owned(),
17251 kind: crate::config::AgentKind::Command,
17252 model: None,
17253 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
17254 extra_args: Vec::new(),
17255 env: BTreeMap::new(),
17256 prompt_delivery: None,
17257 }
17258 }
17259
17260 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
17261 SeatJob {
17262 seat: SeatState::new("impl-A", &spec.id, 1),
17263 spec,
17264 cwd: dir.to_path_buf(),
17265 prompt: "do it".to_owned(),
17266 timeout,
17267 allow_write: true,
17268 sessions: false,
17269 artifacts: dir.join("art"),
17270 stem: "impl-A".to_owned(),
17271 handover: None,
17272 }
17273 }
17274
17275 fn state(resource_wait_secs: u64) -> RunState {
17276 let mut config = Config::default();
17277 config.graph.timeout_resource_wait = resource_wait_secs;
17278 RunState::new(
17279 PathBuf::from("/repo"),
17280 "main".to_owned(),
17281 "abc1234".to_owned(),
17282 "task".to_owned(),
17283 config,
17284 )
17285 }
17286
17287 fn hold(cache: &Path) -> crate::cache::Guard {
17288 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
17289 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
17290 crate::cache::AcquireOutcome::Acquired(g) => g,
17291 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
17292 }
17293 }
17294
17295 async fn run(
17296 jobs: Vec<SeatJob>,
17297 sem: Arc<Semaphore>,
17298 cache: Option<&Path>,
17299 st: &mut RunState,
17300 ) -> Vec<(usize, SeatState, AgentOutcome)> {
17301 let prompts = Prompts::default();
17302 let ctx = WaveCtx {
17303 run: "r",
17304 node: "implement",
17305 prompts: &prompts,
17306 cache,
17307 round: None,
17308 carry_seats: false,
17309 pauses: None,
17310 };
17311 wave(jobs, sem, &ctx, st, 0).await
17312 }
17313
17314 #[tokio::test]
17315 async fn verification_cache_wait_preserves_the_command_budget() {
17316 crate::run::pin_test_home();
17317 let tmp = tempfile::tempdir().unwrap();
17318 let cache = tmp.path().join("target");
17319 let guard = hold(&cache);
17320 let release = tokio::spawn(async move {
17321 tokio::time::sleep(Duration::from_millis(100)).await;
17322 guard.release();
17323 });
17324 let mut st = state(10);
17325 let budget = Duration::from_millis(50);
17326 let wait = repository_wait(&st, None);
17327 let (outcomes, _) = with_cache_lease(
17328 &mut st,
17329 Some(&cache),
17330 "gate",
17331 "gate",
17332 tmp.path(),
17333 "abc1234",
17334 budget,
17335 "test gate",
17336 &wait,
17337 |_, available| async move {
17338 assert_eq!(
17339 available, budget,
17340 "contention must not shorten verification"
17341 );
17342 (Vec::new(), false, Vec::new())
17343 },
17344 )
17345 .await
17346 .unwrap();
17347 release.await.unwrap();
17348 assert!(outcomes.is_empty(), "{outcomes:?}");
17349 assert!(
17350 st.events
17351 .iter()
17352 .any(|event| event.message.contains("waiting for"))
17353 );
17354 assert!(st.resource_wait.is_none());
17355 hold(&cache).release();
17357 }
17358
17359 #[tokio::test]
17360 async fn verification_cache_wait_expires_without_starting_commands() {
17361 crate::run::pin_test_home();
17362 let tmp = tempfile::tempdir().unwrap();
17363 let cache = tmp.path().join("target");
17364 let _held = hold(&cache);
17365 let mut st = state(0);
17366 let wait = repository_wait(&st, None);
17367 let (outcomes, _) = with_cache_lease(
17368 &mut st,
17369 Some(&cache),
17370 "gate",
17371 "gate",
17372 tmp.path(),
17373 "abc1234",
17374 Duration::from_secs(60),
17375 "test gate",
17376 &wait,
17377 |_, _| async { panic!("commands must not start without the cache lease") },
17378 )
17379 .await
17380 .unwrap();
17381 assert_eq!(outcomes.len(), 1);
17382 assert!(outcomes[0].resource_blocked);
17383 assert!(st.resource_wait.is_none());
17384 }
17385
17386 #[tokio::test]
17387 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
17388 crate::run::pin_test_home();
17389 let tmp = tempfile::tempdir().unwrap();
17390 let cache = tmp.path().join("target-a");
17391 let guard = hold(&cache);
17392 tokio::spawn(async move {
17394 tokio::time::sleep(Duration::from_millis(1500)).await;
17395 guard.release();
17396 });
17397 let mut st = state(60);
17398 let results = run(
17399 vec![job(
17400 sh("alpha", "echo ok"),
17401 tmp.path(),
17402 Duration::from_secs(1),
17403 )],
17404 Arc::new(Semaphore::new(1)),
17405 Some(&cache),
17406 &mut st,
17407 )
17408 .await;
17409 assert!(
17410 matches!(results[0].2, AgentOutcome::Ok(_)),
17411 "the wait must not be charged to the agent"
17412 );
17413 assert!(st.resource_block.is_none());
17414 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
17415 assert!(st.resource_wait.is_none() && st.active.is_empty());
17416 }
17417
17418 #[tokio::test]
17419 #[cfg(unix)]
17420 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
17421 use std::os::fd::AsRawFd as _;
17422 crate::run::pin_test_home();
17423 let tmp = tempfile::tempdir().unwrap();
17424 crate::git::git(tmp.path(), &["init"]).await.unwrap();
17425 let common = tmp.path().join(".git");
17426 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17427 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17428 assert_eq!(
17429 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17430 0
17431 );
17432 let mut st = state(1);
17433 st.config.coordination = crate::config::test_coordination();
17434 let mut reader = job(
17435 sh("reader", "echo must-not-run"),
17436 tmp.path(),
17437 Duration::from_secs(30),
17438 );
17439 reader.allow_write = false;
17440 let results = run(
17441 vec![reader.clone()],
17442 Arc::new(Semaphore::new(1)),
17443 None,
17444 &mut st,
17445 )
17446 .await;
17447 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
17448 assert_eq!(results[0].1.turns, 0);
17449 assert!(FailClass::of(&results[0].2).is_none());
17450 assert_eq!(
17451 st.resource_block.as_ref().unwrap().kind,
17452 "repository coordination"
17453 );
17454 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
17455 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
17456 drop(external);
17457 let editor = crate::repo_lock::SeatLease::for_checkout(
17458 tmp.path(),
17459 true,
17460 crate::config::test_coordination().as_ref(),
17461 )
17462 .await
17463 .unwrap();
17464 let pause = Pause::new();
17465 let interrupt = Pause::new();
17466 let prompts = Prompts::default();
17467 let ctx = WaveCtx {
17468 run: "r",
17469 node: "review",
17470 prompts: &prompts,
17471 cache: None,
17472 round: None,
17473 carry_seats: false,
17474 pauses: Some((&pause, &interrupt)),
17475 };
17476 let (results, ()) = tokio::join!(
17477 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
17478 async {
17479 tokio::time::sleep(Duration::from_millis(50)).await;
17480 pause.park();
17481 }
17482 );
17483 assert!(
17484 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
17485 );
17486 assert_eq!(results[0].1.turns, 0);
17487 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
17488 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
17489 drop(editor);
17490 }
17491
17492 #[tokio::test]
17493 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
17494 crate::run::pin_test_home();
17495 let tmp = tempfile::tempdir().unwrap();
17496 let cache = tmp.path().join("target-b");
17497 let _held = hold(&cache);
17498 let mut st = state(1);
17499 let results = run(
17500 vec![job(
17501 sh("alpha", "echo ok"),
17502 tmp.path(),
17503 Duration::from_secs(1),
17504 )],
17505 Arc::new(Semaphore::new(1)),
17506 Some(&cache),
17507 &mut st,
17508 )
17509 .await;
17510 let (_, seat, out) = &results[0];
17511 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
17512 assert!(wave_blocked(&results));
17513 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
17514 assert_eq!(seat.turns, 0, "nothing was asked");
17515 assert!(
17516 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
17517 "no agent was spawned"
17518 );
17519 assert!(FailClass::of(out).is_none(), "not a failure class");
17520 let block = st.resource_block.as_ref().expect("recorded for the resume");
17521 assert_eq!(block.kind, "shared build cache");
17522 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
17523 assert!(st.active.is_empty());
17524 }
17525
17526 #[tokio::test]
17527 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
17528 crate::run::pin_test_home();
17529 let tmp = tempfile::tempdir().unwrap();
17530 let mut st = state(1);
17531 let results = run(
17532 vec![job(
17533 sh("alpha", "echo ok"),
17534 tmp.path(),
17535 Duration::from_secs(1),
17536 )],
17537 Arc::new(Semaphore::new(0)),
17538 None,
17539 &mut st,
17540 )
17541 .await;
17542 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
17543 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
17544 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
17545 }
17546
17547 #[tokio::test]
17548 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
17549 crate::run::pin_test_home();
17550 let tmp = tempfile::tempdir().unwrap();
17551 let mut st = state(60);
17552 let results = run(
17553 vec![job(
17554 sh("alpha", "sleep 30"),
17555 tmp.path(),
17556 Duration::from_secs(1),
17557 )],
17558 Arc::new(Semaphore::new(1)),
17559 None,
17560 &mut st,
17561 )
17562 .await;
17563 let out = &results[0].2;
17564 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
17565 let class = FailClass::of(out).expect("a failure");
17566 assert_eq!(class, FailClass::Timeout);
17567 assert!(should_hand_over(None, &class), "existing handover policy");
17568 assert!(st.resource_block.is_none());
17569 }
17570}