1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.reserve()?;
661 state.event("start", format!("run {} created", state.id));
662 state.save()?;
663 Ok(Self {
664 state,
665 roles,
666 sem: Arc::new(Semaphore::new(max_parallel)),
667 pause: Pause::new(),
668 interrupt: Pause::new(),
669 })
670 }
671
672 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
686 Self::review_taking_over(repo, branch, config, None, origin).await
687 }
688
689 pub async fn review_taking_over(
696 repo: &Path,
697 branch: &str,
698 config: Config,
699 takeover: Option<crate::handover::Takeover>,
700 origin: Origin,
701 ) -> Result<Self> {
702 let repo = git::toplevel(repo).await?;
703 let missing = agent::missing_programs(&config.agents);
704 if !missing.is_empty() {
705 bail!(
706 "these agent programs are not on PATH: {}. Fix the roster in \
707 magi.toml or install them.",
708 missing.join(", ")
709 );
710 }
711 let base_branch =
712 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
713 .await?;
714 if base_branch == branch {
715 bail!("`{branch}` is the base branch; there is nothing to review against");
716 }
717 let fetched_base = resolve_base(
718 &repo,
719 &base_branch,
720 &config.merge.remote,
721 config.coordination.as_ref(),
722 )
723 .await?;
724 let base_commit = fetched_base.commit.clone();
725
726 let roles = config.resolve_roles()?;
727 let max_parallel = config.graph.max_parallel.max(1);
728 let mut state = RunState::new(
729 repo.clone(),
730 base_branch,
731 base_commit.clone(),
732 String::new(),
733 config,
734 );
735 state.fetched_base = Some(fetched_base);
736 state.origin = Some(origin);
737 state.reserve()?;
738
739 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
746 earlier: Vec::new(),
747 home: crate::run::home(),
748 choice: None,
749 task_instruction: None,
750 });
751 let wait = repository_wait(&state, None);
752 let coordination_config = state.config.coordination.clone();
753 let coordination =
754 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
755 .await?;
756 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
757 if let Some(released) = &released {
758 state.event(
759 "release",
760 format!(
761 "took `{branch}` over from run {}: its worktree was released: {}",
762 crate::run::short_of(&released.old_id),
763 released.audit
764 ),
765 );
766 }
767 if let Some(choice) = takeover.choice.as_ref()
771 && let Err(e) =
772 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
773 .await
774 {
775 if let Some(released) = &released {
776 released.restore(&repo, branch).await;
777 }
778 return Err(e.context("applying the owner's answer about the diverged branch"));
779 }
780 for id in takeover.earlier.iter().rev() {
786 if let Ok(old) = RunState::load_under(id, &takeover.home)
787 && let Some(head) = old.pr_head.clone()
788 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
789 {
790 state.pr_head = Some(head);
791 break;
792 }
793 }
794 drop(coordination);
795 let opened = Self::open_review(
796 &repo,
797 branch,
798 state,
799 roles,
800 max_parallel,
801 base_commit,
802 &takeover,
803 )
804 .await;
805 if opened.is_err()
806 && let Some(released) = &released
807 {
808 let _coordination = crate::repo_lock::SeatLease::operation_with(
809 &repo,
810 coordination_config.as_ref(),
811 &wait,
812 )
813 .await?;
814 released.restore(&repo, branch).await;
815 }
816 opened
817 }
818
819 pub async fn recover_pr(
821 repo: &Path,
822 task: &str,
823 selection: crate::pr_recovery::Selection,
824 mut config: Config,
825 ) -> Result<Self> {
826 selection.configure(&mut config)?;
827 selection.check_task(task, &selection.instruction)?;
828 let base_branch =
829 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 let fetched_base = resolve_base(
832 repo,
833 &base_branch,
834 &config.merge.remote,
835 config.coordination.as_ref(),
836 )
837 .await?;
838 let base_commit = fetched_base.commit.clone();
839 selection
840 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
841 .await?;
842 Box::pin(selection.verify_source(repo)).await?;
843 if let Some(proof) = &selection.source_proof {
844 anyhow::ensure!(
845 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
846 "source rebase base is outside the fetched landing base"
847 );
848 }
849 let roles = config.resolve_roles()?;
850 let max_parallel = config.graph.max_parallel.max(1);
851 let mut state = RunState::new(
852 repo.to_path_buf(),
853 base_branch,
854 base_commit.clone(),
855 selection.instruction.clone(),
856 config,
857 );
858 state.fetched_base = Some(fetched_base);
859 state.origin = Some(crate::run::Origin::queue(task));
860 state.pr_head = Some(selection.branch.clone());
861 state.pr_recovery = Some(selection.clone());
862 state.reserve()?;
863 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
864 state.discard_reserved();
866 return Err(e);
867 }
868 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
869 }
870
871 async fn open_selected(
872 repo: &Path,
873 state: RunState,
874 roles: ResolvedRoles,
875 max_parallel: usize,
876 base_commit: String,
877 selection: crate::pr_recovery::Selection,
878 ) -> Result<Self> {
879 let mut state = state;
880 let branch = state.branch_for('A');
881 let wait = repository_wait(&state, None);
882 let prepared = async {
883 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
884 git::git(repo, &["branch", &branch, selection.source()]).await?;
885 drop(coordination);
886 anyhow::Ok(())
887 }
888 .await;
889 if let Err(e) = prepared {
890 state.discard_reserved();
893 return Err(e);
894 }
895 state.event(
896 "recovery",
897 format!(
898 "selected implementation {} for {} (public lease {})",
899 selection.source(),
900 selection.url,
901 selection.head
902 ),
903 );
904 let takeover = crate::handover::Takeover {
905 earlier: Vec::new(),
906 home: crate::run::home(),
907 choice: None,
908 task_instruction: Some(selection.instruction.clone()),
909 };
910 if let Err(e) = state.save() {
911 state.discard_reserved();
912 return Err(e);
913 }
914 let id = state.id.clone();
915 let opened = Self::open_review(
916 repo,
917 &branch,
918 state,
919 roles,
920 max_parallel,
921 base_commit,
922 &takeover,
923 )
924 .await;
925 match opened {
926 Ok(runner) => Ok(runner),
927 Err(e) => {
928 if let Ok(mut partial) = RunState::load(&id) {
929 partial.status = RunStatus::Blocked;
930 partial.driver_exited = true;
931 partial.event(
932 "recovery",
933 format!("startup failed; retained source and candidate {branch}: {e:#}"),
934 );
935 partial.save()?;
936 }
937 Err(crate::pr_recovery::StartupFailed {
938 run: id,
939 cause: format!("{e:#}"),
940 }
941 .into())
942 }
943 }
944 }
945
946 async fn inherit(
951 repo: &Path,
952 branch: &str,
953 base_commit: &str,
954 takeover: &crate::handover::Takeover,
955 ) -> Option<crate::recovery::RecoveryRecord> {
956 use crate::recovery::{Relation, assess};
957 let newest = takeover.earlier.last()?;
958 let mut chosen = None;
959 for id in takeover.earlier.iter().rev() {
960 if let Ok(old) = RunState::load_under(id, &takeover.home)
961 && !old.reviews.is_empty()
962 && (old.winner().is_some_and(|w| w.branch == branch)
963 || old.pr_head.as_deref() == Some(branch))
964 {
965 chosen = Some(old);
966 break;
967 }
968 }
969 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
970 .await
971 .unwrap_or_default();
972 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
973 Some(last) if last.head == tip => Relation::Same,
974 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
975 _ => Relation::Diverged,
976 };
977 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
978 Some(assess(from, chosen.as_ref(), base_commit, relation))
979 }
980
981 async fn open_review(
984 repo: &Path,
985 branch: &str,
986 mut state: RunState,
987 roles: ResolvedRoles,
988 max_parallel: usize,
989 base_commit: String,
990 takeover: &crate::handover::Takeover,
991 ) -> Result<Self> {
992 let wait = repository_wait(&state, None);
993 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
994 if state.pr_recovery.is_none() {
995 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
996 }
997 let start = review_base(
1000 repo,
1001 &state.config.merge.remote,
1002 &state.base_branch,
1003 &base_commit,
1004 branch,
1005 )
1006 .await;
1007 let log = git::log_oneline(repo, &start, branch)
1008 .await
1009 .unwrap_or_default();
1010 let log_text = if log.trim().is_empty() {
1011 "(no commit messages)"
1012 } else {
1013 log.trim()
1014 };
1015 let mut instruction = format!(
1016 "Review the work already on branch `{branch}`. There is no task \
1017 statement: what the change claims to do is whatever its commits \
1018 say.\n\n{log_text}"
1019 );
1020 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1024 if let Some(task) = takeover
1025 .task_instruction
1026 .as_deref()
1027 .filter(|t| !t.trim().is_empty())
1028 && rec.reviewed_head.is_some()
1029 {
1030 instruction = format!(
1031 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1032 task.trim()
1033 );
1034 rec.criteria_inherited = true;
1035 }
1036 state.event("recovery", rec.summary());
1037 state.recovery = Some(rec);
1038 }
1039 if let Some(selected) = &state.pr_recovery {
1040 instruction = selected.instruction.clone();
1041 }
1042 state.task_instruction = takeover.task_instruction.clone();
1043 state.instruction = instruction;
1044 state.reviewed_commits = Some(
1045 git::subjects(repo, &start, branch)
1046 .await
1047 .unwrap_or_default(),
1048 );
1049
1050 for id in &takeover.earlier {
1053 if let Ok(old) = RunState::load_under(id, &takeover.home)
1054 && old.repo == state.repo
1055 && (old.pr_head.as_deref() == Some(branch)
1056 || old.winner().is_some_and(|w| w.branch == branch))
1057 {
1058 for hold in old.branch_owner_holds {
1059 if !state.branch_owner_holds.contains(&hold) {
1060 state.event(
1061 "approval-policy",
1062 format!(
1063 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1064 hold.control.key(),
1065 hold.file,
1066 hold.head
1067 ),
1068 );
1069 state.branch_owner_holds.push(hold);
1070 }
1071 }
1072 }
1073 }
1074 let reviewed_head = git::rev_parse(repo, branch).await?;
1075 state.observe_branch_owner_holds(&reviewed_head).await;
1076
1077 let worktree = state.worktree_root().join("under-review");
1080 if let Some(parent) = worktree.parent() {
1081 tokio::fs::create_dir_all(parent).await.ok();
1082 }
1083 let path = worktree.to_string_lossy().to_string();
1084 git::git(repo, &["worktree", "add", &path, branch])
1085 .await
1086 .with_context(|| {
1087 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1088 })?;
1089
1090 let setup = match ensure_setup_config(&mut state, repo).await {
1091 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1092 Err(e) => Err(e),
1093 };
1094 if let Err(e) = setup {
1095 git::worktree_remove_clean(repo, &worktree).await.ok();
1096 return Err(e);
1097 }
1098 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1099 .await
1100 .unwrap_or(0);
1101 if commits == 0 {
1102 git::worktree_remove_clean(repo, &worktree).await.ok();
1103 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1104 }
1105 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1106 .await
1107 .map(|f| f.len())
1108 .unwrap_or(0);
1109 if files == 0
1110 && let (Ok(head_tree), Ok(base_tree)) = (
1111 git::tree_of(&worktree, "HEAD").await,
1112 git::tree_of(&worktree, &base_commit).await,
1113 )
1114 && head_tree == base_tree
1115 {
1116 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1117 git::worktree_remove_clean(repo, &worktree).await.ok();
1118 bail!(
1119 "`{branch}` at {} has a tree identical to base {}; this usually means \
1120 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1121 against `{}/{branch}`) rather than an empty change",
1122 short(&head),
1123 short(&base_commit),
1124 state.config.merge.remote
1125 );
1126 }
1127 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1128 .await
1129 .unwrap_or_default();
1130
1131 state.candidates.push(Candidate {
1132 index: 0,
1133 label: 'A',
1134 agent: EXISTING_BRANCH.to_owned(),
1137 branch: branch.to_owned(),
1138 worktree,
1139 summary: String::new(),
1140 stat,
1141 files,
1142 commits,
1143 empty: false,
1144 failed: None,
1145 verified_noop: None,
1146 duration_ms: 0,
1147 folded: false,
1148 });
1149 state.tally = Some(Tally {
1150 first_choice: BTreeMap::from([('A', 0)]),
1151 borda: BTreeMap::new(),
1152 winner: 'A',
1153 rankings: 0,
1154 unanimous_initial: false,
1155 deliberated: false,
1156 changed_votes: 0,
1157 unanimous_final: false,
1158 tie_break: None,
1159 judges: 0,
1163 present: 0,
1164 quorum: 0,
1165 met_quorum: true,
1166 uncontested: Some("review-only run: nothing competed".to_owned()),
1167 });
1168 state.status = RunStatus::Reviewing;
1169 state.prep_complete = Some(true);
1170 state.event(
1171 "start",
1172 format!(
1173 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1174 state.id
1175 ),
1176 );
1177 state.save()?;
1178 Ok(Self {
1179 state,
1180 roles,
1181 sem: Arc::new(Semaphore::new(max_parallel)),
1182 pause: Pause::new(),
1183 interrupt: Pause::new(),
1184 })
1185 }
1186
1187 pub fn resume(id: &str) -> Result<Self> {
1189 let state = RunState::load(id)?;
1190 crate::pr_head::ensure_owner(&state)?;
1191 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1192 bail!(
1193 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1194 );
1195 }
1196 if let Some(to) = &state.released_to {
1197 bail!(
1198 "run {} cannot be resumed: its worktree was released to run {}",
1199 state.short(),
1200 crate::run::short_of(to)
1201 );
1202 }
1203 let roles = state.config.resolve_roles()?;
1204 if let Some(id) = &state.config.graph.selected_implementer
1205 && state.candidates.iter().any(|c| c.agent != *id)
1206 {
1207 bail!(
1208 "replacement implementer pin does not match the saved candidates; task must stay held"
1209 );
1210 }
1211 let max_parallel = state.config.graph.max_parallel.max(1);
1212 Ok(Self {
1213 state,
1214 roles,
1215 sem: Arc::new(Semaphore::new(max_parallel)),
1216 pause: Pause::new(),
1217 interrupt: Pause::new(),
1218 })
1219 }
1220
1221 pub async fn execute(&mut self) -> Result<()> {
1228 if let Some(selection) = &self.state.pr_recovery
1229 && (selection.source_proof.is_some() || selection.source_run.is_some())
1230 {
1231 Box::pin(selection.verify_source(&self.state.repo)).await?;
1232 }
1233 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1236 || self.state.land_approval.is_some()
1237 || self.state.merge_decision.is_some()
1238 || !self.state.merge_repairs.is_empty()
1239 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1240 {
1241 let claim = FixClaim::acquire(&self.state.dir())?;
1242 let disk = RunState::load(&self.state.id)?;
1243 let checkpoint = |s: &RunState| {
1244 (
1245 s.merge_repairs.len(),
1246 s.merge_repairs
1247 .last()
1248 .map(|r| (r.question.clone(), r.phase)),
1249 s.land_approval
1250 .as_ref()
1251 .map(|a| (a.question.clone(), a.head.clone())),
1252 )
1253 };
1254 if checkpoint(&disk) != checkpoint(&self.state) {
1255 bail!(
1256 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1257 );
1258 }
1259 Some(claim)
1260 } else {
1261 None
1262 };
1263 let mut execution = self.claim_driver().await?;
1264 let runner = &mut *execution.runner;
1265 let wait = runner.repository_wait();
1266 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1269 let result = match result {
1270 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1271 record_repository_block(&mut runner.state, &e);
1272 runner.park_here()?;
1273 Ok(())
1274 }
1275 other => other,
1276 };
1277 execution.driver.finish(&mut runner.state).await?;
1278 let ended = if result.is_err() {
1279 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1280 } else {
1281 crate::notices::run_ended(&runner.state)
1282 };
1283 if let Some(notice) = ended {
1284 crate::notices::raise(notice);
1285 }
1286 result
1287 }
1288
1289 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1290 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1291 }
1292
1293 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1294 let wait = self.repository_wait();
1295 coordinate_operation(&mut self.state, cwd, &wait).await
1296 }
1297
1298 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1300 let owner = match crate::pr_head::owner_lock_wait(
1301 &crate::run::home(),
1302 &self.state.id,
1303 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1304 )
1305 .await
1306 {
1307 Ok(owner) => owner,
1308 Err(error) => {
1309 let detail = format!("worktree maintenance or ownership update: {error:#}");
1310 self.state.resource_block = Some(crate::run::ResourceBlock {
1314 node: "start".into(),
1315 kind: "worktree maintenance".into(),
1316 detail: detail.clone(),
1317 at: Timestamp::now(),
1318 });
1319 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1320 }
1321 };
1322 let disk = RunState::load(&self.state.id)?;
1323 crate::pr_head::ensure_owner(&disk)?;
1324 if disk.liveness(false) == crate::run::Liveness::Live {
1325 bail!("run already has a live driver; wait for its execution to end");
1326 }
1327 if let Some(to) = &disk.released_to {
1328 bail!(
1329 "run {} cannot continue: its worktree was released to run {}",
1330 self.state.short(),
1331 crate::run::short_of(to)
1332 );
1333 }
1334 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1335 let mut execution = DriverExecution {
1336 runner: self,
1337 driver,
1338 };
1339 let _owner = owner;
1342 let state = &mut execution.runner.state;
1343 state.parked = false;
1348 state.resource_wait = None;
1349 state.resource_block = None;
1350 state.clear_active();
1357 let pid = std::process::id();
1372 state.driver_pid = Some(pid);
1373 state.driver_started_at = crate::proc::process_started_at(pid);
1374 state.driver_exited = false;
1375 state.begin_pass();
1376 execution.driver.bind(state);
1377 state.save()?;
1378 execution.driver.record(state)?;
1379 Ok(execution)
1380 }
1381
1382 async fn execute_graph(&mut self) -> Result<()> {
1383 self.state.configure_recovery_resume()?;
1384 self.persist_candidates().await?;
1387 if self
1388 .state
1389 .merge_decision
1390 .as_ref()
1391 .is_some_and(|decision| !decision.closed)
1392 && self.state.status != RunStatus::Landing
1393 {
1394 if crate::merge_decision::resumable_hold(&self.state) {
1395 self.state.merge = Some(MergeOutcome {
1396 mode: MergeMode::Pr,
1397 ok: false,
1398 detail: self
1399 .state
1400 .merge_decision
1401 .as_ref()
1402 .unwrap()
1403 .target
1404 .pr
1405 .clone(),
1406 empty: false,
1407 });
1408 self.run_land().await?;
1409 self.settle_questions();
1410 return Ok(());
1411 }
1412 bail!(
1413 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1414 );
1415 }
1416 if self.state.config.graph.selected_implementer.is_some()
1417 && self.state.status == RunStatus::Blocked
1418 && self.state.tally.is_none()
1419 && self.state.candidate_preservation.is_empty()
1420 && self.state.interrupted_implementations.is_empty()
1421 {
1422 bail!(
1423 "selected implementation is held; explicitly release the owning task to retry it"
1424 );
1425 }
1426
1427 if self.state.merge_repairs.last().is_some_and(|r| {
1428 matches!(
1429 r.phase,
1430 crate::run::MergeRepairPhase::Requested
1431 | crate::run::MergeRepairPhase::Fixing
1432 | crate::run::MergeRepairPhase::Reviewing
1433 )
1434 }) {
1435 if self.repair_merge_review().await? {
1436 self.run_land().await?;
1437 }
1438 self.settle_questions();
1439 return Ok(());
1440 }
1441 if self.state.pr_publication_pending {
1444 self.merge().await?;
1445 if self.resource_halt()? {
1446 return Ok(());
1447 }
1448 self.state.save()?;
1449 self.settle_questions();
1450 return Ok(());
1451 }
1452 if self.state.status == RunStatus::Stalled {
1465 if self.recover_stall().await? {
1466 self.finish_after_tally().await?;
1467 } else {
1468 self.state.save()?;
1470 }
1471 return Ok(());
1472 }
1473 if self.state.status == RunStatus::Landing
1483 || crate::handover::landing_checkpoint(&self.state)
1484 {
1485 self.state.land_sync_pending = true;
1486 self.state.save()?;
1487 self.run_land().await?;
1488 self.settle_questions();
1493 return Ok(());
1494 }
1495 if self.state.github_text.is_some()
1500 && self.state.merge.is_none()
1501 && self.state.status == RunStatus::Gating
1502 {
1503 self.merge().await?;
1504 if self.resource_halt()? {
1505 return Ok(());
1506 }
1507 self.state.save()?;
1508 return Ok(());
1509 }
1510 if let Err(e) = self.prep().await {
1511 if e.is::<crate::repo_lock::ResourceBlocked>() {
1512 return Err(e);
1513 }
1514 self.state.status = RunStatus::Blocked;
1515 self.state
1516 .prep_failure
1517 .get_or_insert_with(|| format!("{e:#}"));
1518 self.state
1519 .event("prep", format!("preparation stopped: {e:#}"));
1520 self.state.save()?;
1521 return Err(e);
1522 }
1523 if self.park_here()? {
1524 return Ok(());
1525 }
1526 self.advise().await?;
1527 if self.park_here()? {
1528 return Ok(());
1529 }
1530 self.implement().await?;
1531 if self.park_here()? {
1532 return Ok(());
1533 }
1534 if self.state.status == RunStatus::VerifiedNoop {
1538 return Ok(());
1539 }
1540 self.judge().await?;
1541 if self.park_here()? {
1542 return Ok(());
1543 }
1544 self.deliberate().await?;
1545 if self.park_here()? {
1546 return Ok(());
1547 }
1548 self.vote().await?;
1549 if self.park_here()? {
1550 return Ok(());
1551 }
1552 self.tally()?;
1553 if self.state.status == RunStatus::Stalled {
1558 self.state.save()?;
1562 return Ok(());
1563 }
1564 self.finish_after_tally().await?;
1565 Ok(())
1566 }
1567
1568 fn stop_for_resource(&mut self) -> Result<()> {
1573 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1575 {
1576 self.state.status = RunStatus::Blocked;
1577 }
1578 self.state.save()
1579 }
1580
1581 fn resource_halt(&mut self) -> Result<bool> {
1583 if self.state.resource_block.is_none() {
1584 return Ok(false);
1585 }
1586 self.stop_for_resource()?;
1587 Ok(true)
1588 }
1589
1590 fn park_here(&mut self) -> Result<bool> {
1597 let parking = self.pause.parked() || self.interrupt.parked();
1598 if !parking && self.state.resource_block.is_some() {
1599 self.stop_for_resource()?;
1600 return Ok(true);
1601 }
1602 if !self.pause.parked() && !self.interrupt.parked() {
1607 return Ok(false);
1608 }
1609 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1610 let why = match cause.as_ref().map(ParkCause::label) {
1611 Some(reason) => format!(
1612 "parked after `{}` ({reason}) — resume to carry on from here",
1613 self.state.status.as_str()
1614 ),
1615 None => format!(
1616 "parked after `{}` — resume to carry on from here",
1617 self.state.status.as_str()
1618 ),
1619 };
1620 self.state.event("park", why);
1621 self.state
1622 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1623 self.state.parked = true;
1624 self.state.save()?;
1625 Ok(true)
1626 }
1627
1628 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1630 if self.state.prep_failure.is_none() {
1631 return Ok(());
1632 }
1633 let failed_at = self
1634 .state
1635 .events
1636 .iter()
1637 .rev()
1638 .find(|event| event.node == "prep")
1639 .map_or(self.state.updated_at, |event| event.at);
1640 anyhow::ensure!(
1641 approval_at > failed_at,
1642 "preparation retry requires a new explicit release after the failure"
1643 );
1644 self.state.prep_failure = None;
1645 self.state.prep_complete = Some(false);
1646 self.state
1647 .event("prep-retry", "operator approved retry of preparation");
1648 Ok(())
1649 }
1650
1651 pub fn on_pause(&mut self, pause: Pause) {
1653 self.pause = pause;
1654 }
1655
1656 pub fn watch_interrupt(&mut self, pause: Pause) {
1662 self.interrupt = pause;
1663 }
1664
1665 fn settle_questions(&mut self) {
1685 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1686 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1687 }
1688 }
1689
1690 async fn finish_after_tally(&mut self) -> Result<()> {
1693 self.graft().await?;
1696 if self.resource_halt()? {
1697 return Ok(());
1698 }
1699 self.fold_losers().await?;
1700 self.sync_to_base().await?;
1705 if self.resource_halt()? {
1706 return Ok(());
1707 }
1708 if self.state.status == RunStatus::AlreadyInBase {
1709 return Ok(());
1710 }
1711 self.review_loop().await?;
1712 if self.resource_halt()? {
1713 return Ok(());
1714 }
1715 self.sync_to_base().await?;
1716 if self.resource_halt()? {
1717 return Ok(());
1718 }
1719 if self.state.status == RunStatus::AlreadyInBase {
1720 return Ok(());
1721 }
1722 self.gate().await?;
1723 self.merge().await?;
1724 if self.resource_halt()? {
1725 return Ok(());
1726 }
1727 self.state.save()?;
1728 Ok(())
1729 }
1730
1731 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1738 let result = match ensure_setup_config(&mut self.state, repo).await {
1739 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1740 Err(e) => Err(e),
1741 };
1742 if let Err(e) = result {
1743 git::worktree_remove_clean(repo, wt).await.ok();
1744 if in_prep {
1745 self.rollback_prep(repo).await;
1746 }
1747 self.state.status = RunStatus::Blocked;
1748 self.state.event(
1749 "setup",
1750 format!("worktree setup failed in {}: {e:#}", wt.display()),
1751 );
1752 self.state.save()?;
1753 return Err(e);
1754 }
1755 Ok(())
1756 }
1757
1758 async fn rollback_prep(&mut self, repo: &Path) {
1760 let root = self.state.worktree_root();
1761 for c in std::mem::take(&mut self.state.candidates) {
1762 if matches!(
1763 git::worktree_remove_clean(repo, &c.worktree).await,
1764 Ok(true)
1765 ) {
1766 git::branch_delete(repo, &c.branch).await.ok();
1767 } else {
1768 self.state.candidates.push(c);
1769 }
1770 }
1771 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1772 for j in 1..=seats {
1773 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1774 .await
1775 .ok();
1776 }
1777 for k in 1..=self.state.config.graph.advisors {
1778 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1779 .await
1780 .ok();
1781 }
1782 }
1783
1784 async fn prep(&mut self) -> Result<()> {
1785 if let Some(reason) = &self.state.prep_failure {
1786 bail!(
1787 "candidate preparation previously failed: {reason}; correct the source before retrying"
1788 );
1789 }
1790 if self
1791 .state
1792 .prep_complete
1793 .unwrap_or(!self.state.candidates.is_empty())
1794 {
1795 return Ok(());
1796 }
1797 let repo = self.state.repo.clone();
1798 let _coordination = self.coordinate(&repo).await?;
1801 let base = self.state.base_commit.clone();
1802 let plan = self.state.sources.plan(&repo, &base).await?;
1803 self.rollback_prep(&repo).await;
1804 if !self.state.candidates.is_empty() {
1805 bail!(
1806 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1807 );
1808 }
1809 self.state.prep_complete = Some(false);
1810 self.state.status = RunStatus::Prep;
1811 self.state.save()?;
1812 let repo = self.state.repo.clone();
1813 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1814 let root = self.state.worktree_root();
1815 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1816
1817 let hooks_dir = self.state.dir().join("hooks");
1820 if self.state.config.blind.commit_msg_hook {
1821 std::fs::create_dir_all(&hooks_dir)
1822 .with_context(|| format!("create {}", hooks_dir.display()))?;
1823 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1824 let path = hooks_dir.join("commit-msg");
1825 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1826 make_executable(&path)?;
1827 git::acquire_worktree_config(&repo).await?;
1835 self.state.enabled_worktree_config = true;
1836 }
1837
1838 ensure_setup_config(&mut self.state, &repo).await?;
1841
1842 for (index, (spec, label)) in self
1843 .roles
1844 .implementers
1845 .clone()
1846 .into_iter()
1847 .zip(labels)
1848 .enumerate()
1849 {
1850 let branch = self.state.branch_for(label);
1851 let worktree = root.join(format!("cand-{label}"));
1852 self.state.event(
1856 "prep-intent",
1857 serde_json::json!({
1858 "branch": branch,
1859 "worktree": worktree,
1860 "start": start,
1861 "picks": plan.picks,
1862 })
1863 .to_string(),
1864 );
1865 self.state.save()?;
1866 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1867 if self.state.config.blind.commit_msg_hook {
1868 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1869 }
1870 git::local_exclude(&worktree, "/.magi/").await?;
1871 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1872 git::worktree_remove_clean(&repo, &worktree).await.ok();
1875 git::branch_delete(&repo, &branch).await.ok();
1876 self.rollback_prep(&repo).await;
1877 self.state.status = RunStatus::Blocked;
1878 self.state
1879 .event("prep", format!("worktree setup failed: {e:#}"));
1880 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1881 self.state.save()?;
1882 return Err(e);
1883 }
1884 self.state.candidates.push(Candidate {
1885 index,
1886 label,
1887 agent: spec.id.clone(),
1888 branch,
1889 worktree,
1890 summary: String::new(),
1891 stat: String::new(),
1892 files: 0,
1893 commits: 0,
1894 empty: false,
1895 failed: None,
1896 verified_noop: None,
1897 duration_ms: 0,
1898 folded: false,
1899 });
1900 self.state.save()?;
1901 for pick in &plan.picks {
1902 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1903 {
1904 self.state.status = RunStatus::Blocked;
1905 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1906 self.state
1907 .event("prep", format!("cannot apply referenced commit: {e}"));
1908 self.state.save()?;
1909 return Err(e);
1910 }
1911 }
1912 }
1913
1914 for j in 1..=self.roles.judges.len() {
1915 let wt = root.join(format!("judge-{j}"));
1916 if !wt.exists() {
1917 git::worktree_add_detached(&repo, &wt, &base).await?;
1918 self.setup_seat_worktree(&repo, &wt, true).await?;
1919 }
1920 }
1921
1922 if self.state.config.graph.advise {
1930 for k in 1..=self.state.config.graph.advisors {
1931 let wt = root.join(format!("advisor-{k}"));
1932 if !wt.exists() {
1933 git::worktree_add_detached(&repo, &wt, &base).await?;
1934 self.setup_seat_worktree(&repo, &wt, true).await?;
1935 }
1936 }
1937 }
1938
1939 let authors: Vec<&str> = self
1944 .roles
1945 .implementers
1946 .iter()
1947 .map(|a| a.id.as_str())
1948 .collect();
1949 let overlap: Vec<String> = self
1950 .roles
1951 .judges
1952 .iter()
1953 .enumerate()
1954 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1955 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1956 .collect();
1957 if !overlap.is_empty() {
1958 let note = format!(
1959 "{} also authored a candidate; blind, but the panel is less \
1960 independent than {} distinct agents would be",
1961 overlap.join(", "),
1962 self.roles.judges.len()
1963 );
1964 self.state.event("prep", note);
1965 }
1966
1967 self.state.event(
1968 "prep",
1969 format!(
1970 "{} candidates, {} judges, base {} ({})",
1971 self.state.candidates.len(),
1972 self.roles.judges.len(),
1973 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1974 self.state.base_branch
1975 ),
1976 );
1977 self.state.prep_complete = Some(true);
1978 self.state.status = RunStatus::Implementing;
1979 self.state.save()?;
1980 Ok(())
1981 }
1982
1983 async fn advise(&mut self) -> Result<()> {
2016 let implement_untouched = self
2017 .state
2018 .candidates
2019 .iter()
2020 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2021 if !self.state.config.graph.advise || self.state.advise_attempted {
2022 return Ok(());
2023 }
2024 if !implement_untouched {
2025 self.state.event(
2026 "advise",
2027 "skipping the design-deliberation stage: at least one \
2028 candidate already shows implementation progress, so this \
2029 run is past the point the stage exists to run before"
2030 .to_owned(),
2031 );
2032 self.state.advise_attempted = true;
2033 self.state.save()?;
2034 return Ok(());
2035 }
2036 let run_id = self.state.id.clone();
2037 let prompts = self.state.config.prompts.clone();
2038 let instruction = self.state.instruction.clone();
2039 let language = self.state.config.graph.language.clone();
2040 let root = self.state.worktree_root();
2041 let n = self.state.config.graph.advisors;
2042 let where_recorded = self.state.dir().join("run.json");
2043
2044 let seats = match self.state.config.advisors() {
2045 Ok(seats) if !seats.is_empty() => seats,
2046 Ok(_) => {
2047 self.state.event(
2048 "advise",
2049 format!(
2050 "[graph] advisors is 0; skipping the design-deliberation \
2051 stage and continuing without a synthesis brief (see {})",
2052 where_recorded.display()
2053 ),
2054 );
2055 self.state.advise_attempted = true;
2056 self.state.save()?;
2057 return Ok(());
2058 }
2059 Err(e) => {
2060 self.state.event(
2061 "advise",
2062 format!(
2063 "could not resolve advisor seats ({e:#}); continuing \
2064 without a design-deliberation brief (see {})",
2065 where_recorded.display()
2066 ),
2067 );
2068 self.state.advise_attempted = true;
2069 self.state.save()?;
2070 return Ok(());
2071 }
2072 };
2073
2074 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2075 let artifacts = agent::artifacts_dir(&self.state.dir());
2076 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2077
2078 let mut jobs = Vec::new();
2079 for (i, spec) in seats.iter().cloned().enumerate() {
2080 let seat_key = format!("advisor-{}", i + 1);
2081 let seat = self.seat(&seat_key, &spec.id);
2082 jobs.push(SeatJob {
2083 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2084 spec,
2085 seat,
2086 cwd: worktrees[i % worktrees.len()].clone(),
2087 timeout,
2088 allow_write: false,
2089 sessions: false,
2090 artifacts: artifacts.clone(),
2091 stem: seat_key,
2092 handover: None,
2093 });
2094 }
2095
2096 self.state.event(
2097 "advise",
2098 format!(
2099 "{} advisor seat(s) sketching a design in parallel",
2100 jobs.len()
2101 ),
2102 );
2103 let mut quota_losses = Vec::new();
2104 let cache = self.state.config.cache_dir();
2105 let ctx = WaveCtx {
2106 carry_seats: false,
2107 pauses: Some((&self.pause, &self.interrupt)),
2108 run: &run_id,
2109 node: "advise",
2110 prompts: &prompts,
2111 cache: cache.as_deref(),
2112 round: None,
2113 };
2114 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2115 let results = ask_json_wave::<Proposal>(
2116 jobs,
2117 Arc::clone(&self.sem),
2118 self.state.config.graph.retries,
2119 &advisor_roster,
2120 &ctx,
2121 &mut quota_losses,
2122 &mut self.state,
2123 &|p: &Proposal| p.validate(),
2124 )
2125 .await;
2126 if self.state.resource_block.is_some() {
2127 return self.stop_for_resource();
2128 }
2129 self.state.quota.extend(quota_losses);
2130
2131 let mut records = Vec::with_capacity(results.len());
2132 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2133 let agent_id = seat.agent.clone();
2134 self.state.seats.insert(seat.key.clone(), seat);
2135 match res {
2136 Ok((proposal, out)) => {
2137 self.state
2138 .event("advise", format!("advisor-{} proposed a design", i + 1));
2139 records.push(advise::AdvisorRecord::proposed(
2140 i + 1,
2141 agent_id,
2142 proposal,
2143 out.duration_ms,
2144 ));
2145 }
2146 Err(e) => {
2147 self.state.event(
2148 "advise",
2149 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2150 );
2151 records.push(advise::AdvisorRecord::failed(
2152 i + 1,
2153 agent_id,
2154 e.to_string(),
2155 ));
2156 }
2157 }
2158 }
2159
2160 let mut advice = advise::Advice {
2161 records,
2162 synthesis: None,
2163 };
2164 if advice.proposals().is_empty() {
2165 self.state.event(
2166 "advise",
2167 "no advisor produced a usable proposal; continuing without a \
2168 synthesis brief"
2169 .to_owned(),
2170 );
2171 } else {
2172 match self
2173 .synthesize_brief(
2174 &advice,
2175 &instruction,
2176 &language,
2177 &worktrees[0],
2178 &artifacts,
2179 &run_id,
2180 &prompts,
2181 cache.as_deref(),
2182 )
2183 .await
2184 {
2185 Ok(Some(text)) => {
2186 self.state.event(
2187 "advise",
2188 "synthesized a design brief for the implementer".to_owned(),
2189 );
2190 advice.synthesis = Some(text);
2191 }
2192 Ok(None) => {
2193 self.state.event(
2194 "advise",
2195 "the synthesis seat produced nothing usable; continuing \
2196 without a design brief"
2197 .to_owned(),
2198 );
2199 }
2200 Err(e) => {
2201 if e.is::<crate::repo_lock::ResourceBlocked>() {
2202 return Err(e);
2203 }
2204 self.state.event(
2205 "advise",
2206 format!("could not synthesize a design brief: {e:#}"),
2207 );
2208 }
2209 }
2210 }
2211 advise::apply_reflection(&mut advice);
2212
2213 self.state.advice = Some(advice);
2214 self.state.advise_attempted = true;
2215 self.state.save()?;
2216 Ok(())
2217 }
2218
2219 #[allow(clippy::too_many_arguments)]
2231 async fn synthesize_brief(
2232 &mut self,
2233 advice: &advise::Advice,
2234 instruction: &str,
2235 language: &str,
2236 cwd: &Path,
2237 artifacts: &Path,
2238 run_id: &str,
2239 prompts: &Prompts,
2240 cache: Option<&Path>,
2241 ) -> Result<Option<String>> {
2242 let chain = agent::pick_chain(
2243 &self.state.config.agents,
2244 self.state.config.roles.synthesizer.as_ref(),
2245 &agent::installed,
2246 "synthesizer",
2247 )?;
2248 let proposals = advice.proposals();
2249 let mut prompt = prompt::with_overlay(
2250 prompt::synthesize_brief(instruction, &proposals, language),
2251 prompts.overlay("advise"),
2252 );
2253 if cache.is_some() {
2254 prompt.push('\n');
2259 prompt.push_str(&prompt::build_cache_note("advise", false));
2260 }
2261 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2262 let mut last = None;
2267 for (n, spec) in chain.iter().enumerate() {
2268 if n > 0 {
2269 self.state
2270 .event("advise", format!("synthesis falling back to {}", spec.id));
2271 }
2272 let mut seat = self.seat("advise-synthesis", &spec.id);
2273 let wait = self.repository_wait();
2274 let outcome = wait
2275 .scope(agent::invoke(
2276 spec,
2277 &mut seat,
2278 &Invocation {
2279 scope: crate::agent::InvocationScope::Repository,
2280 coordination: self.state.config.coordination.as_ref(),
2281 fetched_base: None,
2282 cwd,
2283 prompt: &prompt,
2284 timeout,
2285 allow_write: false,
2286 unsandboxed: false,
2287 sessions: false,
2288 artifacts,
2289 stem: &if n == 0 {
2290 "advise-synthesis".to_owned()
2291 } else {
2292 format!("advise-synthesis-{}", spec.id)
2293 },
2294 run: run_id,
2295 node: "advise",
2296 cache_dir: None,
2297 attachments: &[],
2298 writable: &[],
2299 },
2300 ))
2301 .await;
2302 if outcome.is_ok() {
2303 self.state.seats.insert(seat.key.clone(), seat);
2304 }
2305 let advance = agent::chain_advances(&outcome);
2306 last = Some(outcome);
2307 if !advance {
2308 break;
2309 }
2310 }
2311 let out = last.expect("a chain holds at least one agent")?;
2314 if !out.usable() {
2315 return Ok(None);
2316 }
2317 let text =
2318 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2319 Ok((!text.trim().is_empty()).then_some(text))
2320 }
2321
2322 async fn implement(&mut self) -> Result<()> {
2325 self.persist_candidates().await?;
2326 let run_id = self.state.id.clone();
2331 let prompts = self.state.config.prompts.clone();
2332 let todo: Vec<usize> = self
2333 .state
2334 .candidates
2335 .iter()
2336 .enumerate()
2337 .filter(|(_, c)| {
2338 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2339 && c.failed.is_none()
2340 && !c.empty
2341 })
2342 .map(|(i, _)| i)
2343 .collect();
2344 if todo.is_empty() {
2345 return self.after_implement();
2346 }
2347 self.state.status = RunStatus::Implementing;
2348
2349 let language = self.state.config.graph.language.clone();
2350 let github_language = self.state.config.graph.github_language.clone();
2351 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2352 let sessions = self.state.config.graph.sessions;
2353 let artifacts = agent::artifacts_dir(&self.state.dir());
2354 let brief = self
2358 .state
2359 .advice
2360 .as_ref()
2361 .and_then(|a| a.synthesis.as_deref())
2362 .map(str::to_owned);
2363 let attachments = self.state.attachments.clone();
2364
2365 let mut jobs = Vec::new();
2366 for &i in &todo {
2367 let (index, label, worktree) = {
2368 let c = &self.state.candidates[i];
2369 (c.index, c.label, c.worktree.clone())
2370 };
2371 let spec = self.roles.implementers[index].clone();
2372 let seat_key = format!("impl-{label}");
2373 let seat = self.seat(&seat_key, &spec.id);
2374 let instruction = seeded_instruction(&self.state);
2375 jobs.push(SeatJob {
2376 spec,
2377 seat,
2378 prompt: prompt::implement_in(
2379 &instruction,
2380 &worktree.to_string_lossy(),
2381 &language,
2382 &github_language,
2383 brief.as_deref(),
2384 &attachments,
2385 ),
2386 cwd: worktree,
2387 timeout,
2388 allow_write: true,
2389 sessions,
2390 artifacts: artifacts.clone(),
2391 stem: format!("impl-{label}"),
2392 handover: None,
2393 });
2394 }
2395
2396 self.state.event(
2397 "implement",
2398 format!("{} candidates in parallel", jobs.len()),
2399 );
2400 let mut sent = jobs.clone();
2406 let cache = self.state.config.cache_dir();
2407 let ctx = WaveCtx {
2408 carry_seats: false,
2409 pauses: Some((&self.pause, &self.interrupt)),
2410 run: &run_id,
2411 node: "implement",
2412 prompts: &prompts,
2413 cache: cache.as_deref(),
2414 round: None,
2415 };
2416 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2417 if !wave_blocked(&results) {
2418 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2419 .await;
2420 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2421 .await?;
2422 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2423 .await;
2424 }
2425 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2426
2427 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2428 && results
2429 .iter()
2430 .any(|(_, _, out)| FailClass::of(out).is_some());
2431 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2432 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2433 if self
2437 .state
2438 .candidate_preservation
2439 .contains_key(&self.state.candidates[i].label)
2440 {
2441 self.state.seats.insert(seat.key.clone(), seat);
2442 }
2443 continue;
2444 }
2445 let seat_key = seat.key.clone();
2446 let agent = seat.agent.clone();
2455 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2456 self.state.seats.insert(seat.key.clone(), seat);
2457 let label = self.state.candidates[i].label;
2458 self.state.interrupted_implementations.remove(&label);
2459
2460 let (summary, duration, failed, verified_claim) = match out {
2461 AgentOutcome::Ok(o) => {
2462 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2463 let failed = (!o.usable()).then(|| {
2464 if o.timed_out {
2465 "agent timed out".to_owned()
2466 } else {
2467 format!(
2468 "agent exited with {}",
2469 crate::notices::exit_status(o.exit_code)
2470 )
2471 }
2472 });
2473 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2474 (text, o.duration_ms, failed, verified_claim)
2475 }
2476 AgentOutcome::Dropped(o) => {
2482 let why = o
2483 .dropped
2484 .as_ref()
2485 .map(|d| d.why.as_str())
2486 .unwrap_or("the CLI ended the stream without delivering its answer");
2487 (
2488 String::new(),
2489 o.duration_ms,
2490 Some(format!("the CLI dropped the stream ({why})")),
2491 None,
2492 )
2493 }
2494 AgentOutcome::Quota(o) => {
2495 self.state.quota.push(QuotaLoss {
2496 seat: seat_key,
2497 node: "implement".to_owned(),
2498 at: Timestamp::now(),
2499 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2500 });
2501 (
2502 String::new(),
2503 o.duration_ms,
2504 Some("rate limited (quota); produced no change".to_owned()),
2505 None,
2506 )
2507 }
2508 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2509 (String::new(), 0, Some(e), None)
2510 }
2511 };
2512
2513 let c = &mut self.state.candidates[i];
2516 if !exhausted_the_fallback_chain {
2517 c.agent = agent;
2518 }
2519 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2520 c.duration_ms = duration;
2521 c.failed = failed;
2522 c.verified_noop = verified_claim;
2523 self.state
2524 .candidate_preservation
2525 .insert(label, "awaiting persistence".to_owned());
2526 }
2527 self.state.save()?;
2528 self.persist_candidates().await?;
2529
2530 if resource_blocked {
2531 return self.stop_for_resource();
2532 }
2533 if selected_failed {
2534 self.state.status = RunStatus::Blocked;
2535 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2536 self.state.save()?;
2537 bail!(
2538 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2539 );
2540 }
2541 self.after_implement()
2542 }
2543
2544 async fn persist_candidates(&mut self) -> Result<()> {
2546 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2547 for label in labels {
2548 let i = self
2549 .state
2550 .candidates
2551 .iter()
2552 .position(|c| c.label == label)
2553 .context("pending candidate is missing")?;
2554 let c = self.state.candidates[i].clone();
2555 let interrupted = self.state.interrupted_implementations.contains(&label);
2556 let result = async {
2557 let wait = crate::repo_lock::WaitPolicy::new(
2558 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2559 || false,
2560 );
2561 let _coordination =
2562 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2563 if let Err(e) = git::snapshot_uncommitted(
2566 &c.worktree,
2567 &self
2568 .state
2569 .dir()
2570 .join("artifacts")
2571 .join(format!("candidate-{label}-recovery")),
2572 )
2573 .await
2574 {
2575 self.state.event(
2576 "preserve",
2577 format!("candidate {label} snapshot incomplete: {e:#}"),
2578 );
2579 }
2580 let rescued = git::rescue_commit(
2581 &c.worktree,
2582 &format!("magi: candidate {label} (uncommitted work)"),
2583 )
2584 .await?;
2585 self.state.note_withheld("implement", &rescued.withheld);
2586 let base = &self.state.base_commit;
2587 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2588 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2589 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2590 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2591 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2592 if c.empty && commits > 0 && !patch.trim().is_empty()
2593 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2594 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2595 || self.state.tally.is_some())
2596 {
2597 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2601 }
2602 let c = &mut self.state.candidates[i];
2603 c.commits = commits;
2604 c.stat = stat;
2605 c.files = files;
2606 let no_changes = commits == 0 || patch.trim().is_empty();
2607 c.empty = no_changes && !interrupted;
2608 if interrupted && no_changes {
2609 c.failed = None;
2610 c.verified_noop = None;
2611 }
2612 if !c.empty {
2613 c.failed = None;
2614 c.verified_noop = None;
2615 }
2616 self.state.event(
2617 "preserve",
2618 format!("candidate {label}: {files} files, {commits} commits"),
2619 );
2620 Ok::<_, anyhow::Error>(())
2621 }
2622 .await;
2623 match result {
2624 Ok(()) => {
2625 self.state.candidate_preservation.remove(&label);
2626 if !interrupted || self.state.candidates[i].files > 0 {
2627 self.state.interrupted_implementations.remove(&label);
2628 }
2629 }
2630 Err(e) => {
2631 let reason = format!("candidate {label} preservation failed: {e:#}");
2632 self.state.event("preserve", &reason);
2633 self.state.candidate_preservation.insert(label, reason);
2634 }
2635 }
2636 if self.state.candidate_preservation.is_empty()
2637 && self.state.status == RunStatus::Blocked
2638 && self.state.candidates.iter().all(|c| c.failed.is_none())
2639 {
2640 self.state.status = RunStatus::Implementing;
2644 self.state.event(
2645 "preserve",
2646 "candidate persistence recovered; implementation can continue",
2647 );
2648 }
2649 self.state.save()?;
2650 }
2651 if !self.state.candidate_preservation.is_empty() {
2652 self.state.status = RunStatus::Blocked;
2653 self.state.save()?;
2654 bail!(
2655 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2656 );
2657 }
2658 Ok(())
2659 }
2660
2661 async fn resume_undelivered(
2689 &mut self,
2690 results: &mut [(usize, SeatState, AgentOutcome)],
2691 sent: &[SeatJob],
2692 prompts: &Prompts,
2693 run_id: &str,
2694 ) {
2695 for (wi, seat, out) in results.iter_mut() {
2696 let Some(dropped) = (match &*out {
2697 AgentOutcome::Dropped(o) => o.dropped.clone(),
2698 _ => None,
2699 }) else {
2700 continue;
2701 };
2702 let Some(job) = sent.get(*wi) else { continue };
2703 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2705 self.state.event(
2706 "implement",
2707 format!(
2708 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2709 work is in the tree",
2710 seat.key, dropped.output_tokens, dropped.why
2711 ),
2712 );
2713 continue;
2714 }
2715 if !has_context(&job.spec, seat, job.sessions) {
2723 self.state.event(
2724 "implement",
2725 format!(
2726 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2727 is no session left to resume",
2728 seat.key, dropped.output_tokens, dropped.why
2729 ),
2730 );
2731 continue;
2732 }
2733 self.state.event(
2734 "implement",
2735 format!(
2736 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2737 conversation",
2738 seat.key, dropped.output_tokens, dropped.why
2739 ),
2740 );
2741 let mut retry = job.clone();
2742 retry.seat = seat.clone();
2743 retry.prompt = prompt::resume_after_drop(&dropped.why);
2744 retry.timeout = retry_budget(job.timeout, true);
2745 retry.stem = format!("{}-resume", job.stem);
2746 let cache = self.state.config.cache_dir();
2747 let ctx = WaveCtx {
2748 carry_seats: false,
2749 pauses: Some((&self.pause, &self.interrupt)),
2750 run: run_id,
2751 node: "implement",
2752 prompts,
2753 cache: cache.as_deref(),
2754 round: None,
2755 };
2756 let (resumed_seat, resumed) =
2757 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2758 *seat = resumed_seat;
2759 *out = resumed;
2760 }
2761 }
2762
2763 async fn resume_seat_handovers(
2828 &mut self,
2829 results: &mut [(usize, SeatState, AgentOutcome)],
2830 sent: &mut [SeatJob],
2831 prompts: &Prompts,
2832 run_id: &str,
2833 ) -> Result<()> {
2834 let instruction = seeded_instruction(&self.state);
2835 let language = self.state.config.graph.language.clone();
2836 let github_language = self.state.config.graph.github_language.clone();
2837 let brief = self
2838 .state
2839 .advice
2840 .as_ref()
2841 .and_then(|a| a.synthesis.as_deref())
2842 .map(str::to_owned);
2843 let attachments = self.state.attachments.clone();
2844 for (wi, seat, out) in results.iter_mut() {
2845 let others: BTreeSet<String> = sent
2848 .iter()
2849 .enumerate()
2850 .filter(|(j, _)| j != wi)
2851 .map(|(_, j)| j.spec.id.clone())
2852 .collect();
2853 let Some(job) = sent.get_mut(*wi) else {
2854 continue;
2855 };
2856 let start = self
2861 .roles
2862 .implementer_roster
2863 .iter()
2864 .position(|s| s.id == job.spec.id)
2865 .unwrap_or(0);
2866 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2867 let mut fallback_attempt = 0usize;
2868 let mut prev: Option<FailClass> = None;
2869 while let Some(cur) = FailClass::of(&*out) {
2870 if !should_hand_over(prev.as_ref(), &cur) {
2871 break;
2872 }
2873 let Some(next) =
2874 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2875 .cloned()
2876 else {
2877 break;
2878 };
2879 tried.insert(next.id.clone());
2880 fallback_attempt += 1;
2881
2882 let coordination = self.coordinate(&job.cwd).await?;
2883 if let Ok(r) = git::rescue_commit(
2884 &job.cwd,
2885 &format!(
2886 "magi: candidate {} (uncommitted work before {} fallback)",
2887 seat.key,
2888 if cur == FailClass::Quota {
2889 "quota"
2890 } else {
2891 "handover"
2892 }
2893 ),
2894 )
2895 .await
2896 {
2897 self.state.note_withheld("implement", &r.withheld);
2898 }
2899 drop(coordination);
2900
2901 record_handover(
2902 &mut self.state,
2903 "implement",
2904 &seat.key,
2905 &seat.agent,
2906 &next.id,
2907 &cur,
2908 &fail_reason(&*out),
2909 );
2910 prev = Some(cur.clone());
2911
2912 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2913 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2914 job.spec = next.clone();
2922 let mut retry = job.clone();
2923 retry.seat = new_seat;
2924 retry.prompt = prompt::implement_in(
2925 &instruction,
2926 &job.cwd.to_string_lossy(),
2927 &language,
2928 &github_language,
2929 brief.as_deref(),
2930 &attachments,
2931 );
2932 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2933 let cache = self.state.config.cache_dir();
2934 let ctx = WaveCtx {
2935 carry_seats: false,
2936 pauses: Some((&self.pause, &self.interrupt)),
2937 run: run_id,
2938 node: "implement",
2939 prompts,
2940 cache: cache.as_deref(),
2941 round: None,
2942 };
2943 let (fallback_seat, fallback_out) = run_one(
2944 retry,
2945 Arc::clone(&self.sem),
2946 &ctx,
2947 &mut self.state,
2948 fallback_attempt,
2949 )
2950 .await;
2951 *seat = fallback_seat;
2952 *out = fallback_out;
2953 }
2954 }
2955 Ok(())
2956 }
2957
2958 async fn resume_unconfirmed_commands(
2982 &mut self,
2983 results: &mut [(usize, SeatState, AgentOutcome)],
2984 sent: &[SeatJob],
2985 prompts: &Prompts,
2986 run_id: &str,
2987 ) {
2988 for (wi, seat, out) in results.iter_mut() {
2989 let AgentOutcome::Ok(o) = &*out else {
2990 continue;
2991 };
2992 if !has_unconfirmed_command(&o.commands) {
2993 continue;
2994 }
2995 let Some(job) = sent.get(*wi) else { continue };
2996 if !has_context(&job.spec, seat, job.sessions) {
2997 self.state.event(
2998 "implement",
2999 format!(
3000 "{}: the reply named a command whose own CLI never confirmed the exit \
3001 status of, but there is no session left to resume",
3002 seat.key
3003 ),
3004 );
3005 continue;
3006 }
3007 self.state.event(
3008 "implement",
3009 format!(
3010 "{}: the reply named a command whose own CLI never confirmed the exit \
3011 status of; resuming the conversation",
3012 seat.key
3013 ),
3014 );
3015 let mut retry = job.clone();
3016 retry.seat = seat.clone();
3017 retry.prompt = prompt::resume_incomplete(
3018 "a command in your last reply had no confirmed exit status",
3019 );
3020 retry.timeout = retry_budget(job.timeout, true);
3021 retry.stem = format!("{}-confirm", job.stem);
3022 let cache = self.state.config.cache_dir();
3023 let ctx = WaveCtx {
3024 carry_seats: false,
3025 pauses: Some((&self.pause, &self.interrupt)),
3026 run: run_id,
3027 node: "implement",
3028 prompts,
3029 cache: cache.as_deref(),
3030 round: None,
3031 };
3032 let (resumed_seat, resumed) =
3033 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3034 *seat = resumed_seat;
3035 *out = resumed;
3036 }
3037 }
3038
3039 async fn continue_fix_report(
3060 &mut self,
3061 mut seat: SeatState,
3062 parse_err: String,
3063 job: &SeatJob,
3064 prompts: &Prompts,
3065 run_id: &str,
3066 round: usize,
3067 ) -> (
3068 SeatState,
3069 Option<FixReport>,
3070 Option<String>,
3071 ContinuationRecord,
3072 ) {
3073 let mut last_err = parse_err;
3074 let mut cumulative_wait_ms = 0u64;
3075 let mut attempts = 0usize;
3076 loop {
3077 if !has_context(&job.spec, &seat, job.sessions) {
3078 self.state.event(
3079 "fix",
3080 format!(
3081 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3082 session left to resume into"
3083 ),
3084 );
3085 let outcome = if attempts == 0 {
3086 ContinuationOutcome::NoSession
3087 } else {
3088 ContinuationOutcome::Exhausted
3089 };
3090 return (
3091 seat,
3092 None,
3093 Some(format!("unparsable fix report: {last_err}")),
3094 ContinuationRecord {
3095 attempts,
3096 cumulative_wait_ms,
3097 outcome,
3098 },
3099 );
3100 }
3101 if attempts >= MAX_FIX_CONTINUATIONS {
3102 self.state.event(
3103 "fix",
3104 format!(
3105 "round {round}: fixer's reply still had no adoption report after \
3106 {attempts} continuation(s) ({last_err}); giving up"
3107 ),
3108 );
3109 return (
3110 seat,
3111 None,
3112 Some(format!(
3113 "unparsable fix report after {attempts} continuation(s): {last_err}"
3114 )),
3115 ContinuationRecord {
3116 attempts,
3117 cumulative_wait_ms,
3118 outcome: ContinuationOutcome::Exhausted,
3119 },
3120 );
3121 }
3122 attempts += 1;
3123 self.state.event(
3124 "fix",
3125 format!(
3126 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3127 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3128 ),
3129 );
3130 let mut retry = job.clone();
3131 retry.seat = seat.clone();
3132 retry.prompt = prompt::resume_incomplete(&last_err);
3133 retry.timeout = retry_budget(job.timeout, true);
3134 retry.stem = format!("{}-continue{attempts}", job.stem);
3135 let cache = self.state.config.cache_dir();
3136 let ctx = WaveCtx {
3137 carry_seats: false,
3138 pauses: Some((&self.pause, &self.interrupt)),
3139 run: run_id,
3140 node: "fix",
3141 prompts,
3142 cache: cache.as_deref(),
3143 round: Some(round),
3144 };
3145 let (resumed_seat, resumed_out) = run_one(
3146 retry,
3147 Arc::clone(&self.sem),
3148 &ctx,
3149 &mut self.state,
3150 attempts,
3151 )
3152 .await;
3153 seat = resumed_seat;
3154 match resumed_out {
3155 AgentOutcome::Ok(o) => {
3156 cumulative_wait_ms += o.duration_ms;
3157 match verdict::extract_json::<FixReport>(&o.text) {
3158 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3159 self.state.event(
3160 "fix",
3161 format!(
3162 "round {round}: fixer's adoption report recovered after \
3163 {attempts} continuation(s)"
3164 ),
3165 );
3166 return (
3167 seat,
3168 Some(report),
3169 None,
3170 ContinuationRecord {
3171 attempts,
3172 cumulative_wait_ms,
3173 outcome: ContinuationOutcome::Resumed,
3174 },
3175 );
3176 }
3177 Ok(_) => {
3185 last_err = "the reply parsed, but it reported a command whose own CLI \
3186 never confirmed an exit status"
3187 .to_owned();
3188 }
3189 Err(e) => last_err = e.to_string(),
3190 }
3191 }
3192 AgentOutcome::Quota(o) => {
3193 cumulative_wait_ms += o.duration_ms;
3194 self.state.quota.push(QuotaLoss {
3195 seat: seat.key.clone(),
3196 node: "fix".to_owned(),
3197 at: Timestamp::now(),
3198 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3199 });
3200 self.state.event(
3201 "fix",
3202 format!(
3203 "round {round}: continuation rate limited (quota); not retrying now"
3204 ),
3205 );
3206 return (
3207 seat,
3208 None,
3209 Some("rate limited (quota) while recovering the fix report".to_owned()),
3210 ContinuationRecord {
3211 attempts,
3212 cumulative_wait_ms,
3213 outcome: ContinuationOutcome::QuotaLost,
3214 },
3215 );
3216 }
3217 AgentOutcome::Dropped(o) => {
3218 cumulative_wait_ms += o.duration_ms;
3219 let why = o
3220 .dropped
3221 .as_ref()
3222 .map(|d| d.why.as_str())
3223 .unwrap_or("the CLI ended the stream without delivering its answer");
3224 last_err = format!("the CLI dropped the stream ({why})");
3225 }
3226 AgentOutcome::Failed(e) => last_err = e,
3227 AgentOutcome::ResourceBlocked(e) => {
3230 return (
3231 seat,
3232 None,
3233 Some(format!("the continuation was not started: {e}")),
3234 ContinuationRecord {
3235 attempts: attempts.saturating_sub(1),
3236 cumulative_wait_ms,
3237 outcome: ContinuationOutcome::NoSession,
3238 },
3239 );
3240 }
3241 }
3242 }
3243 }
3244
3245 fn after_implement(&mut self) -> Result<()> {
3246 if !self.state.candidate_preservation.is_empty() {
3247 bail!("candidate preservation incomplete; judging is unsafe");
3248 }
3249 if self.state.leaks.is_empty() {
3251 let cfg = self.state.config.blind.clone();
3252 let mut leaks = Vec::new();
3253 for c in &self.state.candidates {
3254 let Some(patch) =
3255 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3256 else {
3257 continue;
3258 };
3259 leaks.extend(blind::scan(
3260 &format!("candidate {} patch", c.label),
3261 &patch,
3262 &cfg.vendor_tokens,
3263 ));
3264 }
3265 if !leaks.is_empty() {
3266 let summary = leaks
3267 .iter()
3268 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3269 .collect::<Vec<_>>()
3270 .join(", ");
3271 match cfg.on_leak {
3272 LeakPolicy::Fail => {
3273 self.state.status = RunStatus::Failed;
3274 self.state
3275 .event("blind", format!("vendor text in a patch: {summary}"));
3276 self.state.leaks = leaks;
3277 self.state.save()?;
3278 self.settle_questions();
3279 bail!(
3280 "blind.on_leak = \"fail\" and vendor text reached a \
3281 judged patch: {summary}"
3282 );
3283 }
3284 LeakPolicy::Redact => self.state.event(
3285 "blind",
3286 format!("redacting vendor text for judging: {summary}"),
3287 ),
3288 LeakPolicy::Warn => self.state.event(
3289 "blind",
3290 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3291 ),
3292 }
3293 self.state.leaks = leaks;
3294 }
3295 }
3296
3297 if self.state.viable().is_empty() {
3298 if self.state.all_candidates_verified_noop() {
3299 self.state.status = RunStatus::VerifiedNoop;
3310 self.state.save()?;
3311 self.settle_questions();
3312 return Ok(());
3313 }
3314 self.state.status = RunStatus::Failed;
3315 self.state.save()?;
3316 self.settle_questions();
3317 bail!("no candidate produced a change; nothing to judge");
3318 }
3319 self.state.status = RunStatus::Judging;
3320 self.state.save()?;
3321 Ok(())
3322 }
3323
3324 async fn judge(&mut self) -> Result<()> {
3327 let run_id = self.state.id.clone();
3332 let prompts = self.state.config.prompts.clone();
3333 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3334 return Ok(());
3335 }
3336 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3337 if viable.len() == 1 {
3338 self.state.judge_skipped = true;
3345 self.state.event(
3346 "judge",
3347 format!(
3348 "only candidate {} produced a change; judging skipped",
3349 viable[0].label
3350 ),
3351 );
3352 self.state.save()?;
3353 return Ok(());
3354 }
3355 self.state.status = RunStatus::Judging;
3356
3357 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3358 let language = self.state.config.graph.language.clone();
3359 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3360 let sessions = self.state.config.graph.sessions;
3361 let artifacts = agent::artifacts_dir(&self.state.dir());
3362 let root = self.state.worktree_root();
3363 let base_short = short(&self.state.base_commit);
3364
3365 let mut jobs = Vec::new();
3366 let mut orders = Vec::new();
3367 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3368 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3369 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3370 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3371 let seat_key = format!("judge-{}", j + 1);
3372 let seat = self.seat(&seat_key, &spec.id);
3373 jobs.push(SeatJob {
3374 prompt: prompt::judge(
3375 &self.state.instruction,
3376 &views,
3377 self.roles.judges.len(),
3378 &base_short,
3379 &language,
3380 ),
3381 spec,
3382 seat,
3383 cwd: root.join(format!("judge-{}", j + 1)),
3384 timeout,
3385 allow_write: false,
3386 sessions,
3387 artifacts: artifacts.clone(),
3388 stem: format!("judge-{}", j + 1),
3389 handover: None,
3390 });
3391 }
3392
3393 self.state.event(
3394 "judge",
3395 format!(
3396 "{} judges ranking {} candidates blind",
3397 jobs.len(),
3398 viable.len()
3399 ),
3400 );
3401 let labels_for_check = labels.clone();
3402 let mut quota_losses = Vec::new();
3403 let cache = self.state.config.cache_dir();
3404 let ctx = WaveCtx {
3405 carry_seats: false,
3406 pauses: Some((&self.pause, &self.interrupt)),
3407 run: &run_id,
3408 node: "judge",
3409 prompts: &prompts,
3410 cache: cache.as_deref(),
3411 round: None,
3412 };
3413 let results = ask_json_wave::<Ranking>(
3414 jobs,
3415 Arc::clone(&self.sem),
3416 self.state.config.graph.retries,
3417 &self.roles.judge_roster,
3418 &ctx,
3419 &mut quota_losses,
3420 &mut self.state,
3421 &move |r: &Ranking| r.validate(&labels_for_check),
3422 )
3423 .await;
3424 if self.state.resource_block.is_some() {
3425 return self.stop_for_resource();
3426 }
3427 self.state.quota.extend(quota_losses);
3428
3429 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3430 let agent_id = seat.agent.clone();
3431 self.state.seats.insert(seat.key.clone(), seat);
3432 let mut record = Judgement {
3433 judge: j + 1,
3434 seat: format!("judge-{}", j + 1),
3435 agent: agent_id,
3436 ranking: Vec::new(),
3437 reasons: BTreeMap::new(),
3438 confidence: None,
3439 strengths: BTreeMap::new(),
3440 order: orders[j].clone(),
3441 failed: None,
3442 duration_ms: 0,
3443 };
3444 match res {
3445 Ok((ranking, out)) => {
3446 record.ranking = ranking.normalized();
3447 record.reasons = ranking.reasons;
3448 record.strengths = ranking.strengths;
3449 record.confidence = ranking.confidence;
3450 record.duration_ms = out.duration_ms;
3451 self.state.event(
3452 "judge",
3453 format!(
3454 "judge {} ranked {}",
3455 j + 1,
3456 record.ranking.iter().collect::<String>()
3457 ),
3458 );
3459 }
3460 Err(e) => {
3461 record.failed = Some(e.to_string());
3462 self.state
3463 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3464 }
3465 }
3466 self.state.judgements.push(record);
3467 self.state.save()?;
3468 }
3469 Ok(())
3470 }
3471
3472 async fn deliberate(&mut self) -> Result<()> {
3475 let run_id = self.state.id.clone();
3480 let prompts = self.state.config.prompts.clone();
3481 if !self.state.deliberation.is_empty() {
3482 return Ok(());
3483 }
3484 let tops: Vec<char> = self
3485 .state
3486 .judgements
3487 .iter()
3488 .filter_map(|j| j.ranking.first().copied())
3489 .collect();
3490 let rounds = self.state.config.graph.deliberate_rounds;
3491 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3492 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3493 self.state.event(
3494 "deliberate",
3495 format!("judges agreed on {} outright; no deliberation", tops[0]),
3496 );
3497 }
3498 self.state.status = RunStatus::Voting;
3499 self.state.save()?;
3500 return Ok(());
3501 }
3502
3503 self.state.status = RunStatus::Deliberating;
3504 self.state.event(
3505 "deliberate",
3506 format!(
3507 "split: first choices were {} — opening {rounds} round(s)",
3508 tops.iter().collect::<String>()
3509 ),
3510 );
3511
3512 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3513 let language = self.state.config.graph.language.clone();
3514 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3515 let sessions = self.state.config.graph.sessions;
3516 let artifacts = agent::artifacts_dir(&self.state.dir());
3517 let root = self.state.worktree_root();
3518 let base_short = short(&self.state.base_commit);
3519
3520 for round in 1..=rounds {
3524 let mut turns: Vec<DeliberationTurn> = Vec::new();
3525 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3526 if self.state.judgements[j].failed.is_some() {
3527 continue;
3528 }
3529 let seat_key = format!("judge-{}", j + 1);
3530 let spec = self.occupant(&seat_key, spec);
3531 let mut seat = self.seat(&seat_key, &spec.id);
3532 let transcript = self.transcript(&turns, j);
3533 let build = |context: Option<&str>| {
3534 prompt::deliberate(
3535 &self.state.instruction,
3536 context,
3537 &transcript,
3538 round,
3539 rounds,
3540 &language,
3541 )
3542 };
3543 let block = self.candidate_block(&viable, &base_short);
3544 let full = build(Some(&block));
3545 let text = if has_context(&spec, &seat, sessions) {
3546 build(None)
3547 } else {
3548 full.clone()
3549 };
3550 let job = SeatJob {
3551 spec,
3552 seat: seat.clone(),
3553 prompt: text,
3554 cwd: root.join(format!("judge-{}", j + 1)),
3555 timeout,
3556 allow_write: false,
3557 sessions,
3558 artifacts: artifacts.clone(),
3559 stem: format!("delib-{round}-judge-{}", j + 1),
3560 handover: Some(full),
3561 };
3562 let cache = self.state.config.cache_dir();
3563 let ctx = WaveCtx {
3564 carry_seats: false,
3565 pauses: Some((&self.pause, &self.interrupt)),
3566 run: &run_id,
3567 node: "deliberate",
3568 prompts: &prompts,
3569 cache: cache.as_deref(),
3570 round: None,
3571 };
3572 let mut losses = Vec::new();
3576 let mut results = ask_wave_with::<String>(
3577 vec![job],
3578 Arc::clone(&self.sem),
3579 0,
3580 &self.roles.judge_roster,
3581 &ctx,
3582 &mut losses,
3583 &mut self.state,
3584 &|text: &str| {
3585 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3586 },
3587 )
3588 .await;
3589 self.state.quota.extend(losses);
3590 let (updated, res, _) = results.pop().expect("one job in, one result out");
3591 seat = updated;
3592 let agent_id = seat.agent.clone();
3593 self.state.seats.insert(seat.key.clone(), seat);
3594 let body = match res {
3595 Ok((body, _)) => body,
3596 Err(e) => {
3599 self.state
3600 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3601 continue;
3602 }
3603 };
3604 let tentative = verdict::extract_json::<Position>(&body)
3605 .ok()
3606 .and_then(|p| p.tentative)
3607 .and_then(|s| s.trim().chars().next())
3608 .map(|c| c.to_ascii_uppercase());
3609 self.state.event(
3610 "deliberate",
3611 format!(
3612 "round {round}: judge {} now favours {}",
3613 j + 1,
3614 tentative.map_or("—".to_owned(), |c| c.to_string())
3615 ),
3616 );
3617 turns.push(DeliberationTurn {
3618 judge: j + 1,
3619 agent: agent_id,
3620 body: blind::sanitize_prose(&body, &self.state.config.blind),
3621 tentative,
3622 });
3623 }
3624 self.state
3625 .deliberation
3626 .push(DeliberationRound { round, turns });
3627 self.state.save()?;
3628 }
3629
3630 self.state.status = RunStatus::Voting;
3631 self.state.save()?;
3632 Ok(())
3633 }
3634
3635 async fn vote(&mut self) -> Result<()> {
3638 let run_id = self.state.id.clone();
3643 let prompts = self.state.config.prompts.clone();
3644 if !self.state.votes.is_empty() {
3645 return Ok(());
3646 }
3647 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3648 if viable.len() == 1 {
3649 return Ok(());
3650 }
3651 self.state.status = RunStatus::Voting;
3652
3653 let language = self.state.config.graph.language.clone();
3654 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3655 let sessions = self.state.config.graph.sessions;
3656 let artifacts = agent::artifacts_dir(&self.state.dir());
3657 let root = self.state.worktree_root();
3658 let base_short = short(&self.state.base_commit);
3659 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3660
3661 let mut jobs = Vec::new();
3662 let mut seats_at = Vec::new();
3663 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3664 if self
3665 .state
3666 .judgements
3667 .get(j)
3668 .is_some_and(|r| r.failed.is_some())
3669 {
3670 continue;
3671 }
3672 let seat_key = format!("judge-{}", j + 1);
3673 let spec = self.occupant(&seat_key, spec);
3674 let seat = self.seat(&seat_key, &spec.id);
3675 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3676 let text = if has_context(&spec, &seat, sessions) {
3677 prompt::final_vote(&viable, &language)
3678 } else {
3679 full.clone()
3680 };
3681 jobs.push(SeatJob {
3682 spec,
3683 seat,
3684 prompt: text,
3685 cwd: root.join(format!("judge-{}", j + 1)),
3686 timeout,
3687 allow_write: false,
3688 sessions,
3689 artifacts: artifacts.clone(),
3690 stem: format!("vote-judge-{}", j + 1),
3691 handover: Some(full),
3692 });
3693 seats_at.push(j);
3694 }
3695
3696 self.state.event(
3697 "vote",
3698 format!(
3699 "collecting {} final votes one by one, privately",
3700 jobs.len()
3701 ),
3702 );
3703 let allowed = viable.clone();
3704 let mut quota_losses = Vec::new();
3705 let cache = self.state.config.cache_dir();
3706 let ctx = WaveCtx {
3707 carry_seats: false,
3708 pauses: Some((&self.pause, &self.interrupt)),
3709 run: &run_id,
3710 node: "vote",
3711 prompts: &prompts,
3712 cache: cache.as_deref(),
3713 round: None,
3714 };
3715 let results = ask_json_wave::<FinalVote>(
3716 jobs,
3717 Arc::clone(&self.sem),
3718 self.state.config.graph.retries,
3719 &self.roles.judge_roster,
3720 &ctx,
3721 &mut quota_losses,
3722 &mut self.state,
3723 &move |v: &FinalVote| match v.label() {
3724 Some(c) if allowed.contains(&c) => Ok(()),
3725 other => bail!("vote {other:?} is not one of {allowed:?}"),
3726 },
3727 )
3728 .await;
3729 if self.state.resource_block.is_some() {
3730 return self.stop_for_resource();
3731 }
3732 self.state.quota.extend(quota_losses);
3733
3734 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3735 let agent_id = seat.agent.clone();
3736 self.state.seats.insert(seat.key.clone(), seat);
3737 let initial = self
3738 .state
3739 .judgements
3740 .get(j)
3741 .and_then(|r| r.ranking.first().copied());
3742 let mut record = VoteRecord {
3743 judge: j + 1,
3744 agent: agent_id,
3745 vote: None,
3746 reason: String::new(),
3747 changed: false,
3748 };
3749 match res {
3750 Ok((v, _)) => {
3751 record.vote = v.label();
3752 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3753 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3754 self.state.event(
3755 "vote",
3756 format!(
3757 "judge {} voted {}{}",
3758 j + 1,
3759 record.vote.unwrap_or('?'),
3760 if record.changed { " (changed)" } else { "" }
3761 ),
3762 );
3763 }
3764 Err(e) => {
3765 self.state
3766 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3767 }
3768 }
3769 self.state.votes.push(record);
3770 self.state.save()?;
3771 }
3772 Ok(())
3773 }
3774
3775 fn tally(&mut self) -> Result<()> {
3778 if self.state.tally.is_some() {
3779 return Ok(());
3780 }
3781 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3782 let tops: Vec<char> = self
3783 .state
3784 .judgements
3785 .iter()
3786 .filter_map(|j| j.ranking.first().copied())
3787 .collect();
3788 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3789
3790 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3793 let mut cast: Vec<char> = Vec::new();
3794 for (i, j) in self.state.judgements.iter().enumerate() {
3795 let vote = self
3796 .state
3797 .votes
3798 .iter()
3799 .find(|v| v.judge == i + 1)
3800 .and_then(|v| v.vote)
3801 .or_else(|| j.ranking.first().copied());
3802 if let Some(v) = vote {
3803 *first_choice.entry(v).or_insert(0) += 1;
3804 cast.push(v);
3805 }
3806 }
3807
3808 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3809 for j in &self.state.judgements {
3810 let n = j.ranking.len();
3811 for (pos, label) in j.ranking.iter().enumerate() {
3812 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3813 }
3814 }
3815
3816 let best = first_choice.values().copied().max().unwrap_or(0);
3817 let mut leaders: Vec<char> = first_choice
3818 .iter()
3819 .filter(|(_, v)| **v == best)
3820 .map(|(k, _)| *k)
3821 .collect();
3822 let mut tie_break = None;
3823 if leaders.len() > 1 {
3824 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3825 let borda_leaders: Vec<char> = leaders
3826 .iter()
3827 .copied()
3828 .filter(|l| borda[l] == top_borda)
3829 .collect();
3830 tie_break = Some(if borda_leaders.len() == 1 {
3831 format!(
3832 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3833 leaders.len()
3834 )
3835 } else {
3836 format!(
3837 "{} way tie on both first-choice votes and Borda points, broken by label order",
3838 leaders.len()
3839 )
3840 });
3841 leaders = borda_leaders;
3842 leaders.sort_unstable();
3843 }
3844 let winner = *leaders
3845 .first()
3846 .or(viable.first())
3847 .context("no candidate to declare a winner from")?;
3848
3849 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3850 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3851 let deliberated = !self.state.deliberation.is_empty();
3852
3853 let quota_seats: std::collections::BTreeSet<&str> =
3857 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3858 let mut present = 0usize;
3859 for (i, j) in self.state.judgements.iter().enumerate() {
3860 if quota_seats.contains(j.seat.as_str()) {
3861 continue;
3862 }
3863 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3864 let voted = self
3865 .state
3866 .votes
3867 .iter()
3868 .any(|v| v.judge == i + 1 && v.vote.is_some());
3869 if ranked || voted {
3870 present += 1;
3871 }
3872 }
3873 let needs_quorum = viable.len() > 1;
3879 let judges_total = if needs_quorum {
3880 self.roles.judges.len()
3881 } else {
3882 0
3883 };
3884 let quorum = if needs_quorum {
3885 judges_total / 2 + 1
3886 } else {
3887 0
3888 };
3889 let met_quorum = !needs_quorum || present >= quorum;
3890 let uncontested = (!needs_quorum).then(|| {
3891 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3892 });
3893
3894 self.state.event(
3895 "tally",
3896 match &uncontested {
3897 Some(reason) => format!("winner {winner} — {reason}"),
3898 None => format!(
3899 "winner {winner} — votes {} | initial {} | {} changed | \
3900 {present}/{judges_total} judges{}",
3901 first_choice
3902 .iter()
3903 .map(|(k, v)| format!("{k}:{v}"))
3904 .collect::<Vec<_>>()
3905 .join(" "),
3906 if unanimous_initial {
3907 "unanimous"
3908 } else {
3909 "split"
3910 },
3911 changed_votes,
3912 if met_quorum {
3913 String::new()
3914 } else {
3915 format!(" — below quorum ({quorum} required)")
3916 },
3917 ),
3918 },
3919 );
3920 if !met_quorum {
3921 self.state.event(
3922 "stall",
3923 format!(
3924 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3925 the run stops here, resumable"
3926 ),
3927 );
3928 }
3929 self.state.tally = Some(Tally {
3930 first_choice,
3931 borda,
3932 winner,
3933 rankings: tops.len(),
3934 unanimous_initial,
3935 deliberated,
3936 changed_votes,
3937 unanimous_final,
3938 tie_break,
3939 judges: judges_total,
3940 present,
3941 quorum,
3942 met_quorum,
3943 uncontested,
3944 });
3945 self.state.status = if met_quorum {
3946 RunStatus::Reviewing
3947 } else {
3948 RunStatus::Stalled
3949 };
3950 self.state.save()?;
3951 Ok(())
3952 }
3953
3954 #[allow(clippy::too_many_lines)]
3975 async fn recover_stall(&mut self) -> Result<bool> {
3976 let run_id = self.state.id.clone();
3981 let prompts = self.state.config.prompts.clone();
3982 let quota_seats: BTreeSet<&str> =
3987 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3988 let absent: Vec<String> = self
3989 .state
3990 .judgements
3991 .iter()
3992 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3993 .map(|j| j.seat.clone())
3994 .collect();
3995 if absent.is_empty() {
3996 return Ok(false);
3997 }
3998 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3999 if viable.len() <= 1 {
4000 return Ok(false);
4001 }
4002 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4003 let language = self.state.config.graph.language.clone();
4004 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4005 let sessions = self.state.config.graph.sessions;
4006 let artifacts = agent::artifacts_dir(&self.state.dir());
4007 let root = self.state.worktree_root();
4008 let base_short = short(&self.state.base_commit);
4009 let candidates: Vec<Candidate> = viable.clone();
4010
4011 let mut positions: Vec<usize> = absent
4013 .iter()
4014 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4015 .collect();
4016 if positions.is_empty() {
4017 return Ok(false);
4018 }
4019 positions.sort_unstable();
4020 positions.dedup();
4021
4022 let mut judge_jobs = Vec::new();
4024 for &j in &positions {
4025 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4026 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4027 let seat_key = format!("judge-{}", j + 1);
4028 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4029 let seat = self.seat(&seat_key, &spec.id);
4030 judge_jobs.push(SeatJob {
4031 spec,
4032 seat,
4033 prompt: prompt::judge(
4034 &self.state.instruction,
4035 &views,
4036 self.roles.judges.len(),
4037 &base_short,
4038 &language,
4039 ),
4040 cwd: root.join(seat_key),
4041 timeout,
4042 allow_write: false,
4043 sessions,
4044 artifacts: artifacts.clone(),
4045 stem: format!("judge-{}-recover", j + 1),
4046 handover: None,
4047 });
4048 }
4049
4050 let labels_for_check = labels.clone();
4051 let mut judge_losses = Vec::new();
4052 let retries = self.state.config.graph.retries;
4053 let cache = self.state.config.cache_dir();
4054 let ctx = WaveCtx {
4055 carry_seats: false,
4056 pauses: Some((&self.pause, &self.interrupt)),
4057 run: &run_id,
4058 node: "judge",
4059 prompts: &prompts,
4060 cache: cache.as_deref(),
4061 round: None,
4062 };
4063 let results = ask_json_wave::<Ranking>(
4064 judge_jobs,
4065 Arc::clone(&self.sem),
4066 retries,
4067 &self.roles.judge_roster,
4068 &ctx,
4069 &mut judge_losses,
4070 &mut self.state,
4071 &move |r: &Ranking| r.validate(&labels_for_check),
4072 )
4073 .await;
4074 if self.state.resource_block.is_some() {
4075 self.stop_for_resource()?;
4076 return Ok(false);
4077 }
4078
4079 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4081 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4082 let agent_id = seat.agent.clone();
4083 self.state.seats.insert(seat.key.clone(), seat);
4084 let record = &mut self.state.judgements[j];
4085 match res {
4086 Ok((ranking, out)) => {
4087 record.agent = agent_id;
4088 record.ranking = ranking.normalized();
4089 record.reasons = ranking.reasons;
4090 record.strengths = ranking.strengths;
4091 record.confidence = ranking.confidence;
4092 record.failed = None;
4093 record.duration_ms = out.duration_ms;
4094 recovered.insert(j);
4095 self.state.event(
4096 "recover",
4097 format!("judge {} ranked again after the limit", j + 1),
4098 );
4099 }
4100 Err(e) => {
4101 self.state
4102 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4103 }
4104 }
4105 }
4106
4107 let mut vote_jobs = Vec::new();
4109 let mut vote_pos: Vec<usize> = Vec::new();
4110 for &j in &recovered {
4111 let seat_key = format!("judge-{}", j + 1);
4112 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4113 let seat = self.seat(&seat_key, &spec.id);
4114 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4115 let text = if has_context(&spec, &seat, sessions) {
4116 prompt::final_vote(&labels, &language)
4117 } else {
4118 full.clone()
4119 };
4120 vote_jobs.push(SeatJob {
4121 spec,
4122 seat,
4123 prompt: text,
4124 cwd: root.join(seat_key),
4125 timeout,
4126 allow_write: false,
4127 sessions,
4128 artifacts: artifacts.clone(),
4129 stem: format!("vote-judge-{}-recover", j + 1),
4130 handover: Some(full),
4131 });
4132 vote_pos.push(j);
4133 }
4134 let allowed = labels.clone();
4135 let mut vote_losses = Vec::new();
4136 let vote_retries = self.state.config.graph.retries;
4137 let vote_cache = self.state.config.cache_dir();
4138 let ctx = WaveCtx {
4139 carry_seats: false,
4140 pauses: Some((&self.pause, &self.interrupt)),
4141 run: &run_id,
4142 node: "vote",
4143 prompts: &prompts,
4144 cache: vote_cache.as_deref(),
4145 round: None,
4146 };
4147 let votes = ask_json_wave::<FinalVote>(
4148 vote_jobs,
4149 Arc::clone(&self.sem),
4150 vote_retries,
4151 &self.roles.judge_roster,
4152 &ctx,
4153 &mut vote_losses,
4154 &mut self.state,
4155 &move |v: &FinalVote| match v.label() {
4156 Some(c) if allowed.contains(&c) => Ok(()),
4157 other => bail!("vote {other:?} is not one of {allowed:?}"),
4158 },
4159 )
4160 .await;
4161 if self.state.resource_block.is_some() {
4162 self.stop_for_resource()?;
4163 return Ok(false);
4164 }
4165 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4166 let agent_id = seat.agent.clone();
4167 self.state.seats.insert(seat.key.clone(), seat);
4168 match res {
4169 Ok((v, _)) => {
4170 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4171 rec.vote = v.label();
4172 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4173 } else {
4174 self.state.votes.push(VoteRecord {
4175 judge: j + 1,
4176 agent: agent_id,
4177 vote: v.label(),
4178 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4179 changed: false,
4180 });
4181 }
4182 self.state.event(
4183 "recover",
4184 format!("judge {} voted again after the limit", j + 1),
4185 );
4186 }
4187 Err(e) => {
4188 self.state
4189 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4190 }
4191 }
4192 }
4193
4194 let recovered_keys: BTreeSet<String> = recovered
4198 .iter()
4199 .map(|&j| format!("judge-{}", j + 1))
4200 .collect();
4201 self.state
4202 .quota
4203 .retain(|q| !recovered_keys.contains(&q.seat));
4204 for loss in judge_losses.into_iter().chain(vote_losses) {
4208 if recovered_keys.contains(&loss.seat) {
4209 continue;
4210 }
4211 self.state.quota.retain(|q| q.seat != loss.seat);
4212 self.state.quota.push(loss);
4213 }
4214
4215 self.state.tally = None;
4217 self.tally()?;
4218 Ok(self
4219 .state
4220 .tally
4221 .as_ref()
4222 .map(|t| t.met_quorum)
4223 .unwrap_or(false))
4224 }
4225
4226 async fn graft(&mut self) -> Result<()> {
4242 let Err(e) = self.graft_inner().await else {
4243 return Ok(());
4244 };
4245 if e.is::<crate::repo_lock::ResourceBlocked>() {
4246 return Err(e);
4247 }
4248 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4249 self.state.status = RunStatus::Blocked;
4250 self.state.event("graft", format!("blocked: {e:#}"));
4251 self.state.save()?;
4252 return Err(e);
4253 }
4254 if let Some(rec) = self.state.graft.clone()
4259 && !rec.outcome.settled()
4260 {
4261 if let Some(winner) = self.state.winner().cloned() {
4262 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4263 if re.is::<crate::repo_lock::ResourceBlocked>() {
4264 return Err(re);
4265 }
4266 self.state.status = RunStatus::Blocked;
4267 self.state.event("graft", format!("blocked: {re:#}"));
4268 self.state.save()?;
4269 return Err(re);
4270 }
4271 }
4272 if let Some(g) = self.state.graft.as_mut() {
4273 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4274 }
4275 }
4276 self.state.event("graft", format!("skipped: {e:#}"));
4277 if let Err(e) = self.state.save() {
4278 tracing::warn!("save after graft: {e:#}");
4279 }
4280 Ok(())
4281 }
4282
4283 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4286 let _coordination = self.coordinate(wt).await?;
4287 crate::recovery::restore_repair(wt, pre)
4288 .await
4289 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4290 }
4291
4292 fn graft_skipped(&mut self, why: &str) {
4294 let message = format!("skipped: {why}");
4295 if !self
4296 .state
4297 .events
4298 .iter()
4299 .any(|e| e.node == "graft" && e.message == message)
4300 {
4301 self.state.event("graft", message);
4302 }
4303 }
4304
4305 async fn graft_inner(&mut self) -> Result<()> {
4306 if let Some(rec) = self.state.graft.clone()
4309 && !rec.outcome.settled()
4310 {
4311 if self.state.reviews.is_empty()
4315 && let Some(winner) = self.state.winner().cloned()
4316 {
4317 self.graft_restore(&winner.worktree, &rec.pre).await?;
4318 }
4319 if rec.outcome == GraftOutcome::WaitingForRestore {
4320 self.state.graft = None;
4321 self.state.save()?;
4322 } else {
4323 if let Some(g) = self.state.graft.as_mut() {
4324 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4325 }
4326 self.state.event(
4327 "graft",
4328 "rolled back: the previous round was interrupted; the winner is untouched",
4329 );
4330 self.state.save()?;
4331 return Ok(());
4332 }
4333 }
4334 let plan = match graft::plan(&self.state) {
4335 Ok(plan) => plan,
4336 Err(skip) => {
4337 if skip != graft::Skip::AlreadyDone {
4338 self.graft_skipped(skip.reason());
4339 self.state.save()?;
4340 }
4341 return Ok(());
4342 }
4343 };
4344 let Some(winner) = self.state.winner().cloned() else {
4345 return Ok(());
4346 };
4347 let wt = winner.worktree.clone();
4348 if !git::is_clean(&wt).await? {
4349 self.graft_skipped("the winner's worktree has uncommitted changes");
4350 self.state.save()?;
4351 return Ok(());
4352 }
4353 let pre = git::rev_parse(&wt, "HEAD").await?;
4354 self.state.graft = Some(GraftRecord {
4357 pre: pre.clone(),
4358 post: None,
4359 lines: None,
4360 limit: None,
4361 agent: None,
4362 outcome: GraftOutcome::Running,
4363 requested: graft::requested(&plan),
4364 adoption: None,
4365 adoption_error: None,
4366 });
4367 self.state.event(
4368 "graft",
4369 format!(
4370 "porting strengths of candidate(s) {} onto {}",
4371 plan.strengths
4372 .iter()
4373 .map(|(l, _, _)| l.to_string())
4374 .collect::<Vec<_>>()
4375 .join(", "),
4376 plan.winner
4377 ),
4378 );
4379 self.state.save()?;
4380
4381 let fix_prompt = prompt::graft(
4382 &self.state.instruction,
4383 plan.winner,
4384 &plan.strengths,
4385 &self.state.config.graph.language,
4386 );
4387 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4388 let sessions = self.state.config.graph.sessions;
4389 let artifacts = agent::artifacts_dir(&self.state.dir());
4390 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4391 let (_, seat, out) = self
4392 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4393 SeatJob {
4394 prompt: fix_prompt.clone(),
4395 spec,
4396 seat,
4397 cwd: wt.clone(),
4398 timeout,
4399 allow_write: true,
4400 sessions,
4401 artifacts: artifacts.clone(),
4402 stem: "graft".to_owned(),
4403 handover: None,
4404 }
4405 })
4406 .await;
4407 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4408 self.state.graft.as_mut().expect("graft record").outcome =
4411 GraftOutcome::WaitingForRestore;
4412 return self.stop_for_resource();
4413 }
4414 let mut why: Option<String> = None;
4415 match &out {
4416 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4417 why = Some("the fixer produced nothing usable".to_owned());
4418 }
4419 AgentOutcome::Ok(_) => {}
4420 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4421 AgentOutcome::Quota(o) => {
4422 self.state.quota.push(QuotaLoss {
4423 seat: seat.key.clone(),
4424 node: "graft".to_owned(),
4425 at: Timestamp::now(),
4426 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4427 });
4428 why = Some("rate limited (quota)".to_owned());
4429 }
4430 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4431 why = Some(format!(
4432 "the fixer failed: {}",
4433 e.lines()
4434 .next()
4435 .unwrap_or("")
4436 .chars()
4437 .take(160)
4438 .collect::<String>()
4439 ));
4440 }
4441 }
4442 let agent_id = seat.agent.clone();
4443 self.state.seats.insert(seat.key.clone(), seat);
4444
4445 let mut post = None;
4446 let mut lines = None;
4447 let _coordination = self.coordinate(&wt).await?;
4448 if why.is_none() {
4449 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4450 if let Ok(r) = &rescued {
4451 self.state.note_withheld("graft", &r.withheld);
4452 }
4453 let head = git::rev_parse(&wt, "HEAD").await?;
4454 let dirty = match &rescued {
4458 Ok(r) => git::status_porcelain(&wt)
4459 .await
4460 .map(|st| {
4461 st.lines().any(|l| {
4462 !r.withheld
4463 .iter()
4464 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4465 })
4466 })
4467 .unwrap_or(true),
4468 Err(_) => true,
4469 };
4470 if dirty {
4471 why = Some("the fixer left changes that could not be committed".to_owned());
4472 } else if head == pre {
4473 why = Some("the fixer changed nothing".to_owned());
4474 } else {
4475 lines = git::changed_lines(&wt, &pre, &head).await;
4476 post = Some(head);
4477 }
4478 }
4479 let (adoption, adoption_error) = match &out {
4480 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4481 _ => (None, None),
4482 };
4483 let requested = graft::requested(&plan);
4484 let outcome = match why {
4485 None => {
4486 self.state.event(
4487 "graft",
4488 format!(
4489 "applied: {}",
4490 lines.map_or_else(
4491 || "changed lines unavailable".to_owned(),
4492 |n| format!("{n} changed lines")
4493 )
4494 ),
4495 );
4496 GraftOutcome::Applied
4497 }
4498 Some(why) => {
4499 crate::recovery::restore_repair(&wt, &pre)
4502 .await
4503 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4504 self.state.event(
4505 "graft",
4506 format!("rolled back: {why}; the winner is untouched"),
4507 );
4508 GraftOutcome::RolledBack(why)
4509 }
4510 };
4511 self.state.graft = Some(GraftRecord {
4512 pre,
4513 post,
4514 lines,
4515 limit: None,
4516 agent: Some(agent_id),
4517 outcome,
4518 requested,
4519 adoption,
4520 adoption_error,
4521 });
4522 self.state.save()?;
4523 Ok(())
4524 }
4525
4526 async fn fold_losers(&mut self) -> Result<()> {
4529 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4530 return Ok(());
4531 };
4532 if !self.state.candidate_preservation.is_empty() {
4533 bail!("candidate preservation incomplete; refusing cleanup");
4534 }
4535 let repo = self.state.repo.clone();
4536 let _coordination = self.coordinate(&repo).await?;
4537 let mut folded = Vec::new();
4538 for i in 0..self.state.candidates.len() {
4539 let c = &self.state.candidates[i];
4540 if c.label == winner || c.folded {
4541 continue;
4542 }
4543 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4544 remove_preserved_worktree(&repo, &wt).await?;
4545 if git::branch_exists(&repo, &branch).await?
4546 && !git::branch_delete(&repo, &branch).await?
4547 {
4548 bail!("candidate branch removal refused");
4549 }
4550 self.state.candidates[i].folded = true;
4551 self.state.save()?;
4552 folded.push(label.to_string());
4553 }
4554 let root = self.state.worktree_root();
4556 for j in 1..=self.roles.judges.len() {
4557 let wt = root.join(format!("judge-{j}"));
4558 if wt.exists() {
4559 remove_preserved_worktree(&repo, &wt).await?;
4560 }
4561 }
4562 if self.state.config.graph.advise {
4565 for k in 1..=self.state.config.graph.advisors {
4566 let wt = root.join(format!("advisor-{k}"));
4567 if wt.exists() {
4568 remove_preserved_worktree(&repo, &wt).await?;
4569 }
4570 }
4571 }
4572 if !folded.is_empty() {
4573 self.state
4574 .event("fold", format!("folded candidates {}", folded.join(", ")));
4575 self.state.save()?;
4576 }
4577 Ok(())
4578 }
4579
4580 async fn sync_to_base(&mut self) -> Result<()> {
4620 if self.state.status == RunStatus::AlreadyInBase {
4621 return Ok(());
4622 }
4623 let conflicted = self
4624 .state
4625 .base_sync
4626 .as_ref()
4627 .is_some_and(|s| s.conflict.is_some());
4628 let Some(winner) = self.state.winner().cloned() else {
4629 return Ok(());
4630 };
4631
4632 let repo = self.state.repo.clone();
4633 let remote = self.state.config.merge.remote.clone();
4634 let base_branch = self.state.base_branch.clone();
4635 let tracking = format!("{remote}/{base_branch}");
4636
4637 git::fetch(&repo, &remote, &base_branch).await.ok();
4638 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4643 return Ok(());
4644 };
4645
4646 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4647 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4648 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4649
4650 if (behind > 0 || conflicted || head == tip)
4659 && self
4660 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4661 .await?
4662 {
4663 return Ok(());
4664 }
4665 if conflicted {
4666 return Ok(());
4667 }
4668
4669 if behind == 0 {
4670 if let Some(from) = self
4679 .state
4680 .rebase_fixes
4681 .iter()
4682 .rev()
4683 .find_map(|r| r.from.clone())
4684 && from != head
4685 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4686 .await
4687 .is_ok_and(|o| o.ok())
4688 {
4689 git::sync_to_head(&winner.worktree).await?;
4690 }
4691 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4697 if let Some(why) = &conflict {
4698 self.state.status = RunStatus::Blocked;
4699 self.state.event("land", why.clone());
4700 }
4701 self.state.base_sync = Some(BaseSync {
4702 tip,
4703 behind: 0,
4704 attempts,
4705 conflict,
4706 already_in: None,
4707 });
4708 self.state.save()?;
4709 return Ok(());
4710 }
4711
4712 if attempts >= BASE_SYNC_ROUNDS {
4713 let why = format!(
4714 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4715 rebase(s); rebasing again would only race it",
4716 winner.branch
4717 );
4718 self.state.status = RunStatus::Blocked;
4719 self.state.base_sync = Some(BaseSync {
4720 tip,
4721 behind,
4722 attempts,
4723 conflict: Some(why.clone()),
4724 already_in: None,
4725 });
4726 self.state.event("land", why);
4727 self.state.save()?;
4728 return Ok(());
4729 }
4730
4731 self.state.event(
4732 "land",
4733 format!(
4734 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4735 winner.branch
4736 ),
4737 );
4738 self.state.save()?;
4739
4740 let branch_tracking = format!("{remote}/{}", winner.branch);
4746 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4747 let remote_tip = if self.state.task_pr_branch().is_none()
4748 && matches!(&fetched_branch, Ok(o) if o.ok())
4749 {
4750 git::rev_parse(&repo, &branch_tracking).await.ok()
4751 } else {
4752 None
4753 };
4754 if let Some(theirs) = &remote_tip
4758 && !git::is_ancestor(&repo, theirs, &head).await
4759 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4760 .await
4761 .is_ok_and(|missing| missing.is_empty())
4762 {
4763 let why = format!(
4764 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4765 them",
4766 short(theirs),
4767 winner.branch
4768 );
4769 self.state.status = RunStatus::Blocked;
4770 self.state.base_sync = Some(BaseSync {
4771 tip,
4772 behind,
4773 attempts,
4774 conflict: Some(why.clone()),
4775 already_in: None,
4776 });
4777 self.state.event("land", why);
4778 self.state.save()?;
4779 return Ok(());
4780 }
4781
4782 let scratch = self.state.dir().join("base-sync");
4783 let rebased = match crate::rebase::rebase_with_fixer(
4784 &mut self.state,
4785 &scratch,
4786 &winner.branch,
4787 &tracking,
4788 )
4789 .await
4790 {
4791 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4792 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4793 Err(e) => Err(e),
4794 };
4795 let attempts = attempts + 1;
4796 match rebased {
4797 Ok(None) => {
4798 git::sync_to_head(&winner.worktree).await?;
4802 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4803 let mut conflict = None;
4804 if let Some(pinned) = &remote_tip {
4805 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4806 match pushed {
4807 Ok(o) if o.ok() => self.state.event(
4808 "land",
4809 format!("pushed rebased {} to {remote}", winner.branch),
4810 ),
4811 Ok(o) => {
4812 conflict = Some(format!(
4813 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4814 winner.branch,
4815 short(pinned),
4816 o.stderr.chars().take(600).collect::<String>()
4817 ));
4818 }
4819 Err(e) => {
4820 conflict = Some(format!(
4821 "rebased {} locally but could not push it: {e:#}",
4822 winner.branch
4823 ));
4824 }
4825 }
4826 }
4827 if let Some(why) = &conflict {
4828 self.state.status = RunStatus::Blocked;
4829 self.state.event("land", why.clone());
4830 }
4831 self.state.base_sync = Some(BaseSync {
4832 tip: tip.clone(),
4833 behind: 0,
4834 attempts,
4835 conflict,
4836 already_in: None,
4837 });
4838 self.state
4839 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4840 }
4841 Ok(Some(conflict)) => {
4842 let why = format!(
4843 "{} conflicts with {tracking} and did not rebase: {}",
4844 winner.branch,
4845 conflict.chars().take(600).collect::<String>()
4846 );
4847 self.state.status = RunStatus::Blocked;
4848 self.state.base_sync = Some(BaseSync {
4849 tip,
4850 behind,
4851 attempts,
4852 conflict: Some(why.clone()),
4853 already_in: None,
4854 });
4855 self.state.event("land", why);
4856 }
4857 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4861 self.state.save()?;
4862 return Ok(());
4863 }
4864 Err(e) => {
4865 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4866 self.state.status = RunStatus::Blocked;
4867 self.state.base_sync = Some(BaseSync {
4868 tip,
4869 behind,
4870 attempts,
4871 conflict: Some(why.clone()),
4872 already_in: None,
4873 });
4874 self.state.event("land", why);
4875 }
4876 }
4877 self.state.save()?;
4878 Ok(())
4879 }
4880
4881 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4887 if self.state.task_pr_branch().is_some() {
4890 return None;
4891 }
4892 let repo = self.state.repo.clone();
4893 let remote = self.state.config.merge.remote.clone();
4894 let fetched = git::fetch(&repo, &remote, branch).await;
4895 if !matches!(&fetched, Ok(o) if o.ok()) {
4896 return None;
4897 }
4898 let branch_tracking = format!("{remote}/{branch}");
4899 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4900 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4901 return None;
4902 }
4903 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4904 .await
4905 .is_ok_and(|missing| missing.is_empty())
4906 {
4907 return Some(format!(
4908 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4909 them",
4910 short(&theirs)
4911 ));
4912 }
4913 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4914 Ok(o) if o.ok() => {
4915 self.state
4916 .event("land", format!("pushed rebased {branch} to {remote}"));
4917 None
4918 }
4919 Ok(o) => Some(format!(
4920 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4921 someone may have pushed): {}",
4922 short(&theirs),
4923 o.stderr.chars().take(600).collect::<String>()
4924 )),
4925 Err(e) => Some(format!(
4926 "{branch} is rebased locally but could not be pushed: {e:#}"
4927 )),
4928 }
4929 }
4930
4931 async fn settle_already_in(
4948 &mut self,
4949 branch: &str,
4950 tip: &str,
4951 head: &str,
4952 attempts: usize,
4953 behind: usize,
4954 ) -> Result<bool> {
4955 let repo = self.state.repo.clone();
4956 let remote = self.state.config.merge.remote.clone();
4957 let start = self.state.base_commit.clone();
4958 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4959 Ok(Some(e)) => e,
4960 Ok(None) => return Ok(false),
4961 Err(e) => {
4962 tracing::warn!("already-in-base check for {branch}: {e:#}");
4963 return Ok(false);
4964 }
4965 };
4966 let mut verified = vec![head.to_owned()];
4967 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4968 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4969 if matches!(&fetched, Ok(o) if o.ok())
4970 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4971 && theirs != head
4972 {
4973 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4974 Ok(Some(_)) => verified.push(theirs),
4975 _ => return Ok(false),
4976 }
4977 }
4978 let base_branch = self.state.base_branch.clone();
4979 let message = format!(
4980 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4981 land",
4982 evidence.names(),
4983 evidence.proof.as_str()
4984 );
4985 let closed =
4986 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4987 .await;
4988 match closed {
4989 Ok(Ok(url)) => self
4990 .state
4991 .event("land", format!("closed {url}: superseded on {base_branch}")),
4992 Ok(Err(why)) => self
4993 .state
4994 .event("land", format!("did not close a pull request: {why}")),
4995 Err(e) => {
4996 let why = format!(
4997 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4998 not be closed ({e:#}); resume to retry"
4999 );
5000 self.state.status = RunStatus::Blocked;
5001 self.state.base_sync = Some(BaseSync {
5002 tip: tip.to_owned(),
5003 behind,
5004 attempts,
5005 conflict: Some(why.clone()),
5006 already_in: None,
5007 });
5008 self.state.event("land", why);
5009 self.state.save()?;
5010 return Ok(true);
5011 }
5012 }
5013 self.state.status = RunStatus::AlreadyInBase;
5014 self.state.base_sync = Some(BaseSync {
5015 tip: tip.to_owned(),
5016 behind,
5017 attempts,
5018 conflict: None,
5019 already_in: Some(evidence),
5020 });
5021 self.state.event("land", message);
5022 self.state.save()?;
5023 self.settle_questions();
5024 Ok(true)
5025 }
5026
5027 fn landing_base(&self) -> String {
5037 self.state
5038 .base_sync
5039 .as_ref()
5040 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5041 }
5042
5043 pub async fn fix_selected(
5076 &mut self,
5077 ids: &[String],
5078 reason: &str,
5079 allow_stale: bool,
5080 ) -> Result<()> {
5081 let reason = reason.trim();
5082 if reason.is_empty() {
5083 bail!("a fix request needs a reason — that is the operator's own record of why");
5084 }
5085 if ids.is_empty() {
5086 bail!("no finding id given");
5087 }
5088 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5089 bail!(
5090 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5091 has already concluded — can be given a targeted fix. A run still \
5092 in progress should simply be resumed; a `merged` run's branch has \
5093 already landed, so its answer is a fresh `magi review <branch>`, \
5094 not reopening this run's own record",
5095 self.state.id,
5096 self.state.status.as_str()
5097 );
5098 }
5099 let Some(winner) = self.state.winner().cloned() else {
5100 bail!("run {} has no winning candidate to fix", self.state.id);
5101 };
5102 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5103 bail!(
5104 "branch `{}` no longer exists; this run cannot be extended",
5105 winner.branch
5106 );
5107 }
5108 let home = crate::run::home();
5109 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5110 bail!(
5111 "run {} is currently being worked on by another magi process",
5112 self.state.id
5113 );
5114 }
5115 let _claim = FixClaim::acquire(&self.state.dir())?;
5121
5122 let mut seen = BTreeSet::new();
5126 let mut findings = Vec::new();
5127 let mut missing = Vec::new();
5128 for id in ids {
5129 if !seen.insert(id.clone()) {
5130 continue;
5131 }
5132 match self.state.finding(id) {
5133 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5134 id: f.id.clone(),
5135 severity: f.severity,
5136 reviewer_vote: rec.vote,
5137 round: round.round,
5138 round_head: round.head.clone(),
5139 reviewer: rec.reviewer,
5140 agent: rec.agent.clone(),
5141 file: f.file.clone(),
5142 line: f.line,
5143 title: f.title.clone(),
5144 detail: f.detail.clone(),
5145 outcome: OperatorFixOutcome::Pending,
5146 }),
5147 None => missing.push(id.clone()),
5148 }
5149 }
5150 if !missing.is_empty() {
5151 bail!(
5152 "unknown finding id(s): {}; nothing was changed",
5153 missing.join(", ")
5154 );
5155 }
5156
5157 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5158 let stale_details: Vec<(String, String)> = findings
5159 .iter()
5160 .filter(|f| f.round_head != head_at_request)
5161 .map(|f| (f.id.clone(), f.round_head.clone()))
5162 .collect();
5163 let stale = !stale_details.is_empty();
5164 if stale && !allow_stale {
5165 bail!(
5166 "the branch has moved since some finding(s) were raised — {} — now \
5167 at {}; pass --allow-stale to fix anyway, or re-run review first",
5168 stale_details
5169 .iter()
5170 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5171 .collect::<Vec<_>>()
5172 .join(", "),
5173 short(&head_at_request)
5174 );
5175 }
5176
5177 let request = OperatorFixRequest {
5178 requested_at: Timestamp::now(),
5179 reason: reason.to_owned(),
5180 findings,
5181 head_at_request: head_at_request.clone(),
5182 allow_stale,
5183 stale,
5184 fix: None,
5185 result_head: None,
5186 follow_up_review_run: None,
5187 };
5188 self.state.event(
5189 "fix",
5190 format!(
5191 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5192 request.findings.len(),
5193 request
5194 .findings
5195 .iter()
5196 .map(|f| f.id.as_str())
5197 .collect::<Vec<_>>()
5198 .join(", "),
5199 ),
5200 );
5201 self.state.operator_fixes.push(request);
5208 self.state.save()?;
5209 let request_index = self.state.operator_fixes.len() - 1;
5210
5211 let repo = self.state.repo.clone();
5220 let coordination = self.coordinate(&repo).await?;
5221 if winner.worktree.exists() {
5222 let dirty = git::git(
5225 &winner.worktree,
5226 &["status", "--porcelain", "--untracked-files=all"],
5227 )
5228 .await?;
5229 let only_withheld = dirty.lines().all(|l| {
5230 l.strip_prefix("?? ")
5231 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5232 });
5233 if !only_withheld {
5234 bail!(
5235 "`{}` has uncommitted changes; refusing to touch it — commit or \
5236 discard them first",
5237 winner.worktree.display()
5238 );
5239 }
5240 git::worktree_remove(&self.state.repo, &winner.worktree)
5241 .await
5242 .ok();
5243 }
5244 let fix_worktree = self.state.worktree_root().join("operator-fix");
5245 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5246 git::git(
5247 &self.state.repo,
5248 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5249 )
5250 .await
5251 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5252 if !git::is_clean(&fix_worktree).await? {
5253 git::worktree_remove(&self.state.repo, &fix_worktree)
5254 .await
5255 .ok();
5256 bail!(
5257 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5258 winner.branch
5259 );
5260 }
5261 let repo_for_setup = self.state.repo.clone();
5262 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5263 Ok(()) => {
5264 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5265 }
5266 Err(e) => Err(e),
5267 };
5268 if let Err(e) = setup {
5269 git::worktree_remove(&self.state.repo, &fix_worktree)
5270 .await
5271 .ok();
5272 return Err(e);
5273 }
5274
5275 let run_id = self.state.id.clone();
5276 let prompts = self.state.config.prompts.clone();
5277 let language = self.state.config.graph.language.clone();
5278 let sessions = self.state.config.graph.sessions;
5279 let artifacts = agent::artifacts_dir(&self.state.dir());
5280 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5281 .findings
5282 .iter()
5283 .map(|f| Finding {
5284 id: f.id.clone(),
5285 severity: f.severity,
5286 file: f.file.clone(),
5287 line: f.line,
5288 title: f.title.clone(),
5289 detail: f.detail.clone(),
5290 })
5291 .collect();
5292 let fix_prompt = prompt::operator_fix_in(
5293 &self.state.instruction,
5294 &finding_list,
5295 reason,
5296 &stale_details,
5297 &head_at_request,
5298 &language,
5299 &self.state.config.graph.github_language,
5300 );
5301 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5302 drop(coordination);
5303 let (job, seat, out) = self
5304 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5305 prompt: fix_prompt.clone(),
5306 spec,
5307 seat,
5308 cwd: fix_worktree.clone(),
5309 timeout,
5310 allow_write: true,
5311 sessions,
5312 artifacts: artifacts.clone(),
5313 stem: "operator-fix".to_owned(),
5314 handover: None,
5315 })
5316 .await;
5317 let agent_id = seat.agent.clone();
5318
5319 let mut fix = FixRecord {
5320 agent: agent_id,
5321 addressed: Vec::new(),
5322 rejected: Vec::new(),
5323 notes: String::new(),
5324 committed: false,
5325 failed: None,
5326 duration_ms: 0,
5327 continuation: None,
5328 };
5329 let mut final_seat = seat.clone();
5330 match out {
5331 AgentOutcome::Ok(o) => {
5332 fix.duration_ms = o.duration_ms;
5333 let parsed = verdict::extract_json::<FixReport>(&o.text);
5334 let incomplete_reason = match &parsed {
5335 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5336 "the reply parsed, but it reported a command whose own CLI \
5337 never confirmed an exit status"
5338 .to_owned(),
5339 ),
5340 Ok(_) => None,
5341 Err(e) => Some(e.to_string()),
5342 };
5343 match incomplete_reason {
5344 None => {
5345 let report = parsed.expect("checked Ok above");
5346 fix.addressed = report.addressed;
5347 fix.rejected = report.rejected;
5348 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5349 }
5350 Some(reason) => {
5351 let (resumed_seat, resolved, failure, cont) = self
5352 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5353 .await;
5354 fix.duration_ms += cont.cumulative_wait_ms;
5355 fix.continuation = Some(cont);
5356 final_seat = resumed_seat;
5357 match resolved {
5358 Some(report) => {
5359 fix.addressed = report.addressed;
5360 fix.rejected = report.rejected;
5361 fix.notes =
5362 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5363 }
5364 None => fix.failed = failure,
5365 }
5366 }
5367 }
5368 }
5369 AgentOutcome::Dropped(o) => {
5370 fix.duration_ms = o.duration_ms;
5371 let why = o
5372 .dropped
5373 .as_ref()
5374 .map(|d| d.why.as_str())
5375 .unwrap_or("the CLI ended the stream without delivering its answer");
5376 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5377 }
5378 AgentOutcome::Quota(o) => {
5379 self.state.quota.push(QuotaLoss {
5380 seat: final_seat.key.clone(),
5381 node: "fix".to_owned(),
5382 at: Timestamp::now(),
5383 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5384 });
5385 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5386 }
5387 AgentOutcome::Failed(e) => fix.failed = Some(e),
5388 AgentOutcome::ResourceBlocked(e) => {
5389 fix.failed = Some(format!("the fixer was not started: {e}"));
5390 }
5391 }
5392 if fix.continuation.is_none() {
5393 fix.continuation = Some(ContinuationRecord::not_needed());
5394 }
5395 self.state.seats.insert(final_seat.key.clone(), final_seat);
5396
5397 let rescue_message = format!(
5398 "magi: operator-selected fix ({}) (uncommitted work)",
5399 self.state.operator_fixes[request_index]
5400 .findings
5401 .iter()
5402 .map(|f| f.id.as_str())
5403 .collect::<Vec<_>>()
5404 .join(", ")
5405 );
5406 let repo = self.state.repo.clone();
5407 let coordination = self.coordinate(&repo).await?;
5408 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5409 self.state.note_withheld("fix", &r.withheld);
5410 }
5411 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5412 fix.committed = after != head_at_request;
5413 git::worktree_remove(&self.state.repo, &fix_worktree)
5414 .await
5415 .ok();
5416 drop(coordination);
5417
5418 self.state.event(
5419 "fix",
5420 match &fix.failed {
5421 Some(reason) => format!(
5422 "operator fix: adoption report was lost ({reason}); {}",
5423 if fix.committed {
5424 "committed"
5425 } else {
5426 "NO new commit"
5427 }
5428 ),
5429 None => format!(
5430 "operator fix: {} addressed, {} rejected, {}",
5431 fix.addressed.len(),
5432 fix.rejected.len(),
5433 if fix.committed {
5434 "committed"
5435 } else {
5436 "NO new commit"
5437 }
5438 ),
5439 },
5440 );
5441
5442 for f in &mut self.state.operator_fixes[request_index].findings {
5449 f.outcome = if fix.failed.is_some() {
5450 OperatorFixOutcome::Unreported
5451 } else if fix.addressed.contains(&f.id) {
5452 OperatorFixOutcome::Addressed
5453 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5454 OperatorFixOutcome::Rejected { why: r.why.clone() }
5455 } else {
5456 OperatorFixOutcome::Unreported
5457 };
5458 }
5459
5460 let committed = fix.committed;
5461 if committed {
5462 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5463 }
5464 self.state.operator_fixes[request_index].fix = Some(fix);
5465 self.state.save()?;
5468
5469 if committed {
5470 self.state.event(
5471 "fix",
5472 format!(
5473 "operator fix committed {}; opening a follow-up review-only run",
5474 short(&after)
5475 ),
5476 );
5477 let origin =
5480 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5481 match Self::review(
5482 &self.state.repo,
5483 &winner.branch,
5484 self.state.config.clone(),
5485 origin,
5486 )
5487 .await
5488 {
5489 Ok(mut follow_up) => {
5490 follow_up.state.event(
5491 "start",
5492 format!(
5493 "requested by an operator fix on run {} for finding(s) {}",
5494 self.state.id,
5495 self.state.operator_fixes[request_index]
5496 .findings
5497 .iter()
5498 .map(|f| f.id.as_str())
5499 .collect::<Vec<_>>()
5500 .join(", "),
5501 ),
5502 );
5503 follow_up.state.save()?;
5504 let follow_up_id = follow_up.state.id.clone();
5505 let adopted = match crate::direct::adopt(
5508 &follow_up.state,
5509 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5510 ) {
5511 Ok(a) => a,
5512 Err(e) => {
5513 self.state.event(
5516 "fix",
5517 format!(
5518 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5519 ),
5520 );
5521 self.state.save()?;
5522 return Ok(());
5523 }
5524 };
5525 let executed = follow_up.execute().await;
5526 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5527 if let Some(a) = adopted {
5528 a.finish(&follow_up.state, executed);
5529 }
5530 if let Some(e) = failure {
5531 self.state.event(
5532 "fix",
5533 format!(
5534 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5535 ),
5536 );
5537 }
5538 self.state.operator_fixes[request_index].follow_up_review_run =
5539 Some(follow_up_id);
5540 }
5541 Err(e) => {
5542 self.state.event(
5543 "fix",
5544 format!("committed the fix but could not open a follow-up review: {e:#}"),
5545 );
5546 }
5547 }
5548 self.state.save()?;
5549 }
5550
5551 Ok(())
5552 }
5553
5554 async fn ask_fixer(
5574 &mut self,
5575 winner: &Candidate,
5576 node: &'static str,
5577 round: Option<usize>,
5578 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5579 ) -> (SeatJob, SeatState, AgentOutcome) {
5580 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5581 self.ask_fixer_among(attempts, node, round, None, build)
5582 .await
5583 }
5584
5585 async fn ask_fixer_among(
5590 &mut self,
5591 attempts: Vec<(AgentSpec, String)>,
5592 node: &'static str,
5593 round: Option<usize>,
5594 restore: Option<(&Path, &str)>,
5595 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5596 ) -> (SeatJob, SeatState, AgentOutcome) {
5597 let run_id = self.state.id.clone();
5598 let prompts = self.state.config.prompts.clone();
5599 let cache = self.state.config.cache_dir();
5600 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5601 let mut last = None;
5602 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5603 let seat = self.seat(&key, &spec.id);
5604 let mut job = build(spec.clone(), seat);
5605 if i > 0 {
5606 job.stem = format!("{}-{}", job.stem, spec.id);
5607 }
5608 let ctx = WaveCtx {
5609 carry_seats: false,
5610 pauses: Some((&self.pause, &self.interrupt)),
5611 run: &run_id,
5612 node,
5613 prompts: &prompts,
5614 cache: cache.as_deref(),
5615 round,
5616 };
5617 let (seat, out) =
5618 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5619 let advances = match &out {
5622 AgentOutcome::Ok(o) => agent::output_advances(o),
5623 AgentOutcome::ResourceBlocked(_) => false,
5624 _ => true,
5625 };
5626 if let Some(next) = ids.get(i + 1)
5627 && advances
5628 {
5629 self.state.seats.insert(seat.key.clone(), seat.clone());
5630 let class =
5631 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5632 let reason = match &out {
5633 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5634 other => fail_reason(other),
5635 };
5636 if let Some((wt, rev)) = restore
5640 && let Err(e) = self.graft_restore(wt, rev).await
5641 {
5642 let why =
5643 format!("could not restore the worktree before the next fixer: {e:#}");
5644 tracing::warn!("{why}");
5645 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5646 AgentOutcome::ResourceBlocked(why)
5647 } else {
5648 AgentOutcome::Failed(why)
5649 };
5650 last = Some((job, seat, out));
5651 break;
5652 }
5653 record_handover(
5654 &mut self.state,
5655 node,
5656 &seat.key,
5657 &spec.id,
5658 next,
5659 &class,
5660 &reason,
5661 );
5662 continue;
5663 }
5664 last = Some((job, seat, out));
5665 break;
5666 }
5667 last.expect("the fixer chain always has an entry")
5668 }
5669
5670 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5681 use crate::recovery::{Mode, Repair};
5682 if !self.state.reviews.is_empty() {
5683 return Ok(());
5684 }
5685 let Some(mut rec) = self.state.recovery.clone() else {
5686 return Ok(());
5687 };
5688 if rec.mode != Mode::VerifyRepair {
5689 return Ok(());
5690 }
5691 let wt = winner.worktree.clone();
5692 let mut head = git::rev_parse(&wt, "HEAD").await?;
5693 match rec.repair.clone() {
5694 Repair::Done { .. } => return Ok(()),
5695 Repair::WaitingForRestore { before } => {
5696 self.graft_restore(&wt, &before).await?;
5697 head = before;
5698 rec.repair = Repair::NotStarted;
5699 self.state.recovery = Some(rec.clone());
5700 self.state.save()?;
5701 }
5702 Repair::Started { before } => {
5703 if let Err(e) = self.graft_restore(&wt, &before).await {
5707 if e.is::<crate::repo_lock::ResourceBlocked>() {
5708 return Err(e);
5709 }
5710 return self
5711 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5712 .await;
5713 }
5714 rec.repair = Repair::Done {
5715 before: before.clone(),
5716 after: before,
5717 committed: false,
5718 failed: Some("interrupted; rolled back".to_owned()),
5719 };
5720 self.state
5721 .event("recovery", "the repair was interrupted and rolled back");
5722 self.state.recovery = Some(rec);
5723 self.state.save()?;
5724 return Ok(());
5725 }
5726 Repair::NotStarted => {}
5727 }
5728 if crate::recovery::recheck(&mut rec, &head) {
5729 self.state.event("recovery", rec.summary());
5730 self.state.recovery = Some(rec);
5731 self.state.save()?;
5732 return Ok(());
5733 }
5734 rec.repair = Repair::Started {
5735 before: head.clone(),
5736 };
5737 self.state.recovery = Some(rec.clone());
5738 self.state.save()?;
5739
5740 let language = self.state.config.graph.language.clone();
5741 let sessions = self.state.config.graph.sessions;
5742 let artifacts = agent::artifacts_dir(&self.state.dir());
5743 let fix_prompt = prompt::recovery_fix(
5744 &self.state.instruction,
5745 &rec,
5746 self.state.config.graph.review_rounds,
5747 &language,
5748 );
5749 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5750 self.state.event(
5751 "recovery",
5752 format!(
5753 "repair: the fixer starts from the failed verification on {}",
5754 short(&head)
5755 ),
5756 );
5757 let (_job, seat, out) = self
5758 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5759 prompt: fix_prompt.clone(),
5760 spec,
5761 seat,
5762 cwd: wt.clone(),
5763 timeout,
5764 allow_write: true,
5765 sessions,
5766 artifacts: artifacts.clone(),
5767 stem: "fix-recovery".to_owned(),
5768 handover: None,
5769 })
5770 .await;
5771 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5772 rec.repair = Repair::WaitingForRestore { before: head };
5773 self.state.recovery = Some(rec);
5774 return self.stop_for_resource();
5775 }
5776 let failed = match &out {
5777 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5778 AgentOutcome::Quota(o) => {
5779 self.state.quota.push(QuotaLoss {
5780 seat: seat.key.clone(),
5781 node: "fix".to_owned(),
5782 at: Timestamp::now(),
5783 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5784 });
5785 Some("rate limited (quota); the fixer could not run".to_owned())
5786 }
5787 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5788 AgentOutcome::Failed(e) => Some(e.clone()),
5789 };
5790 self.state.seats.insert(seat.key.clone(), seat);
5791 let coordination = self.coordinate(&wt).await?;
5792 let failed = match failed {
5793 Some(why) => {
5795 if let Err(e) = git::restore_to(&wt, &head).await {
5796 return self
5797 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5798 .await;
5799 }
5800 Some(why)
5801 }
5802 None => {
5803 if let Ok(r) = git::rescue_commit(
5804 &wt,
5805 "magi: recovery repair of the failed verification (uncommitted work)",
5806 )
5807 .await
5808 {
5809 self.state.note_withheld("fix", &r.withheld);
5810 }
5811 None
5812 }
5813 };
5814 let after = git::rev_parse(&wt, "HEAD").await?;
5815 drop(coordination);
5816 self.state.event(
5817 "recovery",
5818 match &failed {
5819 Some(why) => format!("repair: nothing kept ({why})"),
5820 None if after != head => {
5821 format!("repair: committed {}", short(&after))
5822 }
5823 None => "repair: the fixer made no commit".to_owned(),
5824 },
5825 );
5826 rec.repair = Repair::Done {
5827 before: head.clone(),
5828 committed: after != head,
5829 after,
5830 failed,
5831 };
5832 self.state.recovery = Some(rec);
5833 self.state.save()?;
5834 Ok(())
5835 }
5836
5837 async fn stop_recovery(
5840 &mut self,
5841 mut rec: crate::recovery::RecoveryRecord,
5842 head: &str,
5843 why: String,
5844 ) -> Result<()> {
5845 rec.repair = crate::recovery::Repair::Started {
5847 before: head.to_owned(),
5848 };
5849 self.state.recovery = Some(rec);
5850 self.state
5851 .event("recovery", format!("repair could not be undone: {why}"));
5852 self.state.status = RunStatus::Blocked;
5853 self.state.save()?;
5854 bail!("recovery repair could not be rolled back: {why}")
5855 }
5856
5857 async fn review_loop(&mut self) -> Result<()> {
5858 if self
5863 .state
5864 .base_sync
5865 .as_ref()
5866 .is_some_and(|s| s.conflict.is_some())
5867 {
5868 return Ok(());
5869 }
5870 let run_id = self.state.id.clone();
5875 let prompts = self.state.config.prompts.clone();
5876 let Some(winner) = self.state.winner().cloned() else {
5877 return Ok(());
5878 };
5879 let max_rounds = self.state.config.graph.review_rounds;
5880 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5890 if status == RunStatus::Gating {
5893 self.record_contested_handoff();
5894 }
5895 self.state.status = status;
5896 self.state.save()?;
5897 return Ok(());
5898 }
5899 self.state.status = RunStatus::Reviewing;
5900 if self
5910 .state
5911 .reviews
5912 .last()
5913 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5914 {
5915 let shell = self.state.config.shell();
5916 return self
5917 .stop_reviewing(
5918 "the last round's own verification never resolved",
5919 &shell,
5920 &winner.worktree,
5921 )
5922 .await;
5923 }
5924
5925 self.recovery_repair(&winner).await?;
5926 if self.resource_halt()? {
5927 return Ok(());
5928 }
5929
5930 let repo = self.state.repo.clone();
5931 let root = self.state.worktree_root();
5932 let language = self.state.config.graph.language.clone();
5933 let sessions = self.state.config.graph.sessions;
5934 let artifacts = self.merge_repair_artifacts();
5935 let base = self.landing_base();
5936 let base_short = short(&base);
5937 let reviewers = self.roles.reviewers.clone();
5938 let shell = self.state.config.shell();
5939
5940 for round in (self.state.reviews.len() + 1)..=max_rounds {
5941 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5942 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5943 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5944 let prev_verification = self
5953 .state
5954 .reviews
5955 .last()
5956 .and_then(|r| r.verification_summary(&head));
5957 let prior_section = match (&self.state.recovery, round) {
5960 (Some(rec), 1) if self.state.reviews.is_empty() => {
5961 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5962 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5963 Some(crate::recovery::head_of(
5964 &git::diff(&winner.worktree, reviewed, "HEAD")
5965 .await
5966 .unwrap_or_default(),
5967 crate::recovery::SINCE_DIFF_MAX,
5968 ))
5969 }
5970 _ => None,
5971 };
5972 Some(prompt::recovery_review(rec, since.as_deref()))
5973 }
5974 _ => None,
5975 };
5976
5977 let mut jobs = Vec::new();
5981 let coordination = self.coordinate(&repo).await?;
5982 for (r, spec) in reviewers.iter().cloned().enumerate() {
5983 let wt = root.join(format!("review-{}", r + 1));
5984 if wt.exists() {
5987 git::reset_detached(&wt, &head).await?;
5988 } else {
5989 git::worktree_add_detached(&repo, &wt, &head).await?;
5990 }
5991 self.setup_seat_worktree(&repo, &wt, false).await?;
5992 let seat_key = format!("review-{}", r + 1);
5993 let spec = pick_start_spec(
5996 &self.roles.reviewer_roster,
5997 spec,
5998 self.state.seat_history.get(&seat_key),
5999 );
6000 let seat = self.seat(&seat_key, &spec.id);
6001 jobs.push(SeatJob {
6002 prompt: prompt::review(&prompt::ReviewCtx {
6003 instruction: &self.state.instruction,
6004 branch: &winner.branch,
6005 base_short: &base_short,
6006 stat: &stat,
6007 patch: &patch,
6008 verification: prev_verification.as_ref(),
6009 prior: prior_section.as_deref(),
6010 reviewers: reviewers.len(),
6011 round,
6012 rounds: max_rounds,
6013 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6016 lens: Lens::for_seat(r),
6017 language: &language,
6018 github_language: &self.state.config.graph.github_language,
6019 }),
6020 spec,
6021 seat,
6022 cwd: wt,
6023 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6024 allow_write: false,
6025 sessions,
6026 artifacts: artifacts.clone(),
6027 stem: format!("review-{round}-{}", r + 1),
6028 handover: None,
6029 });
6030 }
6031
6032 drop(coordination);
6033 self.state.event(
6034 "review",
6035 format!(
6036 "round {round}: {} reviewers on {}",
6037 jobs.len(),
6038 short(&head)
6039 ),
6040 );
6041 let mut quota_losses = Vec::new();
6042 let review_retries = self.state.config.graph.retries;
6043 let review_cache = self.state.config.cache_dir();
6044 let ctx = WaveCtx {
6045 carry_seats: true,
6046 pauses: Some((&self.pause, &self.interrupt)),
6047 run: &run_id,
6048 node: "review",
6049 prompts: &prompts,
6050 cache: review_cache.as_deref(),
6051 round: Some(round),
6052 };
6053 let results = ask_json_wave::<Review>(
6054 jobs,
6055 Arc::clone(&self.sem),
6056 review_retries,
6057 &self.roles.reviewer_roster,
6058 &ctx,
6059 &mut quota_losses,
6060 &mut self.state,
6061 &|_: &Review| Ok(()),
6062 )
6063 .await;
6064 if self.state.resource_block.is_some() {
6065 return self.stop_for_resource();
6066 }
6067 let round_quota_missing = quota_losses.len();
6071 self.state.quota.extend(quota_losses);
6072
6073 let mut records = Vec::new();
6074 let mut all_findings = Vec::new();
6075 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6076 let agent_id = seat.agent.clone();
6077 self.state.seats.insert(seat.key.clone(), seat);
6078 let mut record = ReviewRecord {
6079 reviewer: r + 1,
6080 agent: agent_id,
6081 summary: String::new(),
6082 findings: Vec::new(),
6083 vote: None,
6084 failed: None,
6085 duration_ms: 0,
6086 attempts,
6092 };
6093 match res {
6094 Ok((review, out)) => {
6095 record.summary =
6103 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6104 record.vote = Some(review.vote);
6105 record.duration_ms = out.duration_ms;
6106 for (n, mut f) in review.findings.into_iter().enumerate() {
6107 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6110 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6111 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6112 f.file = f
6118 .file
6119 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6120 all_findings.push(f.clone());
6121 record.findings.push(f);
6122 }
6123 self.state.event(
6124 "review",
6125 format!(
6126 "round {round}: reviewer {} voted {} with {} finding(s)",
6127 r + 1,
6128 review.vote.label(),
6129 record.findings.len()
6130 ),
6131 );
6132 }
6133 Err(e) => {
6134 record.failed = Some(e.to_string());
6135 self.state.event(
6136 "review",
6137 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6138 );
6139 }
6140 }
6141 records.push(record);
6142 }
6143
6144 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6151 let vote_split =
6152 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6153 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6154 if vote_split {
6155 self.state.event(
6156 "review",
6157 format!(
6158 "round {round}: votes split ({}) — one round of reconsideration",
6159 initial_votes
6160 .iter()
6161 .map(|v| v.label())
6162 .collect::<Vec<_>>()
6163 .join(", ")
6164 ),
6165 );
6166 let panel: Vec<ReviewSeatReport<'_>> = records
6169 .iter()
6170 .filter_map(|r| {
6171 r.vote.map(|vote| ReviewSeatReport {
6172 reviewer: r.reviewer,
6173 vote,
6174 summary: &r.summary,
6175 findings: &r.findings,
6176 })
6177 })
6178 .collect();
6179
6180 let mut jobs = Vec::new();
6181 let mut seats_at = Vec::new();
6182 for (r, spec) in reviewers.iter().cloned().enumerate() {
6183 if records[r].vote.is_none() {
6187 continue;
6188 }
6189 let wt = root.join(format!("review-{}", r + 1));
6190 let seat_key = format!("review-{}", r + 1);
6191 let spec = self.occupant(&seat_key, spec);
6192 let seat = self.seat(&seat_key, &spec.id);
6193 let build = |with_patch: bool| {
6201 prompt::review_reconsider(&ReviewReconsiderCtx {
6202 instruction: &self.state.instruction,
6203 reviewer: r + 1,
6204 lens: Lens::for_seat(r),
6205 panel: &panel,
6206 patch: with_patch.then_some(ReviewPatch {
6207 branch: &winner.branch,
6208 base_short: &base_short,
6209 stat: &stat,
6210 patch: &patch,
6211 }),
6212 round,
6213 rounds: max_rounds,
6214 language: &language,
6215 })
6216 };
6217 let full = build(true);
6218 let prompt = if has_context(&spec, &seat, sessions) {
6219 build(false)
6220 } else {
6221 full.clone()
6222 };
6223 jobs.push(SeatJob {
6224 prompt,
6225 spec,
6226 seat,
6227 cwd: wt,
6228 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6229 allow_write: false,
6230 sessions,
6231 artifacts: artifacts.clone(),
6232 stem: format!("review-{round}-reconsider-{}", r + 1),
6233 handover: Some(full),
6234 });
6235 seats_at.push(r);
6236 }
6237
6238 let mut recon_quota_losses = Vec::new();
6239 let recon_cache = self.state.config.cache_dir();
6240 let recon_ctx = WaveCtx {
6241 carry_seats: true,
6242 pauses: Some((&self.pause, &self.interrupt)),
6243 run: &run_id,
6244 node: "review",
6245 prompts: &prompts,
6246 cache: recon_cache.as_deref(),
6247 round: Some(round),
6248 };
6249 let recon_results = ask_json_wave::<ReviewRevote>(
6250 jobs,
6251 Arc::clone(&self.sem),
6252 review_retries,
6253 &self.roles.reviewer_roster,
6254 &recon_ctx,
6255 &mut recon_quota_losses,
6256 &mut self.state,
6257 &|_: &ReviewRevote| Ok(()),
6258 )
6259 .await;
6260 if self.state.resource_block.is_some() {
6261 return self.stop_for_resource();
6262 }
6263 self.state.quota.extend(recon_quota_losses);
6264
6265 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6266 let agent_id = seat.agent.clone();
6267 self.state.seats.insert(seat.key.clone(), seat);
6268 let mut rec = ReviewRevoteRecord {
6269 reviewer: r + 1,
6270 agent: agent_id,
6271 vote: None,
6272 reason: String::new(),
6273 failed: None,
6274 };
6275 match res {
6276 Ok((rv, _)) => {
6277 rec.vote = Some(rv.vote);
6278 rec.reason =
6279 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6280 self.state.event(
6281 "review",
6282 format!(
6283 "round {round}: reviewer {} revoted {}",
6284 r + 1,
6285 rv.vote.label()
6286 ),
6287 );
6288 }
6289 Err(e) => {
6290 rec.failed = Some(e.to_string());
6291 self.state.event(
6292 "review",
6293 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6294 );
6295 }
6296 }
6297 reconsideration.push(rec);
6298 }
6299 } else if initial_votes.len() > 1 {
6300 self.state.event(
6301 "review",
6302 format!(
6303 "round {round}: votes agreed ({}) — no reconsideration",
6304 initial_votes[0].label()
6305 ),
6306 );
6307 }
6308
6309 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6310 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6311 let defer_e2e =
6322 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6323 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6324 let reason =
6325 format!("{blocking} blocking finding(s) already required a fix this round");
6326 self.state.event(
6327 "verify",
6328 format!(
6329 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6330 {}); it will run once a round has none left",
6331 short(&head)
6332 ),
6333 );
6334 (Vec::new(), false, true, Some(reason))
6335 } else {
6336 let e2e_commands = self.state.config.verify.e2e.clone();
6337 let cache_dir = self.state.config.cache_dir();
6338 let context = format!("round {round}");
6339 let wait = self.repository_wait();
6340 let (e2e, verify_retried) = with_cache_lease(
6341 &mut self.state,
6342 cache_dir.as_deref(),
6343 "e2e",
6344 "e2e",
6345 &winner.worktree,
6346 &head,
6347 verify_timeout,
6348 &context,
6349 &wait,
6350 |state, budget| {
6351 let shell = shell.clone();
6352 let e2e_commands = e2e_commands.clone();
6353 let worktree = winner.worktree.clone();
6354 let context = context.clone();
6355 async move {
6356 run_e2e_with_retry(
6357 state,
6358 &shell,
6359 &e2e_commands,
6360 &worktree,
6361 budget,
6362 &context,
6363 )
6364 .await
6365 }
6366 },
6367 )
6368 .await?;
6369 (e2e, verify_retried, false, None)
6370 };
6371
6372 let expected = records.len();
6373 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6374 let incomplete = answered < expected;
6375 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6376 let policy = self.state.config.graph.incomplete_review;
6377 let clean = round_is_clean(
6378 blocking,
6379 e2e_ok,
6380 answered,
6381 expected,
6382 round_quota_missing,
6383 policy,
6384 );
6385
6386 let mut round_record = ReviewRound {
6387 round,
6388 head: head.clone(),
6389 verified_head: None,
6390 verified_at: None,
6391 reviews: records,
6392 e2e,
6393 verify_retried,
6394 e2e_deferred,
6395 e2e_defer_reason,
6396 fix: None,
6397 blocking,
6398 answered,
6399 expected,
6400 clean,
6401 progressed: false,
6402 vote_split,
6403 reconsideration,
6404 verdict: None,
6405 };
6406 round_record.verdict = ReviewVote::worst(
6410 round_record
6411 .final_votes()
6412 .into_iter()
6413 .map(|(_, _, vote)| vote),
6414 );
6415 if !matches!(
6426 round_record.e2e_status(),
6427 E2eStatus::Deferred | E2eStatus::NotConfigured
6428 ) {
6429 round_record.verified_head = Some(head.clone());
6430 round_record.verified_at = Some(Timestamp::now());
6431 }
6432 let this_round_verification = round_record.verification_summary(&head);
6433
6434 if incomplete {
6435 let missing: Vec<String> = round_record
6436 .reviews
6437 .iter()
6438 .filter(|r| r.failed.is_some())
6439 .map(|r| format!("review-{}", r.reviewer))
6440 .collect();
6441 self.state.event(
6442 "review",
6443 format!(
6444 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6445 missing.join(", ")
6446 ),
6447 );
6448 }
6449
6450 if clean {
6451 self.state.event(
6452 "review",
6453 if incomplete && policy == IncompleteReviewPolicy::Warn {
6454 format!(
6455 "round {round}: clean (warn policy, incomplete panel) — no \
6456 blocking findings from the seats that answered, verification green"
6457 )
6458 } else if incomplete {
6459 format!(
6460 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6461 quorum) — no blocking findings from the seats that answered, \
6462 verification green",
6463 expected - answered
6464 )
6465 } else {
6466 format!("round {round}: clean — no blocking findings, verification green")
6467 },
6468 );
6469 self.state.reviews.push(round_record);
6470 self.state.status = RunStatus::Gating;
6471 self.state.save()?;
6472 return Ok(());
6473 }
6474
6475 if incomplete && blocking == 0 && e2e_ok {
6483 self.state.reviews.push(round_record);
6484 self.state.save()?;
6485 if round == max_rounds {
6486 self.state.status = RunStatus::Blocked;
6487 self.state.event(
6488 "review",
6489 format!(
6490 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6491 refusing to call it clean",
6492 expected - answered
6493 ),
6494 );
6495 return Ok(());
6496 }
6497 continue;
6498 }
6499
6500 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6512 self.state.reviews.push(round_record);
6513 return self
6514 .stop_reviewing(
6515 "the round's own verification could not run",
6516 &shell,
6517 &winner.worktree,
6518 )
6519 .await;
6520 }
6521
6522 if round == max_rounds {
6523 self.state.reviews.push(round_record);
6524 return self
6525 .stop_reviewing(
6526 &format!(
6527 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6528 ),
6529 &shell,
6530 &winner.worktree,
6531 )
6532 .await;
6533 }
6534
6535 let blocking_findings: Vec<_> = all_findings
6538 .iter()
6539 .filter(|f| f.severity.blocks())
6540 .cloned()
6541 .collect();
6542 let fix_prompt = prompt::fix_in(
6543 &self.state.instruction,
6544 &blocking_findings,
6545 this_round_verification.as_ref(),
6546 round,
6547 max_rounds,
6548 &language,
6549 &self.state.config.graph.github_language,
6550 );
6551 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6552 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6553 let (job, seat, out) = self
6554 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6555 prompt: fix_prompt.clone(),
6556 spec,
6557 seat,
6558 cwd: winner.worktree.clone(),
6559 timeout,
6560 allow_write: true,
6561 sessions,
6562 artifacts: artifacts.clone(),
6563 stem: format!("fix-{round}"),
6564 handover: None,
6565 })
6566 .await;
6567 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6570 return self.stop_for_resource();
6571 }
6572 let agent_id = seat.agent.clone();
6573
6574 let mut fix = FixRecord {
6575 agent: agent_id,
6576 addressed: Vec::new(),
6577 rejected: Vec::new(),
6578 notes: String::new(),
6579 committed: false,
6580 failed: None,
6581 duration_ms: 0,
6582 continuation: None,
6583 };
6584 let mut continuation = ContinuationRecord::not_needed();
6585 let mut final_seat = seat.clone();
6586 match out {
6587 AgentOutcome::Ok(o) => {
6588 fix.duration_ms = o.duration_ms;
6589 let parsed = verdict::extract_json::<FixReport>(&o.text);
6590 let incomplete_reason = match &parsed {
6597 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6598 "the reply parsed, but it reported a command whose own CLI never \
6599 confirmed an exit status"
6600 .to_owned(),
6601 ),
6602 Ok(_) => None,
6603 Err(e) => Some(e.to_string()),
6604 };
6605 match incomplete_reason {
6606 None => {
6607 let report = parsed.expect("checked Ok above");
6608 fix.addressed = report.addressed;
6609 fix.rejected = report.rejected;
6610 fix.notes =
6611 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6612 }
6613 Some(reason) => {
6614 let (resumed_seat, resolved, failure, cont) = self
6615 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6616 .await;
6617 fix.duration_ms += cont.cumulative_wait_ms;
6618 continuation = cont;
6619 final_seat = resumed_seat;
6620 match resolved {
6621 Some(report) => {
6622 fix.addressed = report.addressed;
6623 fix.rejected = report.rejected;
6624 fix.notes = blind::sanitize_prose(
6625 &report.notes,
6626 &self.state.config.blind,
6627 );
6628 }
6629 None => fix.failed = failure,
6630 }
6631 }
6632 }
6633 }
6634 AgentOutcome::Dropped(o) => {
6636 fix.duration_ms = o.duration_ms;
6637 let why = o
6638 .dropped
6639 .as_ref()
6640 .map(|d| d.why.as_str())
6641 .unwrap_or("the CLI ended the stream without delivering its answer");
6642 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6643 }
6644 AgentOutcome::Quota(o) => {
6645 self.state.quota.push(QuotaLoss {
6646 seat: final_seat.key.clone(),
6647 node: "fix".to_owned(),
6648 at: Timestamp::now(),
6649 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6650 });
6651 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6652 }
6653 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6654 }
6655 fix.continuation = Some(continuation);
6656 self.state.seats.insert(final_seat.key.clone(), final_seat);
6657 let coordination = self.coordinate(&winner.worktree).await?;
6658 if let Ok(r) = git::rescue_commit(
6659 &winner.worktree,
6660 &format!("magi: review round {round} fixes (uncommitted work)"),
6661 )
6662 .await
6663 {
6664 self.state.note_withheld("fix", &r.withheld);
6665 }
6666 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6667 fix.committed = after != before;
6668 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6676 drop(coordination);
6677 let progressed = diff_after != patch;
6678 let commit_note = if fix.committed {
6679 "committed"
6680 } else {
6681 "NO new commit"
6682 };
6683 let tree_note = if progressed {
6684 "changed vs base"
6685 } else {
6686 "unchanged vs base"
6687 };
6688 self.state.event(
6689 "fix",
6690 match &fix.failed {
6691 Some(reason) => {
6697 format!(
6698 "round {round}: fixer's adoption report was lost ({reason}); \
6699 {commit_note}, tree {tree_note}"
6700 )
6701 }
6702 None => format!(
6703 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6704 {tree_note}{}",
6705 fix.addressed.len(),
6706 fix.rejected.len(),
6707 if continuation.outcome == ContinuationOutcome::Resumed {
6708 format!(
6709 " (adoption report recovered after {} continuation(s))",
6710 continuation.attempts
6711 )
6712 } else {
6713 String::new()
6714 },
6715 ),
6716 },
6717 );
6718 round_record.fix = Some(fix);
6719 round_record.progressed = progressed;
6720 self.state.reviews.push(round_record);
6721 self.state.save()?;
6722
6723 if matches!(
6736 continuation.outcome,
6737 ContinuationOutcome::Exhausted
6738 | ContinuationOutcome::QuotaLost
6739 | ContinuationOutcome::NoSession
6740 ) {
6741 return self
6742 .stop_reviewing(
6743 "the fixer's adoption report never came back, even after resuming its \
6744 own seat; refusing to start another round against the same worktree \
6745 while that is unresolved",
6746 &shell,
6747 &winner.worktree,
6748 )
6749 .await;
6750 }
6751
6752 let streak = self
6753 .state
6754 .reviews
6755 .iter()
6756 .rev()
6757 .take_while(|r| !r.progressed)
6758 .count();
6759 if streak >= STAGNANT_LIMIT {
6760 return self
6761 .stop_reviewing(
6762 &format!(
6763 "the tree has not moved against base for {streak} round(s) in a row"
6764 ),
6765 &shell,
6766 &winner.worktree,
6767 )
6768 .await;
6769 }
6770 }
6771 Ok(())
6772 }
6773
6774 fn record_contested_handoff(&mut self) {
6808 if self.state.contested_handoff.is_some() {
6809 return;
6810 }
6811 let Some(contested) = self
6812 .state
6813 .reviews
6814 .last()
6815 .and_then(ReviewRound::contested_handoff)
6816 else {
6817 return;
6818 };
6819 self.state.event(
6820 "review",
6821 format!(
6822 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6823 wait for the owner's approval",
6824 contested.findings.len(),
6825 contested.rejecters.len()
6826 ),
6827 );
6828 self.state.contested_handoff = Some(contested);
6829 }
6830
6831 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6832 let round_idx = self.state.reviews.len() - 1;
6833 let needs_catchup_run = matches!(
6841 self.state.reviews[round_idx].e2e_status(),
6842 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6843 );
6844 if needs_catchup_run {
6845 let round = self.state.reviews[round_idx].round;
6846 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6847 let commands = self.state.config.verify.e2e.clone();
6848 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6849 let cache_dir = self.state.config.cache_dir();
6850 let context = format!(
6851 "round {round}: verification unresolved, catching up before the final decision"
6852 );
6853 let wait = self.repository_wait();
6854 let (outcomes, verify_retried) = with_cache_lease(
6855 &mut self.state,
6856 cache_dir.as_deref(),
6857 "e2e",
6858 "e2e",
6859 worktree,
6860 &attempted_head,
6861 timeout,
6862 &context,
6863 &wait,
6864 |state, budget| {
6865 let shell = shell.to_vec();
6866 let commands = commands.clone();
6867 let context = context.clone();
6868 async move {
6869 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6870 .await
6871 }
6872 },
6873 )
6874 .await?;
6875 let last = &mut self.state.reviews[round_idx];
6876 last.e2e = outcomes;
6877 last.verify_retried = verify_retried;
6878 last.verified_head = Some(attempted_head);
6885 last.verified_at = Some(Timestamp::now());
6886 if verify_inconclusive(&last.e2e) {
6887 self.state.save()?;
6894 return Ok(());
6895 }
6896 last.e2e_deferred = false;
6897 }
6898 let last = &self.state.reviews[round_idx];
6899 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6900
6901 match last.e2e_status() {
6902 E2eStatus::Failed => {
6903 let red: Vec<String> = last
6904 .e2e
6905 .iter()
6906 .filter(|o| !o.ok())
6907 .map(|o| {
6908 format!(
6909 "`{}` -> {:?}\n{}",
6910 o.command,
6911 o.code,
6912 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6913 )
6914 })
6915 .collect();
6916 self.state
6917 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6918 self.state.status = RunStatus::Blocked;
6919 }
6920 E2eStatus::ResourceBlocked => {
6925 self.state.event(
6926 "review",
6927 format!(
6928 "{why}; e2e could not run (shared build cache unavailable); not \
6929 deciding yet"
6930 ),
6931 );
6932 }
6933 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6934 self.state.event(
6935 "review",
6936 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6937 );
6938 self.record_contested_handoff();
6939 self.state.status = RunStatus::Gating;
6940 }
6941 }
6942 self.state.save()?;
6943 Ok(())
6944 }
6945
6946 async fn gate(&mut self) -> Result<()> {
6949 if self.state.status == RunStatus::Failed
6961 || self
6962 .state
6963 .base_sync
6964 .as_ref()
6965 .is_some_and(|s| s.conflict.is_some())
6966 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6967 != Some(RunStatus::Gating)
6968 {
6969 return Ok(());
6970 }
6971 if self.state.gate_ran {
6972 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6983 self.state.status = RunStatus::Blocked;
6984 self.state.save()?;
6985 }
6986 return Ok(());
6987 }
6988 let Some(winner) = self.state.winner().cloned() else {
6989 return Ok(());
6990 };
6991 self.state.status = RunStatus::Gating;
6992 let mut outcomes = self.run_gate(&winner).await?;
6993 loop {
6994 if verify_inconclusive(&outcomes) {
7005 self.state.save()?;
7006 return Ok(());
7007 }
7008 if outcomes.iter().all(CommandOutcome::ok) {
7009 break;
7010 }
7011 match self.gate_fix_round(&winner, &outcomes).await? {
7012 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7013 GateFix::Stop => break,
7014 GateFix::Defer => {
7015 self.state.save()?;
7016 return Ok(());
7017 }
7018 }
7019 }
7020 let passed = outcomes.iter().all(CommandOutcome::ok);
7021 self.state.gate = outcomes;
7022 self.state.gate_ran = true;
7023 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7024 if !passed {
7025 self.state.status = RunStatus::Blocked;
7026 let spent = self.state.gate_fixes.len();
7027 self.state.event(
7028 "gate",
7029 if spent == 0 {
7030 "gate failed; not merging".to_owned()
7031 } else {
7032 format!("gate failed after {spent} gate-fix round(s); not merging")
7033 },
7034 );
7035 }
7036 self.state.save()?;
7037 Ok(())
7038 }
7039
7040 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7050 let commands = state.config.verify.pre_gate.clone();
7051 if commands.is_empty() {
7052 return;
7053 }
7054 let shell = state.config.shell();
7055 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7056 let (outcomes, _) = run_commands(
7057 state,
7058 "pre_gate",
7059 "pre_gate",
7060 0,
7061 &shell,
7062 &commands,
7063 &winner.worktree,
7064 timeout,
7065 )
7066 .await;
7067 for o in &outcomes {
7068 if !o.ok() {
7069 tracing::warn!(
7070 "pre_gate `{}` failed ({:?}); the gate decides",
7071 o.command,
7072 o.code
7073 );
7074 }
7075 state.event(
7076 "pre_gate",
7077 format!(
7078 "`{}` -> {}",
7079 o.command,
7080 if o.ok() {
7081 "pass".to_owned()
7082 } else {
7083 format!(
7084 "FAIL ({:?})\n{}",
7085 o.code,
7086 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7087 )
7088 }
7089 ),
7090 );
7091 }
7092 state.pre_gate = outcomes;
7093 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7094 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7095 Ok(head) => {
7096 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7097 state.pre_gate_commit = Some(head);
7098 }
7099 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7100 },
7101 Ok(false) => {}
7102 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7103 }
7104 if let Err(e) = state.save() {
7105 tracing::warn!("could not persist the pre_gate record: {e:#}");
7106 }
7107 }
7108
7109 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7112 let wait = self.repository_wait();
7113 run_landing_gate(&mut self.state, winner, &wait).await
7114 }
7115
7116 async fn gate_fix_round(
7129 &mut self,
7130 winner: &Candidate,
7131 outcomes: &[CommandOutcome],
7132 ) -> Result<GateFix> {
7133 let cap = self.state.config.graph.gate_fix_rounds;
7134 let spent = self.state.gate_fixes.len();
7135 if spent >= cap {
7136 if cap > 0 {
7137 self.state.event(
7138 "gate",
7139 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7140 );
7141 }
7142 return Ok(GateFix::Stop);
7143 }
7144 if !gate_fixable(outcomes) {
7145 self.state.event(
7146 "gate",
7147 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7148 command or similar); not spending a fix round on it",
7149 );
7150 return Ok(GateFix::Stop);
7151 }
7152 let min_free = self.state.config.disk.min_free_bytes;
7153 if min_free > 0 {
7154 match crate::disk::free_bytes(&winner.worktree) {
7155 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7156 Ok(free) => {
7157 self.state.event(
7158 "gate",
7159 format!(
7160 "only {free} bytes free ({min_free} required by `[disk] \
7161 min_free_bytes`); not spending a fix round on a failure the disk \
7162 may explain"
7163 ),
7164 );
7165 return Ok(GateFix::Stop);
7166 }
7167 Err(e) => {
7168 self.state.event(
7169 "gate",
7170 format!("free disk space could not be measured ({e:#}); no fix round"),
7171 );
7172 return Ok(GateFix::Stop);
7173 }
7174 }
7175 }
7176
7177 let attempt = spent + 1;
7178 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7179 let base = self.landing_base();
7180 let fix_prompt = prompt::gate_fix_in(
7181 &self.state.instruction,
7182 &failed,
7183 attempt,
7184 cap,
7185 &self.state.config.graph.language,
7186 &self.state.config.graph.github_language,
7187 );
7188 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7189 let sessions = self.state.config.graph.sessions;
7190 let artifacts = self.merge_repair_artifacts();
7191 self.state.event(
7192 "gate",
7193 format!("gate failed; gate-fix round {attempt} of {cap}"),
7194 );
7195 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7196 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7197 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7200 head: before.clone(),
7201 passed: false,
7202 outcomes: Vec::new(),
7203 });
7204 self.state.save()?;
7205 let (_, seat, out) = self
7206 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7207 prompt: fix_prompt.clone(),
7208 spec,
7209 seat,
7210 cwd: winner.worktree.clone(),
7211 timeout,
7212 allow_write: true,
7213 sessions,
7214 artifacts: artifacts.clone(),
7215 stem: format!("gate-fix-{attempt}"),
7216 handover: None,
7217 })
7218 .await;
7219 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7222 return Ok(GateFix::Defer);
7223 }
7224 let mut record = GateFixRecord {
7225 agent: seat.agent.clone(),
7226 failed,
7227 notes: String::new(),
7228 committed: false,
7229 error: None,
7230 };
7231 match out {
7232 AgentOutcome::Ok(o) => {
7233 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7236 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7237 }
7238 }
7239 AgentOutcome::Dropped(_) => {
7240 record.error = Some("the CLI dropped the stream".to_owned());
7241 }
7242 AgentOutcome::Quota(o) => {
7243 self.state.quota.push(QuotaLoss {
7244 seat: seat.key.clone(),
7245 node: "gate-fix".to_owned(),
7246 at: Timestamp::now(),
7247 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7248 });
7249 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7250 }
7251 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7252 }
7253 self.state.seats.insert(seat.key.clone(), seat);
7254 let coordination = self.coordinate(&winner.worktree).await?;
7255 if let Ok(r) = git::rescue_commit(
7256 &winner.worktree,
7257 &format!("magi: gate fix {attempt} (uncommitted work)"),
7258 )
7259 .await
7260 {
7261 self.state.note_withheld("gate-fix", &r.withheld);
7262 }
7263 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7264 record.committed = after != before;
7265 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7266 let note = record.error.clone();
7267 self.state.gate_fixes.push(record);
7268 self.state.save()?;
7269 if !changed {
7270 self.state.event(
7271 "gate",
7272 match note {
7273 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7274 None => format!("gate-fix round {attempt}: the tree did not change"),
7275 },
7276 );
7277 return Ok(GateFix::Stop);
7278 }
7279 self.state.event(
7280 "gate",
7281 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7282 );
7283
7284 drop(coordination);
7285 let wait = self.repository_wait();
7286 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7287 }
7288
7289 async fn guarded_pr_message(
7296 &mut self,
7297 winner: &Candidate,
7298 facts: Option<&BranchFacts>,
7299 posting: bool,
7300 ) -> Result<Option<PrMessage>> {
7301 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7302 let head = match final_head.as_deref() {
7303 Some(h) => crate::pr_outcome::FinalHead::At(h),
7304 None => crate::pr_outcome::FinalHead::Unknown,
7305 };
7306 let summarized = if posting {
7308 Some(self.summarize_change(winner, final_head.as_deref()).await)
7309 } else {
7310 None
7311 };
7312 if self.state.resource_block.is_some() {
7317 self.state.event(
7318 "pr-summary",
7319 "a shared resource was busy; nothing was published, resume to summarize again",
7320 );
7321 return Ok(None);
7322 }
7323 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7324 anyhow::bail!(
7325 "the final commit moved during public description generation; retry before publishing"
7326 );
7327 }
7328 let note = match (&summarized, &self.state.change_summary) {
7329 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7330 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7331 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7332 (None, _) => ChangeNote::NotAsked,
7333 };
7334 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7335 if !posting {
7336 let identity = crate::scrub::Identity::current();
7337 return Ok(Some(PrMessage {
7338 title: crate::scrub::scrub(&pr.title, &identity),
7339 body: crate::scrub::scrub(&pr.body, &identity),
7340 }));
7341 }
7342 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7345 if let Some(known) = self
7346 .state
7347 .github_text
7348 .clone()
7349 .filter(|g| g.fingerprint == raw_fp)
7350 {
7351 return self.settle_withheld(winner, &pr, known).await;
7352 }
7353 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7354 let mut violations =
7355 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7356 let identity = crate::scrub::Identity::current();
7357 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7358 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7359 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7360 {
7361 violations.push(crate::github_text::Violation::SensitiveData);
7362 }
7363 let has_language = violations
7367 .iter()
7368 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7369 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7370 self.state.event(
7371 "github-text",
7372 if has_language {
7373 format!("description rejected: {violations:?}; requesting one rewrite")
7374 } else {
7375 format!("description rejected: {violations:?}; asking the owner")
7376 },
7377 );
7378 let mut rewritten = false;
7379 let mut rejected_rewrite = None;
7380 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7381 {
7382 let output = text;
7383 if let Some(title) = summary_title(&output) {
7384 let raw_body = summary_without_title(&output);
7387 let clean_title = crate::scrub::scrub(&title, &identity);
7388 let mut body = crate::scrub::scrub(&raw_body, &identity);
7389 let redacted = clean_title != title || body != raw_body;
7390 let title = clean_title;
7391 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7392 if !complete {
7393 self.state.event(
7394 "github-text",
7395 "description rewrite omitted required findings",
7396 );
7397 }
7398 if !body.trim().is_empty() {
7399 body.push_str("\n\n");
7400 body.push_str(&public_pr_evidence(&self.state, head));
7401 }
7402 let marker = format!("magi:run/{}", self.state.id);
7403 if !body.trim().is_empty() && !body.contains(&marker) {
7404 body.push_str(&format!("\n\n{marker}"));
7405 }
7406 let rewrite = PrMessage {
7407 title: title.clone(),
7408 body: body.clone(),
7409 };
7410 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7411 let mut failed = crate::github_text::check_for(
7412 &self.state.config.graph,
7413 &title,
7414 &body,
7415 redecision,
7416 );
7417 if (!crate::github_text::shareable(&title)
7418 || !crate::github_text::shareable(&body))
7419 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7420 {
7421 failed.push(crate::github_text::Violation::SensitiveData);
7422 }
7423 if complete
7424 && !title.trim().is_empty()
7425 && !body.trim().is_empty()
7426 && !failed.is_empty()
7427 {
7428 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7429 title: title.clone(),
7430 body: body.clone(),
7431 failed: failed
7432 .iter()
7433 .map(|v| crate::github_text::category(*v).to_owned())
7434 .collect(),
7435 });
7436 }
7437 if complete
7438 && !title.trim().is_empty()
7439 && !raw_body.trim().is_empty()
7440 && failed.is_empty()
7441 {
7442 decision = redecision;
7443 pr = PrMessage { title, body };
7444 rewritten = true;
7445 self.state
7446 .event("github-text", "description rewrite passed");
7447 if redacted {
7448 self.state.event(
7449 "github-text",
7450 "sensitive data removed from description rewrite",
7451 );
7452 }
7453 }
7454 }
7455 }
7456 if !rewritten {
7457 self.state.event(
7458 "github-text",
7459 "description rewrite unavailable or rejected; asking the owner",
7460 );
7461 if let Some(w) =
7462 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7463 {
7464 self.retire_github_text_question("the withheld text changed");
7467 let record = crate::run::GithubTextAsk {
7468 rewrite: rejected_rewrite,
7469 fingerprint: raw_fp,
7470 title: w.title,
7471 body: w.body,
7472 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7473 question: None,
7474 asks: 0,
7475 resolved: false,
7476 chosen_title: None,
7477 choice: None,
7478 };
7479 return self.settle_withheld(winner, &pr, record).await;
7480 }
7481 }
7482 }
7483 let (title, body) =
7484 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7485 Ok(Some(PrMessage { title, body }))
7486 }
7487
7488 async fn summarize_change(
7499 &mut self,
7500 winner: &Candidate,
7501 head: Option<&str>,
7502 ) -> std::result::Result<(), String> {
7503 let Some(head) = head else {
7504 return Err("the final commit could not be read".to_owned());
7505 };
7506 let remote = self.state.config.merge.remote.clone();
7507 let tracking = format!("{remote}/{}", self.state.base_branch);
7508 if !git::rev_exists(&winner.worktree, &tracking).await {
7509 return Err(format!("the remote base `{tracking}` could not be read"));
7510 }
7511 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7512 Ok(b) if !b.is_empty() => b,
7513 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7514 };
7515 let evidence = crate::pr_outcome::summary_evidence(
7516 &self.state,
7517 crate::pr_outcome::FinalHead::At(head),
7518 );
7519 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7520 if self
7521 .state
7522 .change_summary
7523 .as_ref()
7524 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7525 {
7526 return Ok(());
7527 }
7528 if self.state.change_summary.take().is_some() {
7529 self.state.event(
7530 "pr-summary",
7531 "the final diff or evidence changed; the earlier change summary was discarded",
7532 );
7533 }
7534 if let Err(e) = write_artifact(
7536 &self.state,
7537 &format!("public-description-{fingerprint}.md"),
7538 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7539 ) {
7540 tracing::warn!("could not save public description evidence: {e:#}");
7541 }
7542 let chain = match agent::pick_chain(
7543 &self.state.config.agents,
7544 self.state.config.roles.synthesizer.as_ref(),
7545 &agent::installed,
7546 "synthesizer",
7547 ) {
7548 Ok(chain) => chain,
7549 Err(e) => {
7550 tracing::warn!("no agent for the change summary: {e:#}");
7551 self.state.event("pr-summary", "no summary agent available");
7552 return Err("no agent was available to summarize it".to_owned());
7553 }
7554 };
7555 let subjects = git::subjects(&winner.worktree, &base, head)
7556 .await
7557 .unwrap_or_default();
7558 let stat = git::diff_stat(&winner.worktree, &base, head)
7559 .await
7560 .unwrap_or_default();
7561 let stat: String = stat.chars().take(6_000).collect();
7562 let mut prompt = crate::prompt::change_summary_in(
7563 &base,
7564 head,
7565 &subjects,
7566 stat.trim(),
7567 &self.state.config.graph.github_language,
7568 );
7569 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7570 let sessions = self.state.config.graph.sessions;
7571 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7572 let prompts = self.state.config.prompts.clone();
7573 let cache = self.state.config.cache_dir();
7574 let run = self.state.id.clone();
7575 for (i, spec) in chain.into_iter().enumerate() {
7576 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7577 let github_spec = match agent::for_github_text(
7578 &spec,
7579 &self.state.config.graph.github_language,
7580 &winner.worktree,
7581 ) {
7582 Ok(spec) => spec,
7583 Err(e) => {
7584 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7585 continue;
7586 }
7587 };
7588 let job = SeatJob {
7589 spec: github_spec,
7590 seat,
7591 cwd: winner.worktree.clone(),
7592 prompt: prompt.clone(),
7593 timeout,
7594 allow_write: false,
7595 sessions,
7596 artifacts: agent::artifacts_dir(&self.state.dir()),
7597 stem: if i == 0 {
7598 "change-summary".to_owned()
7599 } else {
7600 format!("change-summary-{}", spec.id)
7601 },
7602 handover: None,
7603 };
7604 let ctx = WaveCtx {
7605 run: &run,
7606 node: "pr-summary",
7607 prompts: &prompts,
7608 cache: cache.as_deref(),
7609 round: None,
7610 carry_seats: false,
7611 pauses: Some((&self.pause, &self.interrupt)),
7612 };
7613 let (seat, outcome) =
7614 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7615 self.state.seats.insert(seat.key.clone(), seat);
7616 let text = match outcome {
7617 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7618 AgentOutcome::ResourceBlocked(why) => {
7619 if self.state.resource_block.is_none() {
7622 self.state.resource_block = Some(crate::run::ResourceBlock {
7623 node: "pr-summary".to_owned(),
7624 kind: "shared resource".to_owned(),
7625 detail: why,
7626 at: Timestamp::now(),
7627 });
7628 }
7629 return Err("a shared resource was busy".to_owned());
7630 }
7631 _ => continue,
7632 };
7633 let blind = &self.state.config.blind;
7634 let Some(title) = summary_title(&text) else {
7635 continue;
7636 };
7637 let body: String = summary_without_title(&text)
7638 .lines()
7639 .filter(|l| !l.contains("magi:run/"))
7640 .collect::<Vec<_>>()
7641 .join("\n");
7642 let body = blind::sanitize_prose(body.trim(), blind);
7643 let title = blind::sanitize_prose(&title, blind);
7644 if body.trim().is_empty() || title.trim().is_empty() {
7645 continue;
7646 }
7647 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7648 continue;
7649 }
7650 if git::rev_parse(&winner.worktree, "HEAD")
7652 .await
7653 .ok()
7654 .as_deref()
7655 != Some(head)
7656 {
7657 return Err("the branch moved while the summary was written".to_owned());
7658 }
7659 if git::merge_base(&winner.worktree, &tracking, head)
7660 .await
7661 .ok()
7662 .as_deref()
7663 != Some(base.as_str())
7664 || crate::pr_outcome::summary_evidence(
7665 &self.state,
7666 crate::pr_outcome::FinalHead::At(head),
7667 ) != evidence
7668 {
7669 return Err("the summary evidence changed while it was written".to_owned());
7670 }
7671 self.state.change_summary = Some(crate::run::ChangeSummary {
7672 head: head.to_owned(),
7673 base,
7674 evidence: fingerprint,
7675 title,
7676 body,
7677 });
7678 self.state.event(
7679 "pr-summary",
7680 format!("change summary written by {} for the final diff", spec.id),
7681 );
7682 if let Err(e) = self.state.save() {
7683 tracing::warn!("could not save the change summary: {e:#}");
7684 }
7685 return Ok(());
7686 }
7687 self.state
7688 .event("pr-summary", "no agent produced a change summary");
7689 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7690 }
7691
7692 async fn rewrite_pr_text(
7702 &mut self,
7703 winner: &Candidate,
7704 pr: &PrMessage,
7705 violations: &[crate::github_text::Violation],
7706 ) -> Option<String> {
7707 let prompt = format!(
7708 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7709 pr.title, pr.body
7710 );
7711 let sessions = self.state.config.graph.sessions;
7712 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7713 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7716 let key = format!("impl-{}", winner.label);
7717 if let Some(spec) = self
7718 .state
7719 .config
7720 .agents
7721 .iter()
7722 .find(|a| a.id == winner.agent)
7723 .cloned()
7724 && let Some(existing) = self.state.seats.get(&key)
7725 && existing.agent == spec.id
7726 && agent::has_session(spec.kind, existing, sessions)
7727 {
7728 attempts.push((spec, Some(existing.clone())));
7729 }
7730 match agent::pick_chain(
7731 &self.state.config.agents,
7732 self.state.config.roles.language_judge.as_ref(),
7733 &agent::installed,
7734 "language_judge",
7735 ) {
7736 Ok(chain) => {
7737 for spec in chain {
7738 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7739 continue;
7740 }
7741 attempts.push((spec, None));
7742 }
7743 }
7744 Err(e) if attempts.is_empty() => {
7745 tracing::warn!("no agent for the description rewrite: {e:#}");
7746 self.state
7747 .event("github-text", "no rewrite agent available");
7748 return None;
7749 }
7750 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7751 }
7752 let prompts = self.state.config.prompts.clone();
7753 let cache = self.state.config.cache_dir();
7754 let run = self.state.id.clone();
7755 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7756 let resumed = resumed_seat.is_some();
7757 let seat = resumed_seat.unwrap_or_else(|| {
7758 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7759 });
7760 let github_spec = match agent::for_github_text(
7761 &spec,
7762 &self.state.config.graph.github_language,
7763 &winner.worktree,
7764 ) {
7765 Ok(spec) => spec,
7766 Err(e) => {
7767 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7768 continue;
7769 }
7770 };
7771 let job = SeatJob {
7772 spec: github_spec,
7773 seat,
7774 cwd: winner.worktree.clone(),
7775 prompt: prompt.clone(),
7776 timeout: retry_budget(timeout_full, resumed),
7777 allow_write: false,
7778 sessions,
7779 artifacts: agent::artifacts_dir(&self.state.dir()),
7780 stem: if i == 0 {
7781 "github-text-rewrite".to_owned()
7782 } else {
7783 format!("github-text-rewrite-{}", spec.id)
7784 },
7785 handover: None,
7786 };
7787 let ctx = WaveCtx {
7788 run: &run,
7789 node: "github-text",
7790 prompts: &prompts,
7791 cache: cache.as_deref(),
7792 round: None,
7793 carry_seats: false,
7794 pauses: Some((&self.pause, &self.interrupt)),
7795 };
7796 let (seat, outcome) =
7797 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7798 self.state.seats.insert(seat.key.clone(), seat);
7799 match outcome {
7800 AgentOutcome::Ok(o)
7801 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7802 {
7803 return Some(o.text);
7804 }
7805 AgentOutcome::ResourceBlocked(_) => return None,
7806 _ => {}
7807 }
7808 }
7809 None
7810 }
7811
7812 fn retire_github_text_question(&mut self, why: &str) {
7814 let Some(id) = self
7815 .state
7816 .github_text
7817 .as_ref()
7818 .and_then(|g| g.question.clone())
7819 else {
7820 return;
7821 };
7822 let _ = ask::Questions::open().update(&id, |q| {
7823 q.abandon(why);
7824 Ok(())
7825 });
7826 }
7827
7828 async fn settle_withheld(
7832 &mut self,
7833 winner: &Candidate,
7834 pr: &PrMessage,
7835 mut g: crate::run::GithubTextAsk,
7836 ) -> Result<Option<PrMessage>> {
7837 use crate::github_text as gt;
7838 let store = ask::Questions::open();
7839 let timeout = self.state.config.graph.answer_timeout;
7840 if !g.resolved {
7841 let mut standing = g
7842 .question
7843 .as_ref()
7844 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7845 if let Some(q) = standing.as_ref()
7846 && q.status.open()
7847 && gt::expired(q, timeout)
7848 {
7849 let why = format!("no answer within {}s of asking", timeout.max(1));
7850 standing = store
7851 .update(&q.id, |q| {
7852 q.abandon(&why);
7853 Ok(())
7854 })
7855 .ok()
7856 .map(|(q, ())| q);
7857 }
7858 let mut ask_again: Option<Option<Vec<String>>> = None;
7859 match standing {
7860 None if g.asks <= 1 => {
7863 g.asks = 0;
7864 ask_again = Some(None);
7865 }
7866 None => {
7867 g.resolved = true;
7868 g.chosen_title = None;
7869 g.choice = Some(GithubTextChoice::Fallback);
7870 }
7871 Some(q) if q.status.open() => {
7872 self.park_on_github_text(&g)?;
7873 return Ok(None);
7874 }
7875 Some(q) => match gt::read_reply(&q) {
7876 gt::Reply::Fallback => {
7877 g.resolved = true;
7878 g.chosen_title = None;
7879 g.choice = Some(GithubTextChoice::Fallback);
7880 }
7881 gt::Reply::Original => {
7882 g.resolved = true;
7883 g.chosen_title = None;
7884 g.choice = Some(GithubTextChoice::Original);
7885 }
7886 gt::Reply::Rewrite => {
7887 g.resolved = true;
7888 g.chosen_title = None;
7889 g.choice = Some(if g.rewrite.is_some() {
7890 GithubTextChoice::Rewrite
7891 } else {
7892 GithubTextChoice::Fallback
7893 });
7894 }
7895 gt::Reply::Title(text) => {
7896 let first = text
7897 .lines()
7898 .map(str::trim)
7899 .find(|l| !l.is_empty())
7900 .unwrap_or("");
7901 let decision = if first.is_empty() {
7902 None
7903 } else {
7904 let candidate = PrMessage {
7905 title: first.to_owned(),
7906 body: String::new(),
7907 };
7908 self.judge_pr_language(&winner.worktree, &candidate).await
7909 };
7910 match gt::vet_title(
7911 &text,
7912 decision,
7913 self.state.config.graph.github_english(),
7914 ) {
7915 Ok(title) => {
7916 g.resolved = true;
7917 g.chosen_title = Some(title);
7918 g.choice = Some(GithubTextChoice::MyTitle);
7919 }
7920 Err(cats) if g.asks < 2 => {
7921 ask_again = Some(Some(
7922 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7923 ))
7924 }
7925 Err(_) => {
7926 g.resolved = true;
7927 g.chosen_title = None;
7928 g.choice = Some(GithubTextChoice::Fallback);
7929 }
7930 }
7931 }
7932 },
7933 }
7934 if let Some(rejected) = ask_again {
7935 let w = gt::Withheld {
7938 title: g.title,
7939 body: g.body,
7940 categories: g.categories.clone(),
7941 };
7942 let artifact = match crate::run::write_artifact(
7945 &self.state,
7946 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7947 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7948 ) {
7949 Ok(p) => Some(p),
7950 Err(e) => {
7951 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7952 None
7953 }
7954 };
7955 let (detail, comparison) = gt::question_content(
7956 &w,
7957 &pr.title,
7958 &pr.body,
7959 &self.state.id,
7960 rejected.as_deref(),
7961 artifact.as_deref(),
7962 g.rewrite.as_ref(),
7963 );
7964 let mut q = ask::Question::new(
7965 self.state.id.clone(),
7966 gt::ASK_NODE.to_owned(),
7967 gt::ASK_SEAT.to_owned(),
7968 gt::question_summary(&self.state.config.graph.language, &w),
7969 detail,
7970 gt::question_choices(&w, g.rewrite.as_ref()),
7971 );
7972 q.posting_comparison = Some(comparison);
7973 q.answer_timeout = timeout;
7974 g.question = Some(q.id.clone());
7977 g.asks += 1;
7978 self.state.github_text = Some(g.clone());
7979 self.state.event(
7980 "github-text",
7981 format!("asking the owner about the withheld text ({})", q.short()),
7982 );
7983 self.state.save()?;
7984 store
7985 .put_executable(&mut q)
7986 .context("file the posting-gate question")?;
7987 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7988 tracing::warn!(
7989 "could not notify about posting-gate question {}: {e:#}",
7990 q.short()
7991 );
7992 }
7993 self.park_on_github_text(&g)?;
7994 return Ok(None);
7995 }
7996 }
7997 let resolved = g.chosen_title.clone();
7998 let original = g.choice == Some(GithubTextChoice::Original);
7999 self.state.event(
8001 "github-text",
8002 if original {
8003 format!(
8004 "posting the original text for: {}",
8005 match (g.title, g.body) {
8006 (true, true) => "title, description",
8007 (true, false) => "title",
8008 _ => "description",
8009 }
8010 )
8011 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8012 "posting the saved rewrite title and description".to_owned()
8013 } else if resolved.is_some() {
8014 "posting the owner's replacement title".to_owned()
8015 } else {
8016 "posting the neutral text for the withheld fields".to_owned()
8017 },
8018 );
8019 self.state.github_text = Some(g.clone());
8020 self.state.save()?;
8021 if g.choice == Some(GithubTextChoice::Rewrite)
8022 && let Some(r) = g.rewrite.as_ref()
8023 {
8024 return Ok(Some(PrMessage {
8025 title: r.title.clone(),
8026 body: r.body.clone(),
8027 }));
8028 }
8029 let w = gt::Withheld {
8030 title: g.title,
8031 body: g.body,
8032 categories: g.categories.clone(),
8033 };
8034 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8035 if let Some(t) = resolved {
8036 title = t;
8037 }
8038 let english = gt::LanguageDecision {
8040 title_english: true,
8041 body_english: true,
8042 body_complete: true,
8043 };
8044 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8045 if original {
8046 if g.title {
8049 title = pr.title.clone();
8050 }
8051 if g.body {
8052 body = pr.body.clone();
8053 }
8054 }
8055 Ok(Some(PrMessage { title, body }))
8056 }
8057
8058 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8059 self.state.github_text = Some(g.clone());
8060 self.state.end_pass_parked(ParkCause::Question);
8061 self.state.parked = true;
8062 self.state.event(
8063 "github-text",
8064 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8065 );
8066 self.state.save()
8067 }
8068
8069 async fn judge_pr_language(
8072 &mut self,
8073 cwd: &Path,
8074 pr: &PrMessage,
8075 ) -> Option<crate::github_text::LanguageDecision> {
8076 self.state.config.roles.language_judge.as_ref()?;
8077 if !self.state.config.graph.language_gate() {
8078 return None;
8079 }
8080 let decision =
8081 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8082 self.state.event(
8083 "github-text",
8084 if decision.is_some() {
8085 "language decided by the language judge"
8086 } else {
8087 "language judge unavailable; using built-in heuristics"
8088 },
8089 );
8090 decision
8091 }
8092
8093 async fn merge(&mut self) -> Result<()> {
8094 if self
8109 .state
8110 .base_sync
8111 .as_ref()
8112 .is_some_and(|s| s.conflict.is_some())
8113 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8114 != Some(RunStatus::Gating)
8115 || !self.state.gate_allows_progress()
8123 {
8124 return Ok(());
8125 }
8126 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8135 return Ok(());
8136 }
8137 let Some(winner) = self.state.winner().cloned() else {
8138 return Ok(());
8139 };
8140 let repo = self.state.repo.clone();
8141 let base = self.state.base_branch.clone();
8142 let mode = self.state.config.merge.mode;
8143 let style = self.state.config.merge.style;
8144 let facts = if is_review_run(&self.state) {
8145 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8146 let start = review_base(
8147 &repo,
8148 &self.state.config.merge.remote,
8149 &base,
8150 &self.state.base_commit,
8151 &winner.branch,
8152 )
8153 .await;
8154 branch_facts(&repo, &start, &winner.branch).await
8155 } else {
8156 None
8157 };
8158 let leaked = if is_review_run(&self.state) {
8161 leaked_subjects(&self.state, &winner.branch).await
8162 } else {
8163 Some(Vec::new())
8164 };
8165 let Some(pr) = self
8169 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8170 .await?
8171 else {
8172 return Ok(());
8173 };
8174 let message = pr.commit_message();
8175
8176 let outcome = match mode {
8177 MergeMode::None => MergeOutcome {
8178 mode,
8179 ok: true,
8180 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8181 empty: false,
8182 },
8183 MergeMode::Pr | MergeMode::Local
8184 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8185 {
8186 MergeOutcome {
8187 mode,
8188 ok: false,
8189 detail: empty_candidate_detail(&self.state, &base),
8190 empty: true,
8191 }
8192 }
8193 MergeMode::Local => {
8194 let on = git::current_branch(&repo).await?;
8195 if on.as_deref() != Some(base.as_str()) {
8196 MergeOutcome {
8197 mode,
8198 ok: false,
8199 detail: format!(
8200 "{} has {} checked out, not the base branch {base}",
8201 repo.display(),
8202 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8203 ),
8204 empty: false,
8205 }
8206 } else if !git::is_clean(&repo).await? {
8207 MergeOutcome {
8208 mode,
8209 ok: false,
8210 detail: format!("{} is dirty; refusing to merge", repo.display()),
8211 empty: false,
8212 }
8213 } else {
8214 let out = match style {
8215 MergeStyle::Merge => {
8216 git::merge_no_ff(&repo, &winner.branch, &message).await?
8217 }
8218 MergeStyle::Squash => {
8219 git::merge_squash(&repo, &winner.branch, &message).await?
8220 }
8221 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8222 };
8223 MergeOutcome {
8224 mode,
8225 ok: out.ok(),
8226 detail: if out.ok() { out.stdout } else { out.stderr },
8227 empty: false,
8228 }
8229 }
8230 }
8231 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8232 self.state.pr_publication_pending = true;
8233 self.state.save()?;
8234 let out =
8235 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8236 self.state.pr_publication_pending = out.is_err();
8237 MergeOutcome {
8238 mode,
8239 ok: out.is_ok(),
8240 detail: match out {
8241 Ok(url) => url,
8242 Err(e) => format!("{e:#}"),
8243 },
8244 empty: false,
8245 }
8246 }
8247 MergeMode::Pr => {
8248 let remote = self.state.config.merge.remote.clone();
8249 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8250 if !pushed.ok() {
8251 MergeOutcome {
8252 mode,
8253 ok: false,
8254 detail: pushed.stderr,
8255 empty: false,
8256 }
8257 } else {
8258 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8264 let out = match pr_merge_plan(found) {
8265 PrPlan::Create => {
8266 gh_pr_create(
8267 &winner.worktree,
8268 &base,
8269 &winner.branch,
8270 &pr.title,
8271 &pr.body,
8272 )
8273 .await
8274 }
8275 PrPlan::Adopt { url, title } => {
8276 self.state
8277 .event("merge", format!("Pr: adopted open pull request {url}"));
8278 if self.state.github_text.as_ref().is_some_and(|g| {
8279 g.resolved
8280 && g.choice == Some(GithubTextChoice::Rewrite)
8281 && g.rewrite.is_some()
8282 }) {
8283 land::set_pr_text_verbatim(
8286 &winner.worktree,
8287 &url,
8288 &pr.title,
8289 &pr.body,
8290 )
8291 .await
8292 .map(|()| url)
8293 } else {
8294 if title != pr.title
8295 && (!is_review_run(&self.state)
8296 || leaked
8297 .as_deref()
8298 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8299 && let Err(e) =
8300 if self.state.github_text.as_ref().is_some_and(|g| {
8301 g.resolved
8302 && g.title
8303 && g.choice == Some(GithubTextChoice::Original)
8304 }) {
8305 land::set_pr_title_verbatim(
8307 &winner.worktree,
8308 &url,
8309 &pr.title,
8310 )
8311 .await
8312 } else {
8313 land::set_pr_title(
8314 &mut self.state,
8315 &winner.worktree,
8316 &url,
8317 &pr.title,
8318 )
8319 .await
8320 }
8321 {
8322 tracing::warn!("could not refresh title of {url}: {e:#}");
8323 self.state
8324 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8325 }
8326 Ok(url)
8327 }
8328 }
8329 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8330 };
8331 match out {
8332 Ok(url) => MergeOutcome {
8333 mode,
8334 ok: true,
8335 detail: url,
8336 empty: false,
8337 },
8338 Err(e) => MergeOutcome {
8339 mode,
8340 ok: false,
8341 detail: e.to_string(),
8342 empty: false,
8343 },
8344 }
8345 }
8346 }
8347 };
8348
8349 self.state.status = match (mode, outcome.ok) {
8350 (MergeMode::None, _) => RunStatus::Ready,
8351 (_, true) if outcome.empty => RunStatus::Ready,
8352 (MergeMode::Pr, true) => RunStatus::Ready,
8353 (_, true) => RunStatus::Merged,
8354 (_, false) => RunStatus::Blocked,
8355 };
8356 self.state.event(
8357 "merge",
8358 format!(
8359 "{:?}: {}",
8360 mode,
8361 outcome.detail.lines().next().unwrap_or("")
8362 ),
8363 );
8364 self.state.merge = Some(outcome);
8365 self.state.save()?;
8366
8367 if self.state.config.graph.land
8373 && mode == MergeMode::Pr
8374 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8375 {
8376 self.run_land().await?;
8377 }
8378 self.settle_questions();
8383 Ok(())
8384 }
8385
8386 async fn run_land(&mut self) -> Result<()> {
8397 crate::pr_head::ensure_owner(&self.state)?;
8398 let url = self
8399 .state
8400 .pr
8401 .as_ref()
8402 .map(|pr| pr.url.clone())
8403 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8404 .unwrap_or_default();
8405 let url = url.lines().next().unwrap_or("").trim().to_owned();
8406 if !url.starts_with("http") {
8407 return Ok(());
8408 }
8409 loop {
8412 match land::land(&mut self.state, &url).await {
8413 Ok(_)
8414 if self
8415 .state
8416 .merge_repairs
8417 .last()
8418 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8419 {
8420 if self.repair_merge_review().await? {
8421 continue;
8422 }
8423 }
8424 Ok(pr) if self.state.parked => {
8425 let _ = pr;
8429 }
8430 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8431 self.state.status = RunStatus::Superseded;
8432 self.state.save()?;
8433 }
8434 Ok(pr) => {
8435 self.state.status = match pr.state {
8436 land::PrLifecycle::Merged => RunStatus::Merged,
8437 _ => RunStatus::Blocked,
8438 };
8439 if bump::should_release_bump(self.state.status)
8446 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8447 {
8448 self.state
8454 .event("bump", format!("release bump skipped: {e:#}"));
8455 }
8456 if self.state.status == RunStatus::Merged {
8459 crate::followup::after_merge(&mut self.state, &pr.url).await;
8460 }
8461 self.state.save()?;
8462 }
8463 Err(e) => {
8464 if e.is::<crate::repo_lock::ResourceBlocked>() {
8465 return Err(e);
8466 }
8467 self.state.status = RunStatus::Blocked;
8468 self.state.event("land", format!("gave up: {e}"));
8469 self.state.save()?;
8470 }
8471 }
8472 break;
8473 }
8474 Ok(())
8475 }
8476
8477 async fn repair_merge_review(&mut self) -> Result<bool> {
8480 use crate::run::MergeRepairPhase as Phase;
8481 crate::pr_head::ensure_owner(&self.state)?;
8482 let i = self.state.merge_repairs.len() - 1;
8483 let request = self.state.merge_repairs[i].clone();
8484 if request.phase == Phase::Fixing {
8485 let restored = match self.state.winner().cloned() {
8486 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8487 None => Err(anyhow::anyhow!("repair has no winning branch")),
8488 };
8489 let why = match restored {
8490 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8491 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8492 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8493 };
8494 self.hold_merge_repair(i, why)?;
8495 return Ok(false);
8496 }
8497 let failure = if i >= self.state.config.graph.land_rounds
8498 || self.state.config.graph.review_rounds == 0
8499 {
8500 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8501 } else {
8502 None
8503 };
8504 if let Some(why) = failure {
8505 self.hold_merge_repair(i, why)?;
8506 return Ok(false);
8507 }
8508 if request.phase == Phase::Requested {
8509 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8510 Ok(pr) => pr,
8511 Err(e) => {
8512 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8513 return Ok(false);
8514 }
8515 };
8516 self.state.merge_repairs[i].phase = Phase::Fixing;
8517 self.state.land_approval = None;
8518 self.state.land_armed_head = None;
8519 self.state.save()?;
8520 let evidence = serde_json::to_string_pretty(&request)?;
8523 let reason = format!(
8524 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8525 );
8526 let budget = self.state.config.graph.land_rounds;
8527 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8528 match result {
8529 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8530 Ok(land::Fixed::Failed(why)) => {
8531 self.hold_merge_repair(
8532 i,
8533 format!(
8534 "repair failed: {why}; inspect the branch and run an explicit review"
8535 ),
8536 )?;
8537 return Ok(false);
8538 }
8539 Err(e) => {
8540 self.hold_merge_repair(
8541 i,
8542 format!(
8543 "repair failed: {e:#}; inspect the branch and run an explicit review"
8544 ),
8545 )?;
8546 return Ok(false);
8547 }
8548 }
8549 self.begin_merge_repair_review(i)?;
8553 }
8554 self.review_loop().await?;
8555 self.gate().await?;
8556 if self.resource_halt()? || self.state.parked {
8557 return Ok(false);
8558 }
8559 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8560 != Some(RunStatus::Gating)
8561 || !self.state.gate_allows_progress()
8562 {
8563 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8564 return Ok(false);
8565 }
8566 let winner = self
8569 .state
8570 .winner()
8571 .cloned()
8572 .context("repair has no winning branch")?;
8573 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8574 if self.state.gate_head.as_deref() != Some(after.as_str())
8575 || !git::is_clean(&winner.worktree).await?
8576 {
8577 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8578 return Ok(false);
8579 }
8580 let expected = self
8581 .state
8582 .pr_pushed_head
8583 .as_deref()
8584 .unwrap_or(&request.head)
8585 .to_owned();
8586 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8587 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8590 let refspec = format!("{after}:refs/heads/{public}");
8591 let push = git::git_raw(
8592 &winner.worktree,
8593 &["push", &lease, &self.state.config.merge.remote, &refspec],
8594 )
8595 .await;
8596 match push {
8597 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8598 Ok(out) => {
8599 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8600 return Ok(false);
8601 }
8602 Err(e) => {
8603 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8604 return Ok(false);
8605 }
8606 }
8607 self.state.merge_repairs[i].phase = Phase::Complete;
8608 self.state.status = RunStatus::Landing;
8609 self.state.event(
8610 "repair",
8611 "new review and gate completed; fresh owner merge decision required",
8612 );
8613 self.state.save()?;
8614 Ok(true)
8615 }
8616
8617 fn merge_repair_artifacts(&self) -> PathBuf {
8618 let artifacts = agent::artifacts_dir(&self.state.dir());
8619 match self
8620 .state
8621 .merge_repairs
8622 .last()
8623 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8624 {
8625 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8626 None => artifacts,
8627 }
8628 }
8629
8630 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8631 self.state.reviews.clear();
8632 self.state.gate.clear();
8633 self.state.gate_ran = false;
8634 self.state.gate_head = None;
8635 self.state.gate_fixes.clear();
8636 self.state.gate_repair_e2e = None;
8637 self.state.pre_gate.clear();
8638 self.state.pre_gate_commit = None;
8639 self.state.contested_handoff = None;
8640 self.state.land_reruns = None;
8641 self.state.land_approval = None;
8642 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8643 self.state.status = RunStatus::Reviewing;
8644 self.state.save()
8645 }
8646
8647 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8648 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8649 self.state.merge_repairs[index].failure = Some(why.clone());
8650 self.state.status = RunStatus::Blocked;
8651 self.state.event("repair", why);
8652 self.state.save()
8653 }
8654
8655 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8659 if let Some(existing) = self.state.seats.get(key)
8660 && existing.agent == agent
8661 {
8662 return existing.clone();
8663 }
8664 let fresh = if self.state.seats.contains_key(key) {
8667 handover_seat(key, agent, self.state.next_seat_seed())
8668 } else {
8669 SeatState::new(key, agent, self.state.seed)
8670 };
8671 self.state.seats.insert(key.to_owned(), fresh.clone());
8672 fresh
8673 }
8674
8675 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8681 match self.state.seats.get(key) {
8682 Some(s) if s.agent != spec.id => {
8683 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8684 }
8685 _ => spec,
8686 }
8687 }
8688
8689 fn view(&self, c: &Candidate) -> CandidateView {
8691 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8692 .unwrap_or_default();
8693 let (patch, _) = blind::sanitize_patch(
8694 &format!("candidate {} patch", c.label),
8695 &raw,
8696 &self.state.config.blind,
8697 );
8698 CandidateView {
8699 label: c.label,
8700 branch: c.branch.clone(),
8701 summary: c.summary.clone(),
8702 stat: c.stat.clone(),
8703 patch,
8704 }
8705 }
8706
8707 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8709 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8710 prompt::judge(
8711 "(see above)",
8712 &views,
8713 self.roles.judges.len(),
8714 base_short,
8715 "en",
8716 )
8717 }
8718
8719 fn vote_prompt_full(
8725 &self,
8726 j: usize,
8727 labels: &[char],
8728 language: &str,
8729 candidates: &[Candidate],
8730 base_short: &str,
8731 ) -> String {
8732 let mut text = format!(
8733 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8734 prompt::final_vote(labels, language),
8735 self.state.instruction,
8736 self.candidate_block(candidates, base_short)
8737 );
8738 if let Some(own) = self
8739 .state
8740 .judgements
8741 .get(j)
8742 .filter(|r| !r.ranking.is_empty())
8743 {
8744 let reasons = own
8745 .reasons
8746 .iter()
8747 .map(|(k, v)| format!("- {k}: {v}"))
8748 .collect::<Vec<_>>()
8749 .join("\n");
8750 text.push_str(&format!(
8751 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8752 own.ranking.iter().collect::<String>(),
8753 if reasons.is_empty() {
8754 ""
8755 } else {
8756 ", because:\n"
8757 }
8758 ));
8759 }
8760 if !self.state.deliberation.is_empty() {
8761 text.push_str("\n# What was argued before this vote\n");
8762 for t in self.transcript(&[], j) {
8763 text.push_str(&format!(
8764 "\n## {}{}\n\n{}\n",
8765 t.who,
8766 if t.is_self { " (you)" } else { "" },
8767 t.body.trim()
8768 ));
8769 }
8770 }
8771 text
8772 }
8773
8774 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8781 let mut turns = Vec::new();
8782 for j in &self.state.judgements {
8783 if j.ranking.is_empty() {
8784 continue;
8785 }
8786 let reasons = j
8787 .reasons
8788 .iter()
8789 .map(|(k, v)| format!("- {k}: {v}"))
8790 .collect::<Vec<_>>()
8791 .join("\n");
8792 turns.push(Turn {
8793 who: format!("Judge {} (opening ranking)", j.judge),
8794 is_self: j.judge == self_idx + 1,
8795 body: format!(
8796 "Ranked {}{}{reasons}",
8797 j.ranking.iter().collect::<String>(),
8798 if reasons.is_empty() {
8799 ""
8800 } else {
8801 ", because:\n"
8802 }
8803 ),
8804 });
8805 }
8806 for t in self
8807 .state
8808 .deliberation
8809 .iter()
8810 .flat_map(|r| r.turns.iter())
8811 .chain(current)
8812 {
8813 turns.push(Turn {
8814 who: format!("Judge {}", t.judge),
8815 is_self: t.judge == self_idx + 1,
8816 body: t.body.clone(),
8817 });
8818 }
8819 turns
8820 }
8821}
8822
8823fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8825 agent::has_session(spec.kind, seat, sessions)
8826}
8827
8828fn next_untried_in_roster<'a>(
8849 roster: &'a [AgentSpec],
8850 start: usize,
8851 tried: &BTreeSet<String>,
8852) -> Option<&'a AgentSpec> {
8853 roster
8854 .get(start + 1..)?
8855 .iter()
8856 .find(|s| !tried.contains(&s.id))
8857}
8858
8859fn pick_successor<'a>(
8880 roster: &'a [AgentSpec],
8881 start: usize,
8882 tried: &BTreeSet<String>,
8883 carried: Option<&BTreeSet<String>>,
8884 others: &BTreeSet<String>,
8885) -> Option<&'a AgentSpec> {
8886 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8887 roster
8888 .get(start + 1..)
8889 .and_then(|tail| {
8890 tail.iter()
8891 .filter(free)
8892 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8893 })
8894 .or_else(|| {
8895 carried
8896 .is_some()
8897 .then(|| roster.iter().find(free))
8898 .flatten()
8899 })
8900 .or_else(|| match carried {
8901 Some(c) => next_for_seat(roster, start, tried, c),
8902 None => next_untried_in_roster(roster, start, tried),
8903 })
8904}
8905
8906fn next_for_seat<'a>(
8916 roster: &'a [AgentSpec],
8917 start: usize,
8918 round_tried: &BTreeSet<String>,
8919 carried_failed: &BTreeSet<String>,
8920) -> Option<&'a AgentSpec> {
8921 roster
8922 .get(start + 1..)?
8923 .iter()
8924 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8925 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8926}
8927
8928fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8934 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8935 return spec;
8936 };
8937 let ok = |id: &str| !h.failed.contains(id);
8938 if let Some(last) = h.last_ok.as_deref()
8939 && ok(last)
8940 && let Some(s) = roster.iter().find(|s| s.id == last)
8941 {
8942 return s.clone();
8943 }
8944 if ok(&spec.id) {
8945 return spec;
8946 }
8947 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8948 roster
8949 .get(from + 1..)
8950 .into_iter()
8951 .flatten()
8952 .chain(roster.iter())
8953 .find(|s| ok(&s.id))
8954 .cloned()
8955 .unwrap_or(spec)
8956}
8957
8958pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8962 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8963}
8964
8965const TIMED_OUT: &str = "timed out";
8968
8969impl FailClass {
8970 fn of(out: &AgentOutcome) -> Option<Self> {
8972 match out {
8973 AgentOutcome::Ok(_) => None,
8974 AgentOutcome::Quota(_) => Some(Self::Quota),
8975 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8976 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8977 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8978 AgentOutcome::ResourceBlocked(_) => None,
8981 }
8982 }
8983
8984 fn stem_word(&self) -> &'static str {
8986 match self {
8987 Self::Quota => "quota",
8988 _ => "handover",
8989 }
8990 }
8991}
8992
8993fn failure_signature(msg: &str) -> String {
8997 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8998 if let Some(code) = line
9001 .strip_prefix("process exited with no usable output (exit code ")
9002 .and_then(|s| s.strip_suffix(')'))
9003 .and_then(|s| s.parse::<i32>().ok())
9004 {
9005 line = format!("exited with some({code}) and no usable output");
9006 } else if matches!(
9007 line.as_str(),
9008 "process exited with no usable output (no exit code reported)"
9009 | "process exited with no usable output (exit code unavailable)"
9010 ) {
9011 line = "exited with none and no usable output".to_owned();
9012 }
9013 let mut out = Vec::new();
9014 for word in line.split_whitespace() {
9015 if word.contains('/') || word.contains('\\') {
9016 out.push("<path>".to_owned());
9017 continue;
9018 }
9019 let mut w = String::new();
9020 let mut in_digits = false;
9021 for c in word.chars() {
9022 if c.is_ascii_digit() {
9023 if !in_digits {
9024 w.push('#');
9025 }
9026 in_digits = true;
9027 } else {
9028 in_digits = false;
9029 w.push(c);
9030 }
9031 }
9032 out.push(w);
9033 }
9034 out.join(" ").chars().take(120).collect()
9035}
9036
9037fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9044 match cur {
9045 FailClass::Quota | FailClass::Timeout => true,
9046 FailClass::Other(_) => prev != Some(cur),
9047 }
9048}
9049
9050fn fail_reason(out: &AgentOutcome) -> String {
9052 match out {
9053 AgentOutcome::Ok(_) => String::new(),
9054 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9055 AgentOutcome::Dropped(o) => format!(
9056 "the CLI dropped the stream ({})",
9057 o.dropped
9058 .as_ref()
9059 .map(|d| d.why.as_str())
9060 .unwrap_or("it ended without delivering its answer")
9061 ),
9062 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9063 e.lines().next().unwrap_or("").chars().take(160).collect()
9064 }
9065 }
9066}
9067
9068pub(crate) fn record_handover(
9071 state: &mut RunState,
9072 node: &str,
9073 seat: &str,
9074 from: &str,
9075 to: &str,
9076 class: &FailClass,
9077 reason: &str,
9078) {
9079 let message = if *class == FailClass::Quota {
9080 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9081 } else {
9082 format!("{seat}: handed over {from} -> {to} ({reason})")
9083 };
9084 state.event(node, message);
9085 state.handovers.push(Handover {
9086 at: Timestamp::now(),
9087 node: node.to_owned(),
9088 seat: seat.to_owned(),
9089 from: from.to_owned(),
9090 to: to.to_owned(),
9091 reason: reason.to_owned(),
9092 });
9093}
9094
9095fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9109 commands.iter().any(|c| c.exit_code.is_none())
9110}
9111
9112fn verified_noop_claim(
9125 usable: bool,
9126 commands: &[agent::CommandEvidence],
9127 text: &str,
9128) -> Option<String> {
9129 (usable && !has_unconfirmed_command(commands))
9130 .then(|| verdict::verified_noop(text))
9131 .flatten()
9132}
9133
9134fn short(commit: &str) -> String {
9135 commit.chars().take(7).collect()
9136}
9137
9138fn make_executable(path: &Path) -> Result<()> {
9139 #[cfg(unix)]
9140 {
9141 use std::os::unix::fs::PermissionsExt as _;
9142 let mut perms = std::fs::metadata(path)?.permissions();
9143 perms.set_mode(0o755);
9144 std::fs::set_permissions(path, perms)?;
9145 }
9146 #[cfg(not(unix))]
9147 {
9148 let _ = path;
9149 }
9150 Ok(())
9151}
9152
9153struct WaveCtx<'a> {
9160 run: &'a str,
9163 node: &'a str,
9165 prompts: &'a Prompts,
9166 cache: Option<&'a Path>,
9168 round: Option<usize>,
9171 carry_seats: bool,
9175 pauses: Option<(&'a Pause, &'a Pause)>,
9178}
9179
9180fn repository_wait(
9182 state: &RunState,
9183 pauses: Option<(&Pause, &Pause)>,
9184) -> crate::repo_lock::WaitPolicy {
9185 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9186 let inherited = crate::repo_lock::WaitPolicy::current();
9187 crate::repo_lock::WaitPolicy::new(
9188 Duration::from_secs(state.config.graph.timeout_resource_wait),
9189 move || {
9190 inherited.cancelled()
9191 || pauses
9192 .as_ref()
9193 .is_some_and(|(a, b)| a.parked() || b.parked())
9194 },
9195 )
9196}
9197
9198fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9199 if error.is::<crate::repo_lock::ResourceBlocked>() {
9200 state.resource_block = Some(crate::run::ResourceBlock {
9201 node: "repository".into(),
9202 kind: if error.to_string().contains("disk pressure") {
9203 "disk pressure"
9204 } else {
9205 "repository coordination"
9206 }
9207 .into(),
9208 detail: error.to_string(),
9209 at: Timestamp::now(),
9210 });
9211 state.event("resource", error.to_string());
9212 }
9213}
9214
9215async fn coordinate_operation(
9216 state: &mut RunState,
9217 cwd: &Path,
9218 wait: &crate::repo_lock::WaitPolicy,
9219) -> Result<Option<crate::repo_lock::SeatLease>> {
9220 let result =
9221 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9222 .await;
9223 if let Err(e) = &result {
9224 record_repository_block(state, e);
9225 }
9226 result
9227}
9228
9229async fn run_one(
9231 job: SeatJob,
9232 sem: Arc<Semaphore>,
9233 ctx: &WaveCtx<'_>,
9234 state: &mut RunState,
9235 attempt: usize,
9236) -> (SeatState, AgentOutcome) {
9237 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9238 .await
9239 .pop()
9240 .expect("one job in, one result out");
9241 (seat, out)
9242}
9243
9244async fn wave(
9250 jobs: Vec<SeatJob>,
9251 sem: Arc<Semaphore>,
9252 ctx: &WaveCtx<'_>,
9253 state: &mut RunState,
9254 attempt: usize,
9255) -> Vec<(usize, SeatState, AgentOutcome)> {
9256 let WaveCtx {
9257 run,
9258 node,
9259 prompts,
9260 cache,
9261 round,
9262 carry_seats: _,
9263 pauses,
9264 } = *ctx;
9265 for job in &jobs {
9266 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9267 }
9268 if let Err(e) = state.save() {
9269 tracing::warn!("could not persist in-progress seats: {e:#}");
9274 }
9275 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9291 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9295 let cache_guard = if let Some(cache_dir) = cache {
9296 if jobs_had_a_writer {
9297 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9298 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9299 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9300 {
9301 Ok(Some(g)) => Some(g),
9302 Ok(None) => {
9305 state.resource_block = Some(crate::run::ResourceBlock {
9306 node: node.to_owned(),
9307 kind: "shared build cache".to_owned(),
9308 detail: "the run is parking".to_owned(),
9309 at: Timestamp::now(),
9310 });
9311 return block_wave(
9312 jobs,
9313 state,
9314 node,
9315 attempt,
9316 "parked while waiting for the shared build cache",
9317 );
9318 }
9319 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9322 Err(e) => {
9323 let why = format!("{e:#}");
9327 state.resource_block = Some(crate::run::ResourceBlock {
9328 node: node.to_owned(),
9329 kind: "shared build cache".to_owned(),
9330 detail: why.clone(),
9331 at: Timestamp::now(),
9332 });
9333 state.event(
9334 "resource",
9335 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9336 );
9337 return block_wave(jobs, state, node, attempt, &why);
9338 }
9339 }
9340 } else {
9341 None
9342 }
9343 } else {
9344 None
9345 };
9346 state.seats_spawning(node, attempt);
9349 if let Err(e) = state.save() {
9350 tracing::warn!("could not persist the start of a wave: {e:#}");
9351 }
9352 let slot_limit = resource_limit
9356 + jobs
9357 .iter()
9358 .map(|j| j.timeout)
9359 .max()
9360 .unwrap_or_default()
9361 .saturating_mul(jobs.len() as u32);
9362 let mut set = tokio::task::JoinSet::new();
9363 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9366 let overlay = prompts.overlay(node);
9367 let repository_wait = repository_wait(state, pauses);
9368 for (i, mut job) in jobs.into_iter().enumerate() {
9369 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9370 if cache.is_some() {
9371 job.prompt.push('\n');
9372 job.prompt
9373 .push_str(&prompt::build_cache_note(node, job.allow_write));
9374 }
9375 let sem = Arc::clone(&sem);
9376 let run = run.to_owned();
9377 let node = node.to_owned();
9378 let attachments = if node == "implement" {
9381 state.attachments.clone()
9382 } else {
9383 Vec::new()
9384 };
9385 let cache = cache
9396 .filter(|_| job.allow_write && cache_guard.is_some())
9397 .map(Path::to_path_buf);
9398 let spawned_tx = spawned_tx.clone();
9399 let repository_wait = repository_wait.clone();
9400 let coordination = state.config.coordination.clone();
9401 let fetched_base = state.fetched_base.clone();
9402 set.spawn(async move {
9403 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9407 Ok(p) => p,
9408 Err(_) => {
9409 return (
9410 i,
9411 job.seat,
9412 AgentOutcome::ResourceBlocked(format!(
9413 "no parallel slot became free within {}s",
9414 slot_limit.as_secs()
9415 )),
9416 false,
9417 );
9418 }
9419 };
9420 spawned_tx.send(job.seat.key.clone()).ok();
9421 let mut seat = job.seat;
9422 let out = repository_wait
9423 .scope(agent::invoke(
9424 &job.spec,
9425 &mut seat,
9426 &Invocation {
9427 scope: crate::agent::InvocationScope::Repository,
9428 coordination: coordination.as_ref(),
9429 fetched_base: fetched_base.as_ref(),
9430 cwd: &job.cwd,
9431 prompt: &job.prompt,
9432 timeout: job.timeout,
9433 allow_write: job.allow_write,
9434 unsandboxed: false,
9435 sessions: job.sessions,
9436 artifacts: &job.artifacts,
9437 stem: &job.stem,
9438 run: &run,
9439 node: &node,
9440 cache_dir: cache.as_deref(),
9441 attachments: &attachments,
9442 writable: &[],
9443 },
9444 ))
9445 .await;
9446 let interrupted = out
9447 .as_ref()
9448 .err()
9449 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9450 let out = match out {
9451 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9452 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9453 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9461 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9462 Ok(o) => AgentOutcome::Failed(format!(
9463 "process exited with no usable output ({})",
9464 crate::notices::exit_status(o.exit_code)
9465 )),
9466 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9467 AgentOutcome::ResourceBlocked(e.to_string())
9468 }
9469 Err(e) => AgentOutcome::Failed(e.to_string()),
9470 };
9471 (i, seat, out, interrupted)
9472 });
9473 }
9474 drop(spawned_tx);
9475 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9476 loop {
9477 let joined = tokio::select! {
9478 Some(key) = spawned_rx.recv() => {
9479 state.seat_spawned(&key);
9481 if let Err(e) = state.save() {
9482 tracing::warn!("could not persist a seat's start: {e:#}");
9483 }
9484 continue;
9485 }
9486 joined = set.join_next() => joined,
9487 };
9488 let Some(joined) = joined else {
9489 break;
9490 };
9491 let (i, seat, out, interrupted) = match joined {
9492 Ok(v) => v,
9493 Err(e) => {
9497 tracing::error!("agent task panicked: {e}");
9498 continue;
9499 }
9500 };
9501 state.seat_finished(&seat.key);
9502 if interrupted
9503 && node == "implement"
9504 && let Some(label) = state
9505 .candidates
9506 .iter()
9507 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9508 .map(|candidate| candidate.label)
9509 {
9510 state.candidate_preservation.insert(
9511 label,
9512 "interrupted implementation awaiting persistence".into(),
9513 );
9514 state.interrupted_implementations.insert(label);
9515 }
9516 if let AgentOutcome::ResourceBlocked(why) = &out
9517 && state.resource_block.is_none()
9518 {
9519 state.resource_block = Some(crate::run::ResourceBlock {
9520 node: node.to_owned(),
9521 kind: if why.contains("disk pressure") {
9522 "disk pressure"
9523 } else if why.contains("repository coordination") {
9524 "repository coordination"
9525 } else {
9526 "parallel slot"
9527 }
9528 .to_owned(),
9529 detail: why.clone(),
9530 at: Timestamp::now(),
9531 });
9532 let activity = if interrupted {
9533 "agent execution interrupted"
9534 } else {
9535 "no agent was started"
9536 };
9537 state.event("resource", format!("{node}: {activity} - {why}"));
9538 }
9539 record_jobs(state, node, round, &seat.key, &out);
9540 if let Err(e) = state.save() {
9541 tracing::warn!("could not persist a seat's completion: {e:#}");
9542 }
9543 if collected.len() <= i {
9544 collected.resize_with(i + 1, || None);
9545 }
9546 collected[i] = Some((i, seat, out));
9547 }
9548 if state
9554 .active
9555 .values()
9556 .any(|a| a.node == node && a.attempt == attempt)
9557 {
9558 state
9559 .active
9560 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9561 if let Err(e) = state.save() {
9562 tracing::warn!("could not persist the end of a wave: {e:#}");
9563 }
9564 }
9565 if let Some(cache_dir) = cache
9572 && jobs_had_a_writer
9573 {
9574 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9575 }
9576 if let Some(guard) = cache_guard {
9577 guard.release();
9578 }
9579 collected.into_iter().flatten().collect()
9580}
9581
9582fn record_jobs(
9593 state: &mut RunState,
9594 node: &str,
9595 round: Option<usize>,
9596 seat: &str,
9597 out: &AgentOutcome,
9598) {
9599 let commands: &[agent::CommandEvidence] = match out {
9600 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9601 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9602 };
9603 let checked_at = Timestamp::now();
9604 for c in commands {
9605 state.jobs.push(JobRecord {
9606 node: node.to_owned(),
9607 round,
9608 seat: seat.to_owned(),
9609 id: c.id.clone(),
9610 description: c.description.clone(),
9611 checked_at,
9612 status: match c.exit_code {
9613 Some(0) => JobStatus::Completed,
9614 Some(_) => JobStatus::Failed,
9615 None => JobStatus::Unknown,
9616 },
9617 exit_code: c.exit_code,
9618 result_summary: c.result_summary.clone(),
9619 source: c.source.clone(),
9620 });
9621 }
9622}
9623
9624fn round_is_clean(
9645 blocking: usize,
9646 e2e_ok: bool,
9647 answered: usize,
9648 expected: usize,
9649 quota_missing: usize,
9650 policy: IncompleteReviewPolicy,
9651) -> bool {
9652 if blocking != 0 || !e2e_ok {
9653 return false;
9654 }
9655 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9656 return true;
9657 }
9658 answered > 0 && expected - answered <= quota_missing
9659}
9660
9661pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9685 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9686 return Some(RunStatus::Gating);
9687 }
9688 let last = reviews.last()?;
9689 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9690 if reviews.len() < max_rounds && !stagnant {
9691 return None;
9692 }
9693 if last.incomplete() && last.blocking == 0 {
9694 return Some(RunStatus::Blocked);
9695 }
9696 if last.e2e_status() == E2eStatus::ResourceBlocked {
9697 return None;
9698 }
9699 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9700 RunStatus::Gating
9701 } else {
9702 RunStatus::Blocked
9703 })
9704}
9705
9706pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9721 if nudged {
9722 (full / 4).max(Duration::from_secs(120)).min(full)
9723 } else {
9724 full
9725 }
9726}
9727
9728#[allow(clippy::too_many_arguments)]
9757async fn ask_json_wave<T>(
9758 jobs: Vec<SeatJob>,
9759 sem: Arc<Semaphore>,
9760 retries: usize,
9761 roster: &[AgentSpec],
9762 ctx: &WaveCtx<'_>,
9763 losses: &mut Vec<QuotaLoss>,
9764 state: &mut RunState,
9765 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9766) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9767where
9768 T: serde::de::DeserializeOwned + Send + 'static,
9769{
9770 ask_wave_with(
9771 jobs,
9772 sem,
9773 retries,
9774 roster,
9775 ctx,
9776 losses,
9777 state,
9778 &|text: &str| {
9779 let v = verdict::extract_json::<T>(text)?;
9780 validate(&v)?;
9781 Ok(v)
9782 },
9783 )
9784 .await
9785}
9786
9787#[allow(clippy::too_many_arguments)]
9796async fn ask_wave_with<T>(
9797 jobs: Vec<SeatJob>,
9798 sem: Arc<Semaphore>,
9799 retries: usize,
9800 roster: &[AgentSpec],
9801 ctx: &WaveCtx<'_>,
9802 losses: &mut Vec<QuotaLoss>,
9803 state: &mut RunState,
9804 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9805) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9806where
9807 T: Send + 'static,
9808{
9809 let n = jobs.len();
9810 let originals: Vec<SeatJob> = jobs;
9811 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9812 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9813 let mut nudges: Vec<usize> = vec![0; n];
9820 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9824 let mut tried: Vec<BTreeSet<String>> = specs
9825 .iter()
9826 .map(|s| BTreeSet::from([s.id.clone()]))
9827 .collect();
9828 let starts: Vec<usize> = specs
9829 .iter()
9830 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9831 .collect();
9832 let carry = ctx.carry_seats && !roster.is_empty();
9833 let carried: Vec<BTreeSet<String>> = originals
9836 .iter()
9837 .map(|j| {
9838 state
9839 .seat_history
9840 .get(&j.seat.key)
9841 .filter(|_| carry)
9842 .map(|h| h.failed.clone())
9843 .unwrap_or_default()
9844 })
9845 .collect();
9846 let mut prev: Vec<Option<FailClass>> = originals
9847 .iter()
9848 .map(|j| {
9849 state
9850 .seat_history
9851 .get(&j.seat.key)
9852 .filter(|_| carry)
9853 .and_then(|h| h.last_fail.clone())
9854 })
9855 .collect();
9856 let next_agent =
9857 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9858 let others: BTreeSet<String> = specs
9859 .iter()
9860 .enumerate()
9861 .filter(|(j, _)| *j != i)
9862 .map(|(_, s)| s.id.clone())
9863 .collect();
9864 pick_successor(
9865 roster,
9866 starts[i],
9867 tried,
9868 carry.then(|| &carried[i]),
9869 &others,
9870 )
9871 .cloned()
9872 };
9873 let mut fresh: Vec<Option<String>> = vec![None; n];
9874 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9875 let mut pending: Vec<usize> = (0..n).collect();
9876
9877 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9882 for round in 0..max_rounds {
9883 if pending.is_empty() {
9884 break;
9885 }
9886 let mut batch = Vec::with_capacity(pending.len());
9887 let mut renudged: Vec<&str> = Vec::new();
9888 for &i in &pending {
9889 let src = &originals[i];
9890 let full: &str = match &src.handover {
9895 Some(h) if specs[i].id != src.spec.id => h,
9896 _ => &src.prompt,
9897 };
9898 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9901 (
9902 full.to_owned(),
9903 src.timeout,
9904 format!("{}-{word}-{}", src.stem, specs[i].id),
9905 )
9906 } else if nudges[i] == 0 {
9907 (full.to_owned(), src.timeout, src.stem.clone())
9908 } else {
9909 renudged.push(src.seat.key.as_str());
9910 let why = done[i]
9911 .as_ref()
9912 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9913 .unwrap_or_else(|| "no parsable answer".to_owned());
9914 let nudge = prompt::nudge(&why);
9915 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9916 let prompt = if nudged {
9917 nudge
9918 } else {
9919 format!("{full}\n\n---\n\n{nudge}")
9920 };
9921 (
9922 prompt,
9923 retry_budget(src.timeout, nudged),
9924 format!("{}-retry{}", src.stem, nudges[i]),
9925 )
9926 };
9927 batch.push(SeatJob {
9928 spec: specs[i].clone(),
9929 seat: seats[i].clone(),
9930 cwd: src.cwd.clone(),
9931 prompt,
9932 timeout,
9933 allow_write: src.allow_write,
9934 sessions: src.sessions,
9935 artifacts: src.artifacts.clone(),
9936 stem,
9937 handover: None,
9938 });
9939 }
9940
9941 if !renudged.is_empty() {
9942 state.event(
9943 ctx.node,
9944 format!("retry {round}: re-asking {}", renudged.join(", ")),
9945 );
9946 }
9947 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9948 let mut still = Vec::new();
9949 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9950 seats[i] = seat;
9951 if let AgentOutcome::ResourceBlocked(why) = &out {
9954 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9955 continue;
9956 }
9957 let class = FailClass::of(&out);
9958 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9961 && nudges[i] < retries
9962 && !roster.is_empty();
9963 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9964 let next = should_hand_over(prev[i].as_ref(), &cur)
9965 .then(|| next_agent(i, &tried[i], &specs))
9966 .flatten();
9967 if carry {
9968 let h = state
9969 .seat_history
9970 .entry(originals[i].seat.key.clone())
9971 .or_default();
9972 h.failed.insert(specs[i].id.clone());
9973 h.last_fail = Some(cur.clone());
9974 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9975 h.last_ok = None;
9976 }
9977 if let Err(e) = state.save() {
9980 tracing::warn!("could not persist a seat's failure history: {e:#}");
9981 }
9982 }
9983 if let Some(next) = next {
9984 record_handover(
9985 state,
9986 ctx.node,
9987 &originals[i].seat.key,
9988 &specs[i].id,
9989 &next.id,
9990 &cur,
9991 &fail_reason(&out),
9992 );
9993 tried[i].insert(next.id.clone());
9994 prev[i] = Some(cur.clone());
9995 seats[i] =
9996 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9997 specs[i] = next;
9998 fresh[i] = Some(cur.stem_word().to_owned());
9999 nudges[i] = 0;
10000 done[i] = Some(Err(anyhow::anyhow!(
10001 "handed over after: {}",
10002 fail_reason(&out)
10003 )));
10004 still.push(i);
10005 continue;
10006 }
10007 }
10008 if carry
10009 && !nudge_first
10010 && let Some(cur) = class.clone()
10011 {
10012 let h = state
10014 .seat_history
10015 .entry(originals[i].seat.key.clone())
10016 .or_default();
10017 h.failed.insert(specs[i].id.clone());
10018 h.last_fail = Some(cur);
10019 }
10020 let parsed = match out {
10021 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10022 AgentOutcome::Quota(o) => {
10023 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10024 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10025 }
10026 AgentOutcome::Dropped(o) => {
10030 let why = o
10031 .dropped
10032 .as_ref()
10033 .map(|d| d.why.as_str())
10034 .unwrap_or("the CLI ended the stream without delivering its answer");
10035 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10036 }
10037 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10038 Err(anyhow::anyhow!(e))
10039 }
10040 };
10041 let quota = class == Some(FailClass::Quota);
10042 let failed = parsed.is_err();
10043 done[i] = Some(parsed);
10044 if carry && !failed {
10045 let h = state
10046 .seat_history
10047 .entry(originals[i].seat.key.clone())
10048 .or_default();
10049 h.failed.remove(&specs[i].id);
10050 h.last_ok = Some(specs[i].id.clone());
10051 h.last_fail = None;
10052 }
10053 let agent_failure = class.is_some()
10061 && !nudge_first
10062 && !roster.is_empty()
10063 && next_agent(i, &tried[i], &specs).is_some();
10064 if failed && !quota && !agent_failure && nudges[i] < retries {
10065 nudges[i] += 1;
10066 still.push(i);
10067 }
10068 }
10069 pending = still;
10070 }
10071
10072 for (i, q) in last_quota.into_iter().enumerate() {
10075 if let Some(reset) = q {
10076 losses.push(QuotaLoss {
10077 seat: originals[i].seat.key.clone(),
10078 node: ctx.node.to_owned(),
10079 at: Timestamp::now(),
10080 reset,
10081 });
10082 }
10083 }
10084
10085 seats
10086 .into_iter()
10087 .zip(done)
10088 .zip(nudges)
10089 .map(|((seat, res), attempts)| {
10090 (
10091 seat,
10092 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10093 attempts,
10094 )
10095 })
10096 .collect()
10097}
10098
10099fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10101 results
10102 .iter()
10103 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10104}
10105
10106fn block_wave(
10110 jobs: Vec<SeatJob>,
10111 state: &mut RunState,
10112 node: &str,
10113 attempt: usize,
10114 why: &str,
10115) -> Vec<(usize, SeatState, AgentOutcome)> {
10116 state.resource_wait = None;
10117 state
10118 .active
10119 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10120 if let Err(e) = state.save() {
10121 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10122 }
10123 jobs.into_iter()
10124 .enumerate()
10125 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10126 .collect()
10127}
10128
10129async fn acquire_cache_lease(
10142 state: &mut RunState,
10143 cache_dir: &Path,
10144 owner: &crate::cache::Owner,
10145 budget: Duration,
10146 context: &str,
10147 cancel: &(dyn Fn() -> bool + Sync),
10148) -> Result<Option<crate::cache::Guard>> {
10149 let home = crate::run::home();
10150 let started = Instant::now();
10151 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10152 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10153 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10154 Err(e) => {
10155 state.event(
10156 "verify",
10157 format!("{context}: could not check the shared build cache: {e:#}"),
10158 );
10159 if let Err(e2) = state.save() {
10160 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10161 }
10162 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10163 }
10164 };
10165 state.event(
10166 "verify",
10167 format!(
10168 "{context}: waiting for the shared build cache at {} ({})",
10169 cache_dir.display(),
10170 busy.describe()
10171 ),
10172 );
10173 state.resource_wait = Some(crate::run::ResourceWait {
10176 node: context.to_owned(),
10177 kind: "shared build cache".to_owned(),
10178 holder: busy.describe(),
10179 since: Timestamp::now(),
10180 limit_secs: budget.as_secs(),
10181 });
10182 if let Err(e) = state.save() {
10183 tracing::warn!("could not persist a cache wait: {e:#}");
10184 }
10185 let remaining = budget.saturating_sub(started.elapsed());
10186 let waited = crate::cache::wait_for_unless(
10187 &home,
10188 cache_dir,
10189 owner,
10190 remaining,
10191 Duration::from_secs(5),
10192 cancel,
10193 )
10194 .await;
10195 state.resource_wait = None;
10196 match waited {
10197 Ok(g) => Ok(g),
10198 Err(e) => {
10199 state.event("verify", format!("{context}: {e:#}"));
10200 if let Err(e2) = state.save() {
10201 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10202 }
10203 Err(e)
10204 }
10205 }
10206}
10207
10208#[derive(Debug)]
10213struct CacheCheckError(String);
10214
10215impl std::fmt::Display for CacheCheckError {
10216 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10217 f.write_str(&self.0)
10218 }
10219}
10220
10221impl std::error::Error for CacheCheckError {}
10222
10223#[allow(clippy::too_many_arguments)]
10244async fn with_cache_lease<'s, F, Fut>(
10245 state: &'s mut RunState,
10246 cache_dir: Option<&Path>,
10247 node: &str,
10248 seat: &str,
10249 worktree: &Path,
10250 head: &str,
10251 budget: Duration,
10252 context: &str,
10253 wait: &crate::repo_lock::WaitPolicy,
10254 body: F,
10255) -> Result<(Vec<CommandOutcome>, bool)>
10256where
10257 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10258 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10259{
10260 let started = Instant::now();
10263 let home = crate::run::home();
10264 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10265 let guard = if let Some(cache_dir) = cache_dir {
10266 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10267 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10268 wait.cancelled()
10269 })
10270 .await
10271 .and_then(|g| {
10272 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10273 }) {
10274 Ok(g) => Some(g),
10275 Err(e) => {
10276 return Ok((
10277 vec![CommandOutcome {
10278 command: "(waiting for the shared build cache)".into(),
10279 code: None,
10280 output_tail: e.to_string(),
10281 duration_ms: started.elapsed().as_millis() as u64,
10282 resource_blocked: true,
10283 }],
10284 false,
10285 ));
10286 }
10287 }
10288 } else {
10289 None
10290 };
10291 let _coordination = coordinate_operation(state, worktree, wait).await?;
10292 if let Some(cache_dir) = cache_dir {
10293 let identity = crate::cache::Identity::new(worktree, head);
10294 if let Err(e) = crate::cache::ensure_fresh(
10295 &home,
10296 cache_dir,
10297 &identity,
10298 &state.config.verify.cargo_manifests(),
10299 ) {
10300 state.event(
10301 "verify",
10302 format!(
10303 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10304 worktree.display(),
10305 short(head)
10306 ),
10307 );
10308 if let Some(guard) = guard {
10309 guard.release();
10310 }
10311 return Ok((
10312 vec![CommandOutcome {
10313 command: "(confirming the shared build cache is fresh)".into(),
10314 code: None,
10315 output_tail: e.to_string(),
10316 duration_ms: started.elapsed().as_millis() as u64,
10317 resource_blocked: true,
10318 }],
10319 false,
10320 ));
10321 }
10322 }
10323 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10324 if !timed_out_pids.is_empty() {
10325 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10326 }
10327 drop(_coordination);
10330 if let Some(guard) = guard {
10331 guard.release();
10332 }
10333 Ok((outcomes, retried))
10334}
10335
10336async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10348 wait_for_pids_with(
10349 pids,
10350 crate::proc::pid_alive,
10351 LEASE_RELEASE_POLL,
10352 LEASE_RELEASE_MAX_WAIT,
10353 )
10354 .await;
10355}
10356
10357async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10363 pids: &[u32],
10364 alive: F,
10365 poll: Duration,
10366 max_wait: Duration,
10367) {
10368 let deadline = Instant::now() + max_wait;
10369 loop {
10370 if pids.iter().all(|&pid| !alive(pid)) {
10371 return;
10372 }
10373 if Instant::now() >= deadline {
10374 return;
10375 }
10376 tokio::time::sleep(poll).await;
10377 }
10378}
10379
10380fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10387 outcomes.iter().any(|o| o.resource_blocked)
10388}
10389
10390pub(crate) enum GateFix {
10392 Retry,
10394 Stop,
10397 Defer,
10400}
10401
10402fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10410 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10411 red.peek().is_some()
10412 && red.all(|o| {
10413 !o.resource_blocked
10414 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10415 && !o.output_tail.trim().is_empty()
10416 })
10417}
10418
10419fn e2e_outcome_label(o: &CommandOutcome) -> String {
10423 if o.ok() {
10424 return "pass".to_owned();
10425 }
10426 let reason = if o.build_failed() {
10427 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10428 } else {
10429 format!("FAIL ({:?})", o.code)
10430 };
10431 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10432}
10433
10434async fn run_e2e_with_retry(
10442 state: &mut RunState,
10443 shell: &[String],
10444 commands: &[String],
10445 worktree: &Path,
10446 timeout: Duration,
10447 context: &str,
10448) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10449 let (mut e2e, mut timed_out_pids) = run_commands(
10450 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10451 )
10452 .await;
10453 for o in &e2e {
10454 state.event(
10455 "verify",
10456 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10457 );
10458 }
10459 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10463 if verify_retried {
10464 state.event(
10465 "verify",
10466 format!(
10467 "{context}: verify could not build/link, not a test result — retrying once \
10468 before concluding"
10469 ),
10470 );
10471 let retried = run_commands(
10472 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10473 )
10474 .await;
10475 e2e = retried.0;
10476 timed_out_pids.extend(retried.1);
10479 for o in &e2e {
10480 state.event(
10481 "verify",
10482 format!(
10483 "{context}: retry `{}` -> {}",
10484 o.command,
10485 e2e_outcome_label(o)
10486 ),
10487 );
10488 }
10489 }
10490 (e2e, verify_retried, timed_out_pids)
10491}
10492
10493#[allow(clippy::too_many_arguments)]
10509async fn run_commands(
10510 state: &mut RunState,
10511 node: &str,
10512 task: &str,
10513 attempt: usize,
10514 shell: &[String],
10515 commands: &[String],
10516 cwd: &Path,
10517 timeout: Duration,
10518) -> (Vec<CommandOutcome>, Vec<u32>) {
10519 if commands.is_empty() {
10520 return (Vec::new(), Vec::new());
10525 }
10526 let mut out = Vec::new();
10527 let mut timed_out_pids = Vec::new();
10528 let total = commands.len();
10529 for (idx, command) in commands.iter().enumerate() {
10530 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10531 if let Err(e) = state.save() {
10532 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10533 }
10534 let started = Instant::now();
10535 let mut cmd = tokio::process::Command::new(&shell[0]);
10536 cmd.quiet();
10537 cmd.args(&shell[1..])
10538 .arg(command)
10539 .current_dir(cwd)
10540 .stdin(std::process::Stdio::null())
10541 .stdout(std::process::Stdio::piped())
10542 .stderr(std::process::Stdio::piped())
10543 .kill_on_drop(true);
10544 let mut outputs = Vec::new();
10545 let protected = (|| -> Result<()> {
10546 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10547 outputs.push(guard);
10548 }
10549 if let Some(cache) = state.config.cache_dir() {
10550 outputs.push(crate::retention::OutputLease::cache(
10551 &crate::run::home(),
10552 &cache,
10553 false,
10554 )?);
10555 }
10556 Ok(())
10557 })();
10558 if let Err(e) = protected {
10559 out.push(CommandOutcome {
10560 command: command.clone(),
10561 code: None,
10562 output_tail: format!("output ownership unavailable: {e:#}"),
10563 duration_ms: 0,
10564 resource_blocked: true,
10565 });
10566 continue;
10567 }
10568 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10569 let (code, body) = match spawned {
10570 Ok(mut running) => {
10571 let pid = running.child().id();
10572 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10573 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10574 let mut paths = vec![cwd.to_owned()];
10575 if let Some(cache) = state.config.cache_dir() {
10576 paths.push(cache);
10577 }
10578 let mut pressure = None;
10579 let status = tokio::select! {
10580 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10581 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10582 pressure = Some(reason); None
10583 }
10584 };
10585 let (code, failure) = match status {
10586 Some(Ok(Ok(status))) => (
10587 status.code(),
10588 status
10589 .code()
10590 .is_none()
10591 .then(|| format!("command terminated: {status}")),
10592 ),
10593 Some(Ok(Err(error))) => {
10594 (None, Some(format!("failed waiting for command: {error}")))
10595 }
10596 Some(Err(_)) => {
10597 if !running.coordinated()
10601 && let Some(pid) = pid
10602 {
10603 timed_out_pids.push(pid);
10604 }
10605 (
10606 None,
10607 Some(format!("timed out after {}s", timeout.as_secs())),
10608 )
10609 }
10610 None => (None, None),
10611 };
10612 running.finish().await;
10613 if let Some(reason) = pressure {
10614 state.resource_block = Some(crate::run::ResourceBlock {
10615 node: node.to_owned(),
10616 kind: "disk pressure".to_owned(),
10617 detail: reason.clone(),
10618 at: Timestamp::now(),
10619 });
10620 out.push(CommandOutcome {
10621 command: command.clone(),
10622 code: None,
10623 output_tail: reason,
10624 duration_ms: started.elapsed().as_millis() as u64,
10625 resource_blocked: true,
10626 });
10627 break;
10628 }
10629 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10630 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10631 if let Some(failure) = failure {
10632 body.push('\n');
10633 body.push_str(&failure);
10634 }
10635 (code, body)
10636 }
10637 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10638 };
10639 out.push(CommandOutcome {
10640 command: command.clone(),
10641 code,
10642 output_tail: tail(&body, OUTPUT_TAIL),
10643 duration_ms: started.elapsed().as_millis() as u64,
10644 resource_blocked: false,
10645 });
10646 }
10647 state.task_finished(task);
10648 if let Err(e) = state.save() {
10649 tracing::warn!("could not persist the end of {task}: {e:#}");
10650 }
10651 (out, timed_out_pids)
10652}
10653
10654fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10666 let repo = repo.display();
10667 match style {
10668 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10669 MergeStyle::Squash => {
10670 let subject = message
10673 .lines()
10674 .next()
10675 .unwrap_or(branch)
10676 .replace(['\\', '"', '$', '`'], "");
10677 format!(
10678 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10679 )
10680 }
10681 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10682 }
10683}
10684
10685const PR_TITLE_MAX: usize = 240;
10696
10697fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10708 let line = queue::first_line(text)?;
10709 let chars: Vec<char> = line.chars().collect();
10710 let (kept, mark) = if chars.len() <= max {
10711 (line.to_owned(), "")
10712 } else if let Some(end) = sentence_end(&chars, max) {
10713 (chars[..end].iter().collect::<String>(), "")
10714 } else {
10715 let room = max.saturating_sub(3);
10716 let cut = if chars[room].is_whitespace() {
10719 room
10720 } else {
10721 chars[..room]
10722 .iter()
10723 .rposition(|c| c.is_whitespace())
10724 .unwrap_or(room)
10725 };
10726 let head: String = chars[..cut].iter().collect();
10727 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10728 (head.to_owned(), "...")
10729 };
10730 if kept.is_empty() {
10731 return None;
10732 }
10733 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10734 return None;
10735 }
10736 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10737 return None;
10738 }
10739 Some(format!("{kept}{mark}"))
10740}
10741
10742fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10746 const MIN: usize = 20;
10747 for i in MIN..max.min(chars.len()) {
10748 if !matches!(chars[i], '.' | '!' | '?') {
10749 continue;
10750 }
10751 let Some(&next) = chars.get(i + 1) else {
10752 continue;
10753 };
10754 if !next.is_whitespace() {
10755 continue;
10756 }
10757 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10758 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10759 continue;
10760 }
10761 let word: String = chars[..i]
10762 .iter()
10763 .rev()
10764 .take_while(|c| !c.is_whitespace())
10765 .collect::<Vec<_>>()
10766 .into_iter()
10767 .rev()
10768 .collect();
10769 let word = word.to_ascii_lowercase();
10770 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10771 continue;
10772 }
10773 let end = chars[..i]
10774 .iter()
10775 .rposition(|c| !c.is_whitespace())
10776 .map_or(i, |p| p + 1);
10777 return Some(end);
10778 }
10779 None
10780}
10781
10782struct PrMessage {
10787 title: String,
10788 body: String,
10789}
10790
10791impl PrMessage {
10792 fn commit_message(&self) -> String {
10796 format!("{}\n\n{}", self.title, self.body)
10797 }
10798}
10799
10800fn title_marker(line: &str) -> Option<&str> {
10802 let line = line.trim();
10803 let head = line.get(..6)?;
10804 head.eq_ignore_ascii_case("title:")
10805 .then(|| line[6..].trim())
10806}
10807
10808fn summary_title(summary: &str) -> Option<String> {
10813 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10814 let raw = title_marker(first)?;
10815 if raw.is_empty() {
10816 return None;
10817 }
10818 let title = queue::title_from(raw, PR_TITLE_MAX);
10819 let lower = title.to_ascii_lowercase();
10820 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10821 return None;
10822 }
10823 Some(title)
10824}
10825
10826const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10828
10829pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10831
10832pub(crate) fn is_review_run(state: &RunState) -> bool {
10836 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10837}
10838
10839fn review_title(state: &RunState) -> Option<String> {
10844 english_subject(
10845 state.reviewed_commits.as_ref()?.first()?,
10846 state.config.graph.github_english(),
10847 )
10848}
10849
10850fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10853 let raw = raw.trim();
10854 if raw.is_empty() {
10855 return None;
10856 }
10857 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10858 return None;
10859 }
10860 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10861 return None;
10862 }
10863 let title = queue::title_from(raw, PR_TITLE_MAX);
10864 let lower = title.to_ascii_lowercase();
10865 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10866 return None;
10867 }
10868 Some(title)
10869}
10870
10871#[derive(Debug, Clone, PartialEq, Eq)]
10874struct BranchFacts {
10875 commits: Vec<(String, String)>,
10877 stat: String,
10879}
10880
10881const STAT_FILE_LINES: usize = 25;
10883const COMMITS_MAX_CHARS: usize = 20_000;
10885
10886async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10889 let commits = git::commit_log(repo, base, branch).await.ok()?;
10890 if commits.is_empty() {
10891 return None;
10892 }
10893 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10894 let lines: Vec<&str> = stat.lines().collect();
10895 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10896 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10897 let more = format!(" ... {omitted} more file(s)");
10898 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10899 kept.push(&more);
10900 kept.push(lines[lines.len() - 1]);
10901 kept.join("\n")
10902 } else {
10903 lines.join("\n")
10904 };
10905 Some(BranchFacts { commits, stat })
10906}
10907
10908fn markdown_safe(text: &str) -> String {
10911 text.replace("</details>", "</details>")
10912 .replace("\x60\x60\x60", "~~~")
10913}
10914
10915fn neutral_title(state: &RunState, winner: char) -> String {
10916 format!(
10917 "chore: land candidate {} of run {}",
10918 winner.to_ascii_uppercase(),
10919 state.id
10920 )
10921}
10922
10923pub(crate) async fn repair_landing_gate(
10926 state: &mut RunState,
10927 winner: &Candidate,
10928 outcomes: &[CommandOutcome],
10929) -> Result<GateFix> {
10930 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10931 return Ok(GateFix::Stop);
10932 }
10933 let mut runner = Runner {
10934 roles: state.config.resolve_roles()?,
10935 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10936 state: state.clone(),
10937 pause: Pause::new(),
10938 interrupt: Pause::new(),
10939 };
10940 let result = runner.gate_fix_round(winner, outcomes).await;
10941 *state = runner.state;
10942 result
10943}
10944
10945pub(crate) async fn check_gate_repair_e2e(
10948 state: &mut RunState,
10949 winner: &Candidate,
10950 wait: &crate::repo_lock::WaitPolicy,
10951) -> Result<GateFix> {
10952 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10953 return Ok(GateFix::Retry);
10954 }
10955 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10956 if state
10957 .gate_repair_e2e
10958 .as_ref()
10959 .is_some_and(|r| r.head == head && r.passed)
10960 {
10961 return Ok(GateFix::Retry);
10962 }
10963 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10964 head: head.clone(),
10965 passed: false,
10966 outcomes: Vec::new(),
10967 });
10968 state.save()?;
10969 let commands = state.config.verify.e2e.clone();
10970 let outcomes = if commands.is_empty() {
10971 Vec::new()
10972 } else {
10973 let shell = state.config.shell();
10974 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10975 let cache_dir = state.config.cache_dir();
10976 let (outcomes, _) = with_cache_lease(
10977 state,
10978 cache_dir.as_deref(),
10979 "e2e",
10980 "e2e",
10981 &winner.worktree,
10982 &head,
10983 timeout,
10984 "gate repair e2e",
10985 wait,
10986 |state, budget| async move {
10987 run_e2e_with_retry(
10988 state,
10989 &shell,
10990 &commands,
10991 &winner.worktree,
10992 budget,
10993 "gate repair e2e",
10994 )
10995 .await
10996 },
10997 )
10998 .await?;
10999 outcomes
11000 };
11001 let deferred = verify_inconclusive(&outcomes);
11002 let passed = outcomes.iter().all(CommandOutcome::ok)
11003 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11004 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11005 head,
11006 passed,
11007 outcomes,
11008 });
11009 state.save()?;
11010 if deferred {
11011 Ok(GateFix::Defer)
11012 } else if !passed {
11013 state.event(
11014 "gate",
11015 "verify.e2e failed after gate repair; nothing published",
11016 );
11017 Ok(GateFix::Stop)
11018 } else {
11019 Ok(GateFix::Retry)
11020 }
11021}
11022
11023pub(crate) async fn run_landing_gate(
11025 state: &mut RunState,
11026 winner: &Candidate,
11027 wait: &crate::repo_lock::WaitPolicy,
11028) -> Result<Vec<CommandOutcome>> {
11029 let shell = state.config.shell();
11030 let gate_commands = state.config.verify.gate.clone();
11031 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11032 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11033 Vec::new()
11034 } else {
11035 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11036 let cache_dir = state.config.cache_dir();
11037 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11038 let (outcomes, _) = with_cache_lease(
11039 state,
11040 cache_dir.as_deref(),
11041 "gate",
11042 "gate",
11043 &winner.worktree,
11044 &head,
11045 timeout,
11046 "final gate",
11047 wait,
11048 |state, budget| {
11049 let shell = shell.clone();
11050 let gate_commands = gate_commands.clone();
11051 let winner = winner.clone();
11052 async move {
11053 Runner::run_pre_gate(state, &winner).await;
11054 let (outcomes, timed_out_pids) = run_commands(
11055 state,
11056 "gate",
11057 "gate",
11058 0,
11059 &shell,
11060 &gate_commands,
11061 &winner.worktree,
11062 budget,
11063 )
11064 .await;
11065 (outcomes, false, timed_out_pids)
11066 }
11067 },
11068 )
11069 .await?;
11070 outcomes
11071 };
11072 if outcomes.is_empty() {
11073 state.event("gate", "no gate commands configured; not tested");
11078 }
11079 for o in &outcomes {
11080 state.event(
11081 "gate",
11082 format!(
11083 "`{}` -> {}",
11084 o.command,
11085 if o.ok() {
11086 "pass".to_owned()
11087 } else {
11088 format!(
11089 "FAIL ({:?})\n{}",
11090 o.code,
11091 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11092 )
11093 }
11094 ),
11095 );
11096 }
11097 Ok(outcomes)
11098}
11099
11100pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11106 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11107 ""
11108 } else {
11109 pr_title
11110 }
11111}
11112
11113pub fn landing_subject_source(state: &RunState) -> String {
11117 if is_review_run(state) {
11118 let winner = state.candidates.first().map_or('A', |c| c.label);
11119 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11120 }
11121 state.instruction.clone()
11122}
11123
11124fn summary_without_title(summary: &str) -> String {
11127 let mut lines = summary.trim().lines().peekable();
11128 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11129 lines.next();
11130 }
11131 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11132}
11133
11134#[cfg(test)]
11146fn pr_message(state: &RunState, winner: char) -> PrMessage {
11147 pr_message_with(state, winner, None)
11148}
11149
11150#[cfg(test)]
11153fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11154 let raw = pr_message_raw(
11155 state,
11156 winner,
11157 facts,
11158 crate::pr_outcome::FinalHead::Unreported,
11159 ChangeNote::NotAsked,
11160 );
11161 let id = crate::scrub::Identity::current();
11162 PrMessage {
11163 title: crate::scrub::scrub(&raw.title, &id),
11164 body: crate::scrub::scrub(&raw.body, &id),
11165 }
11166}
11167
11168#[derive(Debug, Clone, Copy)]
11170enum ChangeNote<'a> {
11171 NotAsked,
11173 Summary(&'a crate::run::ChangeSummary),
11175 Unavailable(&'a str),
11177}
11178
11179fn pr_message_raw(
11180 state: &RunState,
11181 winner: char,
11182 facts: Option<&BranchFacts>,
11183 head: crate::pr_outcome::FinalHead<'_>,
11184 change: ChangeNote<'_>,
11185) -> PrMessage {
11186 if !matches!(change, ChangeNote::NotAsked) {
11187 let (title, mut body) = match change {
11188 ChangeNote::Summary(c) => (
11189 c.title.clone(),
11190 format!("## Summary\n\n{}\n\n", c.body.trim()),
11191 ),
11192 ChangeNote::Unavailable(why) => (
11193 neutral_title(state, winner),
11194 format!(
11195 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11196 ),
11197 ),
11198 ChangeNote::NotAsked => unreachable!(),
11199 };
11200 body.push_str(&public_pr_evidence(state, head));
11201 body.push_str(&format!(
11202 "\n---\nmagi:run/{} magi:candidate-{}\n",
11203 state.id,
11204 winner.to_ascii_lowercase()
11205 ));
11206 return PrMessage { title, body };
11207 }
11208 pr_preview(state, winner, facts, head)
11209}
11210
11211fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11214 let mut body = crate::pr_outcome::review_rounds(state);
11215 body.push_str(&crate::pr_outcome::final_verification(state, head));
11216 let open = state.last_round_findings();
11217 if !open.is_empty() {
11218 body.push_str("## Open review findings\n\n");
11219 for f in open {
11220 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11221 }
11222 body.push('\n');
11223 }
11224 if let Some(c) = state.change_summary.as_ref()
11225 && head == crate::pr_outcome::FinalHead::At(&c.head)
11226 {
11227 body.push_str(&format!(
11228 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11229 c.base, c.head
11230 ));
11231 }
11232 body
11233}
11234
11235fn pr_preview(
11237 state: &RunState,
11238 winner: char,
11239 facts: Option<&BranchFacts>,
11240 head: crate::pr_outcome::FinalHead<'_>,
11241) -> PrMessage {
11242 let summary = state
11243 .candidates
11244 .iter()
11245 .find(|c| c.label == winner)
11246 .map(|c| c.summary.as_str())
11247 .unwrap_or_default();
11248 let review = is_review_run(state);
11251 let title = if review {
11252 facts
11253 .and_then(|f| {
11254 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11255 })
11256 .or_else(|| review_title(state))
11257 .or_else(|| {
11258 state
11259 .candidates
11260 .iter()
11261 .find(|c| c.label == winner)
11262 .filter(|c| !c.branch.starts_with("magi/"))
11263 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11264 })
11265 .unwrap_or_else(|| neutral_title(state, winner))
11266 } else {
11267 summary_title(summary).unwrap_or_else(|| {
11268 english_title(
11269 &state.instruction,
11270 PR_TITLE_MAX,
11271 state.config.graph.github_english(),
11272 )
11273 .unwrap_or_else(|| neutral_title(state, winner))
11274 })
11275 };
11276
11277 let mut body = String::new();
11278 let what = summary_without_title(summary);
11279 if !what.is_empty() {
11280 body.push_str("## Summary\n\n");
11281 body.push_str(&what);
11282 body.push_str("\n\n");
11283 }
11284
11285 body.push_str(&crate::pr_outcome::review_rounds(state));
11287 body.push_str(&crate::pr_outcome::final_verification(state, head));
11288 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11289
11290 let open = state.open_findings();
11291 if !open.is_empty() {
11292 body.push_str("## Open review findings\n\n");
11293 for f in &open {
11294 body.push_str(&format!(
11295 "- `{}` [{:?}] {}\n",
11296 f.id, f.severity, "Context retained in local review records."
11297 ));
11298 }
11299 body.push('\n');
11300 }
11301
11302 if let Some(fix) = fix
11303 && !fix.rejected.is_empty()
11304 {
11305 body.push_str("## Declined by the fixer\n\n");
11306 for r in &fix.rejected {
11307 body.push_str(&format!(
11308 "- `{}`: {}\n",
11309 r.id, "Reason retained in local review records."
11310 ));
11311 }
11312 body.push('\n');
11313 }
11314
11315 if review {
11316 body.push_str("## Commits under review\n\n");
11318 if let Some(facts) = facts {
11319 let mut left = COMMITS_MAX_CHARS;
11320 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11321 let mut entry = format!("- {}\n", markdown_safe(subject));
11322 for l in markdown_safe(text).lines() {
11323 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11324 }
11325 if left == 0 {
11326 body.push_str(&format!(
11327 "- ... {} more commit(s)\n",
11328 facts.commits.len() - i
11329 ));
11330 break;
11331 }
11332 if entry.len() > left {
11333 let mut end = left;
11336 while !entry.is_char_boundary(end) {
11337 end -= 1;
11338 }
11339 entry.truncate(end);
11340 entry.push_str("\n ... (truncated)\n");
11341 left = 0;
11342 } else {
11343 left -= entry.len();
11344 }
11345 body.push_str(&entry);
11346 }
11347 if !facts.stat.trim().is_empty() {
11348 body.push_str(&format!(
11349 "\n## Diff stat\n\n```\n{}\n```\n",
11350 markdown_safe(facts.stat.trim())
11351 ));
11352 }
11353 } else {
11354 match &state.reviewed_commits {
11355 Some(subjects) => {
11356 for s in subjects {
11357 body.push_str(&format!("- {}\n", s.trim()));
11358 }
11359 }
11360 None => {
11361 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11364 body.push_str(rest.trim());
11365 body.push('\n');
11366 }
11367 }
11368 }
11369 } else {
11370 let task = state.instruction.trim();
11371 let task = if task.is_empty() {
11372 "(empty task)"
11373 } else {
11374 task
11375 };
11376 body.push_str(&format!(
11377 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11378 task.replace("</details>", "</details>")
11379 ));
11380 }
11381
11382 body.push_str(&format!(
11383 "\n---\nmagi:run/{} magi:candidate-{}\n",
11384 state.id,
11385 winner.to_ascii_lowercase()
11386 ));
11387
11388 PrMessage { title, body }
11389}
11390
11391fn seeded_instruction(state: &RunState) -> String {
11394 let mut inputs = Vec::new();
11395 if let Some(start) = &state.sources.start_from {
11396 inputs.push(format!(
11397 "Start from `{}` at `{}` (including ancestry).",
11398 start.token, start.sha
11399 ));
11400 }
11401 for pick in &state.sources.pick_commits {
11402 inputs.push(format!(
11403 "Pick `{}` at `{}` (single commit).",
11404 pick.token, pick.sha
11405 ));
11406 }
11407 if inputs.is_empty() {
11408 state.instruction.clone()
11409 } else {
11410 format!(
11411 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11412 state.instruction,
11413 inputs.join("\n")
11414 )
11415 }
11416}
11417
11418async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11422 let base = &state.base_branch;
11423 let mut against = base.clone();
11424 if mode == MergeMode::Pr {
11425 let remote = &state.config.merge.remote;
11429 let tracking = format!("{remote}/{base}");
11430 let fetched = git::fetch(repo, remote, base).await;
11431 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11432 against = tracking;
11433 } else {
11434 return false;
11435 }
11436 }
11437 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11438}
11439
11440fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11443 let mut detail = format!(
11444 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11445 nothing to open a pull request for"
11446 );
11447 detail.push_str(
11448 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11449 );
11450 detail
11451}
11452
11453#[derive(Debug, PartialEq, Eq)]
11456enum PrPlan {
11457 Create,
11458 Adopt { url: String, title: String },
11459 Stop(String),
11460}
11461
11462fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11465 match found {
11466 Ok(land::OpenPr::None) => PrPlan::Create,
11467 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11468 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11469 "several open pull requests exist for this branch, not picking one: {}",
11470 urls.join(" ")
11471 )),
11472 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11473 }
11474}
11475
11476pub(crate) async fn gh_pr_create(
11478 cwd: &Path,
11479 base: &str,
11480 head: &str,
11481 title: &str,
11482 body: &str,
11483) -> Result<String> {
11484 let out = tokio::process::Command::new("gh")
11485 .args([
11486 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11487 ])
11488 .current_dir(cwd)
11489 .quiet()
11490 .stdin(std::process::Stdio::null())
11491 .output()
11492 .await
11493 .context("spawn gh")?;
11494 if out.status.success() {
11495 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11496 } else {
11497 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11498 }
11499}
11500
11501async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11503 match std::fs::symlink_metadata(path) {
11504 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11505 git::worktree_remove_clean(repo, path).await?;
11508 return Ok(());
11509 }
11510 Err(e) => return Err(e.into()),
11511 Ok(_) => {}
11512 }
11513 if !git::is_clean(path).await? {
11514 bail!(
11515 "worktree has uncommitted work; refusing removal: {}",
11516 path.display()
11517 );
11518 }
11519 if !git::worktree_remove_clean(repo, path).await? {
11520 bail!("worktree removal refused: {}", path.display());
11521 }
11522 Ok(())
11523}
11524
11525pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11534 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11535 bail!("refusing fold: {reason}");
11536 }
11537 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11538 bail!("candidate preservation incomplete; refusing cleanup");
11539 }
11540 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11544 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11545 let _driver = crate::driver::maintenance(home, &state.id)?;
11546 let mut output_guards = Vec::new();
11547 for entry in std::fs::read_dir(state.worktree_root())
11548 .into_iter()
11549 .flatten()
11550 .flatten()
11551 {
11552 if entry.path().join(".git").is_file() {
11553 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11554 }
11555 }
11556 let repo = state.repo.clone();
11557 let wait = repository_wait(state, None);
11560 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11561 Ok(lease) => lease,
11562 Err(e) => {
11563 state.save_under(home)?;
11564 return Err(e);
11565 }
11566 };
11567 let _source_outputs = if state.status == RunStatus::Merged
11570 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11571 {
11572 verify_merged_fold_sources(state, drop_winner, home).await?
11573 } else {
11574 Vec::new()
11575 };
11576 let root = state.worktree_root();
11577 let winner = state.tally.as_ref().map(|t| t.winner);
11578 let mut removed = Vec::new();
11579
11580 for i in 0..state.candidates.len() {
11581 let c = state.candidates[i].clone();
11582 let is_winner = Some(c.label) == winner;
11583 if is_winner && !drop_winner {
11584 continue;
11585 }
11586 let existed = c.worktree.exists();
11587 remove_preserved_worktree(&repo, &c.worktree).await?;
11590 if existed {
11591 removed.push(c.worktree.to_string_lossy().into_owned());
11592 }
11593 let handed_over = state.released_branches.contains(&c.branch);
11596 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11597 if !git::branch_delete(&repo, &c.branch).await? {
11598 bail!("candidate branch removal refused");
11599 }
11600 removed.push(c.branch.clone());
11601 }
11602 state.candidates[i].folded = true;
11603 }
11604
11605 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11606 let path = name.path();
11607 let keep = !drop_winner
11608 && winner.is_some_and(|w| {
11609 path.file_name()
11610 .is_some_and(|n| n == format!("cand-{w}").as_str())
11611 });
11612 if keep {
11613 continue;
11614 }
11615 remove_preserved_worktree(&repo, &path).await?;
11616 removed.push(path.to_string_lossy().into_owned());
11617 }
11618
11619 remove_if_empty(&root);
11628
11629 if state.enabled_worktree_config && drop_winner {
11630 git::release_worktree_config(&repo).await.ok();
11634 state.enabled_worktree_config = false;
11635 }
11636 state.save_under(home)?;
11637 Ok(removed)
11638}
11639
11640async fn verify_merged_fold_sources(
11641 state: &RunState,
11642 drop_winner: bool,
11643 home: &Path,
11644) -> Result<Vec<std::fs::File>> {
11645 let winner = state.tally.as_ref().map(|t| t.winner);
11646 let mut heads = BTreeSet::new();
11647 for candidate in &state.candidates {
11648 if Some(candidate.label) == winner && !drop_winner {
11649 continue;
11650 }
11651 anyhow::ensure!(
11652 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11653 "candidate recovery source is outside the inventoried run root: {}",
11654 candidate.worktree.display()
11655 );
11656 match std::fs::symlink_metadata(&candidate.worktree) {
11657 Ok(_) => {}
11658 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11659 Err(error) => return Err(error.into()),
11660 }
11661 if !state.released_branches.contains(&candidate.branch)
11662 && git::branch_exists(&state.repo, &candidate.branch).await?
11663 {
11664 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11665 }
11666 }
11667 let mut cargo_guards = Vec::new();
11668 let entries = match std::fs::read_dir(state.worktree_root()) {
11669 Ok(entries) => Some(entries),
11670 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11671 Err(error) => return Err(error.into()),
11672 };
11673 for entry in entries.into_iter().flatten() {
11674 let path = entry?.path();
11675 let metadata = std::fs::symlink_metadata(&path)?;
11676 anyhow::ensure!(
11677 metadata.is_dir() && !metadata.file_type().is_symlink(),
11678 "non-directory or symlink recovery source retained: {}",
11679 path.display()
11680 );
11681 if !drop_winner
11682 && winner.is_some_and(|w| {
11683 path.file_name()
11684 .is_some_and(|name| name == format!("cand-{w}").as_str())
11685 })
11686 {
11687 continue;
11688 }
11689 anyhow::ensure!(
11690 path.join(".git").is_file(),
11691 "unregistered recovery source retained: {}",
11692 path.display()
11693 );
11694 anyhow::ensure!(
11695 git::git(
11696 &path,
11697 &[
11698 "status",
11699 "--porcelain",
11700 "--untracked-files=all",
11701 "--ignored",
11702 "--",
11703 ".",
11704 ":(exclude)target"
11705 ]
11706 )
11707 .await?
11708 .is_empty(),
11709 "source changes or ignored recovery files retained: {}",
11710 path.display()
11711 );
11712 match std::fs::symlink_metadata(path.join("target")) {
11713 Ok(_) => {
11714 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11715 }
11716 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11717 Err(error) => return Err(error.into()),
11718 }
11719 for checkpoint in [
11720 "rebase-merge",
11721 "rebase-apply",
11722 "sequencer",
11723 "CHERRY_PICK_HEAD",
11724 "MERGE_HEAD",
11725 "REBASE_HEAD",
11726 "REVERT_HEAD",
11727 "BISECT_LOG",
11728 "AUTO_MERGE",
11729 "MERGE_MSG",
11730 ] {
11731 let location = git::git(
11732 &path,
11733 &[
11734 "rev-parse",
11735 "--path-format=absolute",
11736 "--git-path",
11737 checkpoint,
11738 ],
11739 )
11740 .await?;
11741 anyhow::ensure!(
11742 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11743 if e.kind() == std::io::ErrorKind::NotFound),
11744 "Git recovery checkpoint retained: {checkpoint}"
11745 );
11746 }
11747 heads.insert(git::rev_parse(&path, "HEAD").await?);
11748 }
11749 crate::fold_evidence::preserve(state, home, &heads).await?;
11750
11751 Ok(cargo_guards)
11752}
11753
11754fn remove_if_empty(dir: &Path) {
11765 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11766 std::fs::remove_dir(dir).ok();
11767 }
11768}
11769
11770pub fn worst_open(state: &RunState) -> Option<Severity> {
11772 state
11773 .reviews
11774 .last()?
11775 .reviews
11776 .iter()
11777 .flat_map(|r| r.findings.iter())
11778 .map(|f| f.severity)
11779 .max()
11780}
11781
11782async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11786 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11787 git::acquire_worktree_config(repo).await?;
11788 state.enabled_worktree_config = true;
11789 }
11790 Ok(())
11791}
11792
11793#[cfg(test)]
11794mod tests {
11795 #[tokio::test]
11796 async fn launcher_base_fetch_requires_remote_success() {
11797 let tmp = tempfile::tempdir().unwrap();
11798 let repo = tmp.path().join("repo");
11799 let remote = tmp.path().join("remote");
11800 std::fs::create_dir(&repo).unwrap();
11801 std::fs::create_dir(&remote).unwrap();
11802 git::git(&remote, &["init", "--bare", "-b", "main"])
11803 .await
11804 .unwrap();
11805 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11806 git::git(
11807 &repo,
11808 &[
11809 "-c",
11810 "user.name=Fixture",
11811 "-c",
11812 "user.email=fixture@example.invalid",
11813 "commit",
11814 "--allow-empty",
11815 "-m",
11816 "fixture",
11817 ],
11818 )
11819 .await
11820 .unwrap();
11821 git::git(
11822 &repo,
11823 &["remote", "add", "origin", remote.to_str().unwrap()],
11824 )
11825 .await
11826 .unwrap();
11827 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11828 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11829 assert_eq!(
11830 resolve_base(&repo, "main", "origin", None)
11831 .await
11832 .unwrap()
11833 .commit,
11834 head
11835 );
11836 #[cfg(unix)]
11837 {
11838 let policy: crate::config::Coordination = toml::from_str(
11839 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11840 ).unwrap();
11841 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11842 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11843 let held = crate::repo_lock::SeatLease::operation_with(
11844 &repo,
11845 Some(&policy),
11846 &crate::repo_lock::WaitPolicy::default(),
11847 )
11848 .await
11849 .unwrap();
11850 assert!(held.is_some());
11851 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11852 tokio::pin!(fetch);
11853 assert!(
11854 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11855 .await
11856 .is_err()
11857 );
11858 assert!(
11859 !repo.join(".git/FETCH_HEAD").exists(),
11860 "no fetch before admission"
11861 );
11862 drop(held);
11863 let fetched = fetch.await.unwrap();
11864 assert_eq!(fetched.commit, head);
11865 assert_eq!(fetched.remote, "origin");
11866 assert_eq!(fetched.branch, "main");
11867 }
11868 git::git(
11870 &repo,
11871 &[
11872 "remote",
11873 "set-url",
11874 "origin",
11875 tmp.path().join("missing").to_str().unwrap(),
11876 ],
11877 )
11878 .await
11879 .unwrap();
11880 let error = resolve_base(&repo, "main", "origin", None)
11881 .await
11882 .unwrap_err()
11883 .to_string();
11884 assert!(error.contains("cannot read origin/main"));
11885 assert!(error.contains("refusing to branch off"));
11886 }
11887
11888 #[test]
11889 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11890 let leaked = vec!["chore(deps): update a crate".to_owned()];
11891 let own = "fix(daemon): apply a chosen action";
11892 assert!(should_retitle("", own, &leaked));
11893 assert!(should_retitle(
11894 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11895 own,
11896 &leaked
11897 ));
11898 assert!(should_retitle(
11899 "chore: land candidate A of run 1",
11900 own,
11901 &leaked
11902 ));
11903 assert!(should_retitle(
11904 "magi: candidate A (uncommitted work)",
11905 own,
11906 &leaked
11907 ));
11908 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11909 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11910 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11911 }
11912
11913 #[test]
11914 fn pr_merge_plan_creates_adopts_or_stops() {
11915 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11916 assert_eq!(
11917 pr_merge_plan(Ok(land::OpenPr::One {
11918 url: "u".into(),
11919 title: "t".into()
11920 })),
11921 PrPlan::Adopt {
11922 url: "u".into(),
11923 title: "t".into()
11924 }
11925 );
11926 let PrPlan::Stop(many) =
11927 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11928 else {
11929 panic!("many must stop");
11930 };
11931 assert!(many.contains('a') && many.contains('b'));
11932 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11933 panic!("a failed lookup must stop");
11934 };
11935 assert!(err.contains("bad token"));
11936 }
11937
11938 use super::*;
11939 use crate::run::GateStatus;
11940 use std::collections::BTreeMap;
11941 use std::time::Duration;
11942
11943 fn conductor() -> AgentSpec {
11944 AgentSpec {
11945 id: "conductor".to_owned(),
11946 kind: crate::config::AgentKind::Command,
11947 model: None,
11948 command: vec!["true".to_owned()],
11949 extra_args: Vec::new(),
11950 env: BTreeMap::new(),
11951 prompt_delivery: None,
11952 max_turn_secs: None,
11953 }
11954 }
11955
11956 fn spec(id: &str) -> AgentSpec {
11957 AgentSpec {
11958 id: id.to_owned(),
11959 kind: crate::config::AgentKind::Command,
11960 model: None,
11961 command: vec!["true".to_owned()],
11962 extra_args: Vec::new(),
11963 env: BTreeMap::new(),
11964 prompt_delivery: None,
11965 max_turn_secs: None,
11966 }
11967 }
11968
11969 fn ids(xs: &[&str]) -> BTreeSet<String> {
11970 xs.iter().map(|s| (*s).to_owned()).collect()
11971 }
11972
11973 #[test]
11974 fn pick_successor_skips_an_agent_another_seat_holds() {
11975 let roster = [spec("a"), spec("b"), spec("c")];
11977 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11978 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11979 }
11980
11981 #[test]
11982 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11983 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11985 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11986 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11987 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11988 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11989 }
11990
11991 #[test]
11992 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11993 let roster = [spec("a"), spec("b")];
11994 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11995 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11996 let carried = ids(&["b"]);
11997 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11998 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11999 }
12000
12001 #[test]
12002 fn pick_successor_returns_none_without_any_untried_successor() {
12003 let roster = [spec("a"), spec("b")];
12004 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12005 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12006 let carried = ids(&["a"]);
12007 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12008 }
12009
12010 #[test]
12011 fn pick_successor_rescue_avoids_another_seats_occupant() {
12012 let roster = [spec("a"), spec("b"), spec("c")];
12014 let carried = ids(&["b", "c"]);
12015 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12016 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12017 }
12018
12019 #[test]
12020 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12021 let roster = [spec("a"), spec("b"), spec("c")];
12022 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12023 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12024 }
12025
12026 #[test]
12027 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12028 let roster = [spec("a"), spec("b"), spec("c")];
12029 let failed = ids(&["a", "b", "c"]);
12030 let mut tried = ids(&["b"]);
12032 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12033 assert_eq!(first.id, "a");
12034 tried.insert(first.id.clone());
12035 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12036 assert_eq!(second.id, "c");
12037 tried.insert(second.id.clone());
12038 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12039 }
12040
12041 #[test]
12042 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12043 let roster = [spec("a"), spec("b")];
12044 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12045 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12046 }
12047
12048 #[test]
12049 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12050 let roster = [spec("a"), spec("b"), spec("c")];
12051 let h = SeatHistory {
12052 failed: ids(&["a"]),
12053 last_ok: Some("b".to_owned()),
12054 last_fail: None,
12055 };
12056 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12057 let gone = SeatHistory {
12059 last_ok: Some("zzz".to_owned()),
12060 ..h.clone()
12061 };
12062 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12063 let failed = SeatHistory {
12064 failed: ids(&["a", "b"]),
12065 last_ok: Some("b".to_owned()),
12066 last_fail: None,
12067 };
12068 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12069 }
12070
12071 #[test]
12072 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12073 let first = SeatState::new("review-1", "alpha", 7);
12074 let next = handover_seat("review-1", "gamma", 7);
12075 assert_ne!(first.claude_session, next.claude_session);
12076 }
12077
12078 #[test]
12079 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12080 let mut state = state_with_summary("x", "y");
12081 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12082 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12083 assert_ne!(a.claude_session, b.claude_session);
12084 }
12085
12086 #[test]
12087 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12088 let roster = [spec("a"), spec("b")];
12089 let h = SeatHistory {
12090 failed: ids(&["a", "b"]),
12091 ..SeatHistory::default()
12092 };
12093 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12094 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12095 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12096 }
12097
12098 #[test]
12105 fn failure_signature_ignores_numbers_and_paths() {
12106 assert_eq!(
12107 failure_signature("exited with Some(2) and no usable output"),
12108 failure_signature("exited with Some(137) and no usable output")
12109 );
12110 assert_eq!(
12111 failure_signature("process exited with no usable output (exit code 2)"),
12112 failure_signature("process exited with no usable output (exit code 137)")
12113 );
12114 for (old, new) in [
12115 (
12116 "exited with Some(2) and no usable output",
12117 "process exited with no usable output (exit code 2)",
12118 ),
12119 (
12120 "exited with None and no usable output",
12121 "process exited with no usable output (exit code unavailable)",
12122 ),
12123 ] {
12124 let previous = FailClass::Other(failure_signature(old));
12125 let current = FailClass::Other(failure_signature(new));
12126 assert_eq!(previous, current, "resumption retains the failure class");
12127 assert!(!should_hand_over(Some(&previous), ¤t));
12128 }
12129 assert_eq!(
12130 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12131 failure_signature("cannot open /var/x.txt: denied")
12132 );
12133 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12134 }
12135
12136 #[test]
12137 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12138 let other = FailClass::Other("x".into());
12139 assert!(should_hand_over(None, &FailClass::Quota));
12140 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12141 assert!(should_hand_over(
12142 Some(&FailClass::Timeout),
12143 &FailClass::Timeout
12144 ));
12145 assert!(should_hand_over(None, &other));
12146 assert!(!should_hand_over(Some(&other), &other));
12147 assert!(should_hand_over(
12148 Some(&other),
12149 &FailClass::Other("y".into())
12150 ));
12151 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12153 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12154 }
12155
12156 #[test]
12157 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12158 let a = SeatState::new("judge-1", "alpha", 7);
12159 let b = handover_seat("judge-1", "beta", 7);
12160 assert_ne!(a.claude_session, b.claude_session);
12161 assert_eq!(b.turns, 0);
12162 }
12163
12164 #[test]
12165 fn a_timeout_is_classified_apart_from_other_failures() {
12166 assert_eq!(
12167 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12168 Some(FailClass::Timeout)
12169 );
12170 assert!(matches!(
12171 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12172 Some(FailClass::Other(_))
12173 ));
12174 }
12175
12176 #[test]
12177 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12178 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12179 let tried = BTreeSet::from(["beta".to_owned()]);
12180 let next = next_untried_in_roster(&roster, 1, &tried);
12183 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12184 }
12185
12186 #[test]
12187 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12188 let roster = vec![spec("alpha"), spec("beta")];
12189 let tried = BTreeSet::from(["beta".to_owned()]);
12190 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12194 }
12195
12196 #[test]
12197 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12198 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12199 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12200 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12204 }
12205
12206 #[test]
12207 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12208 let roster = vec![spec("a"), spec("a"), spec("b")];
12209 let tried = BTreeSet::from(["a".to_owned()]);
12210 let next = next_untried_in_roster(&roster, 0, &tried);
12211 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12212 }
12213
12214 #[test]
12215 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12216 let roster = vec![spec("a"), spec("b")];
12217 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12218 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12219 }
12220
12221 #[test]
12222 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12223 let dir = tempfile::tempdir().unwrap();
12224 let bay = dir.path().join("ffff");
12225
12226 remove_if_empty(&bay);
12228 assert!(!bay.exists());
12229
12230 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12233 remove_if_empty(&bay);
12234 assert!(bay.exists(), "non-empty directory must survive");
12235
12236 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12238 remove_if_empty(&bay);
12239 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12240 }
12241
12242 #[test]
12251 fn a_full_panel_that_found_nothing_is_clean() {
12252 assert!(round_is_clean(
12253 0,
12254 true,
12255 2,
12256 2,
12257 0,
12258 IncompleteReviewPolicy::Block
12259 ));
12260 }
12261
12262 #[test]
12263 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12264 assert!(!round_is_clean(
12265 0,
12266 true,
12267 1,
12268 2,
12269 0,
12270 IncompleteReviewPolicy::Block
12271 ));
12272 }
12273
12274 #[test]
12275 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12276 assert!(!round_is_clean(
12277 1,
12278 true,
12279 1,
12280 2,
12281 0,
12282 IncompleteReviewPolicy::Warn
12283 ));
12284 }
12285
12286 #[test]
12287 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12288 assert!(round_is_clean(
12289 0,
12290 true,
12291 1,
12292 2,
12293 0,
12294 IncompleteReviewPolicy::Warn
12295 ));
12296 }
12297
12298 #[test]
12299 fn a_full_panel_with_an_open_finding_is_not_clean() {
12300 assert!(!round_is_clean(
12301 1,
12302 true,
12303 2,
12304 2,
12305 0,
12306 IncompleteReviewPolicy::Block
12307 ));
12308 }
12309
12310 #[test]
12311 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12312 assert!(!round_is_clean(
12313 0,
12314 false,
12315 2,
12316 2,
12317 0,
12318 IncompleteReviewPolicy::Block
12319 ));
12320 }
12321
12322 #[test]
12329 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12330 assert!(round_is_clean(
12333 0,
12334 true,
12335 1,
12336 2,
12337 1,
12338 IncompleteReviewPolicy::Block
12339 ));
12340 }
12341
12342 #[test]
12343 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12344 assert!(!round_is_clean(
12347 0,
12348 true,
12349 1,
12350 2,
12351 0,
12352 IncompleteReviewPolicy::Block
12353 ));
12354 }
12355
12356 #[test]
12357 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12358 assert!(!round_is_clean(
12359 1,
12360 true,
12361 1,
12362 2,
12363 1,
12364 IncompleteReviewPolicy::Block
12365 ));
12366 assert!(!round_is_clean(
12367 0,
12368 false,
12369 1,
12370 2,
12371 1,
12372 IncompleteReviewPolicy::Block
12373 ));
12374 }
12375
12376 #[test]
12377 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12378 assert!(!round_is_clean(
12382 0,
12383 true,
12384 0,
12385 2,
12386 2,
12387 IncompleteReviewPolicy::Block
12388 ));
12389 }
12390
12391 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12392 CommandOutcome {
12393 command: "test".to_owned(),
12394 code,
12395 output_tail: String::new(),
12396 duration_ms: 0,
12397 resource_blocked,
12398 }
12399 }
12400
12401 #[test]
12402 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12403 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12404 assert!(
12405 !verify_inconclusive(&[outcome(Some(1), false)]),
12406 "an ordinary failure is still evidence about the patch"
12407 );
12408 assert!(verify_inconclusive(&[outcome(None, true)]));
12409 assert!(
12410 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12411 "one inconclusive outcome taints the whole batch"
12412 );
12413 assert!(!verify_inconclusive(&[]));
12414 }
12415
12416 #[tokio::test]
12417 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12418 let calls = std::sync::atomic::AtomicUsize::new(0);
12422 let started = Instant::now();
12423 wait_for_pids_with(
12424 &[123],
12425 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12426 Duration::from_millis(5),
12427 Duration::from_secs(5),
12428 )
12429 .await;
12430 assert!(
12431 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12432 "must keep checking rather than deciding on the first answer"
12433 );
12434 assert!(
12435 started.elapsed() < Duration::from_secs(1),
12436 "must return the moment it is confirmed dead, not wait out the ceiling"
12437 );
12438 }
12439
12440 #[tokio::test]
12441 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12442 let started = Instant::now();
12443 wait_for_pids_with(
12444 &[123],
12445 |_| true, Duration::from_millis(5),
12447 Duration::from_millis(30),
12448 )
12449 .await;
12450 let elapsed = started.elapsed();
12451 assert!(
12452 elapsed >= Duration::from_millis(30),
12453 "must not give up before its own ceiling: {elapsed:?}"
12454 );
12455 assert!(
12456 elapsed < Duration::from_secs(1),
12457 "must not wait past its own ceiling either: {elapsed:?}"
12458 );
12459 }
12460
12461 #[tokio::test]
12462 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12463 let started = Instant::now();
12464 wait_for_pids_with(
12465 &[],
12466 |_| true,
12467 Duration::from_secs(5),
12468 Duration::from_secs(5),
12469 )
12470 .await;
12471 assert!(
12472 started.elapsed() < Duration::from_millis(200),
12473 "an empty pid list has nothing to confirm"
12474 );
12475 }
12476
12477 fn review_round(
12483 clean: bool,
12484 blocking: usize,
12485 answered: usize,
12486 expected: usize,
12487 progressed: bool,
12488 e2e_ok: bool,
12489 ) -> ReviewRound {
12490 ReviewRound {
12491 round: 1,
12492 head: "h".to_owned(),
12493 verified_head: None,
12494 verified_at: None,
12495 reviews: Vec::new(),
12496 e2e: vec![CommandOutcome {
12497 command: "test".to_owned(),
12498 code: Some(if e2e_ok { 0 } else { 1 }),
12499 output_tail: String::new(),
12500 duration_ms: 0,
12501 resource_blocked: false,
12502 }],
12503 verify_retried: false,
12504 e2e_deferred: false,
12505 e2e_defer_reason: None,
12506 fix: None,
12507 blocking,
12508 answered,
12509 expected,
12510 clean,
12511 progressed,
12512 vote_split: false,
12513 reconsideration: Vec::new(),
12514 verdict: None,
12515 }
12516 }
12517
12518 #[test]
12519 fn review_conclusion_is_none_when_nothing_has_run() {
12520 assert_eq!(review_conclusion(&[], 3), None);
12521 }
12522
12523 #[test]
12524 fn review_conclusion_is_none_while_rounds_remain() {
12525 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12526 assert_eq!(review_conclusion(&rounds, 3), None);
12527 }
12528
12529 #[test]
12530 fn review_conclusion_is_gating_once_a_round_is_clean() {
12531 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12532 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12533 }
12534
12535 #[test]
12536 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12537 let rounds = vec![
12538 review_round(false, 1, 2, 2, true, true),
12539 review_round(false, 1, 2, 2, true, true),
12540 ];
12541 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12542 }
12543
12544 #[test]
12545 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12546 let rounds = vec![
12547 review_round(false, 1, 2, 2, true, true),
12548 review_round(false, 1, 2, 2, true, false),
12549 ];
12550 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12551 }
12552
12553 #[test]
12554 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12555 let mut blocked = review_round(false, 1, 2, 2, true, false);
12562 blocked.e2e[0].resource_blocked = true;
12563 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12564 assert_eq!(review_conclusion(&rounds, 2), None);
12565 }
12566
12567 #[test]
12568 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12569 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12571 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12572 }
12573
12574 #[test]
12575 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12576 let rounds = vec![
12577 review_round(false, 1, 2, 2, false, true),
12578 review_round(false, 1, 2, 2, false, true),
12579 ];
12580 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12581 }
12582
12583 fn secs(n: u64) -> Duration {
12584 Duration::from_secs(n)
12585 }
12586
12587 #[tokio::test]
12590 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12591 ask_test_home();
12592 let tmp = tempfile::tempdir().unwrap();
12593 let repo = tmp.path().join("repo");
12594 std::fs::create_dir_all(&repo).unwrap();
12595 init_repo(&repo);
12596 let g = |args: &[&str]| {
12597 let out = std::process::Command::new("git")
12598 .args(args)
12599 .current_dir(&repo)
12600 .quiet()
12601 .output()
12602 .unwrap();
12603 assert!(
12604 out.status.success(),
12605 "{}",
12606 String::from_utf8_lossy(&out.stderr)
12607 );
12608 };
12609 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12610 g(&["checkout", "-b", "historical"]);
12611 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12612 g(&["add", "."]);
12613 g(&["commit", "-m", "old alternative"]);
12614 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12615 g(&["checkout", "-b", "magi/task-selected", &base]);
12616 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12617 g(&["add", "."]);
12618 g(&["commit", "-m", "measurement infrastructure"]);
12619 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12620 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12621 g(&["add", "."]);
12622 g(&["commit", "-m", "dependent implementation"]);
12623 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12624 g(&["checkout", "main"]);
12625 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12626 g(&["add", "."]);
12627 g(&["commit", "-m", "unrelated base change"]);
12628 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12629 let origin = tmp.path().join("origin.git");
12630 g(&[
12631 "clone",
12632 "--bare",
12633 repo.to_str().unwrap(),
12634 origin.to_str().unwrap(),
12635 ]);
12636 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12637 g(&["fetch", "origin"]);
12638 let selection = crate::pr_recovery::Selection {
12639 url: "https://example.test/project/pull/1".into(),
12640 branch: "magi/task-selected".into(),
12641 head: head.clone(),
12642 source_head: None,
12643 source_run: None,
12644 source_proof: None,
12645 disposition: Some(crate::pr_recovery::Disposition::Review),
12646 instruction: "preserve measured behavior".into(),
12647 approved_at: jiff::Timestamp::now(),
12648 };
12649 let mut template = runner_at(RunStatus::Reviewing);
12650 template.state.repo = repo.clone();
12651 template.state.base_commit = current.clone();
12652 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12653 template.state.origin = Some(crate::run::Origin::queue("selected"));
12654 template.state.pr_head = Some(selection.branch.clone());
12655 template.state.pr_recovery = Some(selection.clone());
12656 let mut runner =
12657 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12658 .await
12659 .unwrap();
12660 let wt = runner.state.winner().unwrap().worktree.clone();
12661 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12662 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12663 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12664 assert!(runner.state.seeds.is_empty());
12665 assert_eq!(runner.state.instruction, "preserve measured behavior");
12666 runner.sync_to_base().await.unwrap();
12667 assert_eq!(
12668 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12669 "prerequisite"
12670 );
12671 assert_eq!(
12672 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12673 "measured implementation"
12674 );
12675 assert_eq!(
12676 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12677 "current main change"
12678 );
12679 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12680 assert_eq!(
12681 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12682 head
12683 );
12684
12685 let mut partial = runner_at(RunStatus::Reviewing);
12687 partial.state.repo = repo.clone();
12688 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12689 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12690 run: Some("exit 1".into()),
12691 ..Default::default()
12692 }];
12693 let selection = runner.state.pr_recovery.clone().unwrap();
12694 partial.state.pr_recovery = Some(selection.clone());
12695 partial.state.pr_head = Some(selection.branch.clone());
12696 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12697 let id = partial.state.id.clone();
12698 let error = match Runner::open_selected(
12699 &repo,
12700 partial.state,
12701 partial.roles,
12702 1,
12703 runner.state.base_commit.clone(),
12704 selection,
12705 )
12706 .await
12707 {
12708 Ok(_) => panic!("setup must fail"),
12709 Err(e) => e,
12710 };
12711 assert_eq!(
12712 error
12713 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12714 .unwrap()
12715 .run,
12716 id
12717 );
12718 let failed = RunState::load(&id).unwrap();
12719 assert_eq!(failed.status, RunStatus::Blocked);
12720 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12721 assert!(Runner::resume(&id).is_err());
12722 runner.state.status = RunStatus::Ready;
12723 runner.state.driver_exited = true;
12724 runner.state.save().unwrap();
12725 let mut retry = runner_at(RunStatus::Reviewing);
12726 retry.state.repo = repo.clone();
12727 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12728 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12729 reselected.approved_at = jiff::Timestamp::now();
12730 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12731 retry.state.pr_head = Some(reselected.branch.clone());
12732 retry.state.pr_recovery = Some(reselected.clone());
12733 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12734 .await
12735 .unwrap();
12736 let restarted = Runner::open_selected(
12737 &repo,
12738 retry.state,
12739 retry.roles,
12740 1,
12741 runner.state.base_commit.clone(),
12742 reselected,
12743 )
12744 .await
12745 .unwrap();
12746 assert_eq!(
12747 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12748 .await
12749 .unwrap(),
12750 head
12751 );
12752 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12753 assert_eq!(
12754 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12755 head
12756 );
12757 }
12758
12759 #[tokio::test]
12760 #[cfg(unix)]
12761 async fn fallback_and_interrupted_restoration_require_coordination() {
12762 use std::os::fd::AsRawFd as _;
12763 ask_test_home();
12764 let tmp = tempfile::tempdir().unwrap();
12765 let mut runner = registered_runner(tmp.path()).await;
12766 runner.state.config.graph.timeout_resource_wait = 1;
12767 let winner = runner.state.candidates[0].clone();
12768 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12769 let leftover = winner.worktree.join("leftover");
12770 std::fs::write(&leftover, "untrusted edits").unwrap();
12771 let next = tmp.path().join("next-fixer");
12772 let mut failed = spec("first");
12773 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12774 let mut successor = spec("second");
12775 successor.command = vec![
12776 "sh".into(),
12777 "-c".into(),
12778 format!("touch '{}'; echo done", next.display()),
12779 ];
12780 let attempts = vec![
12781 (failed.clone(), "first".into()),
12782 (successor.clone(), "second".into()),
12783 ];
12784 let lock_path = runner.state.repo.join(".git/updater.lock");
12785 let external = std::fs::File::create(&lock_path).unwrap();
12786 assert_eq!(
12787 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12788 0
12789 );
12790 let artifacts = tmp.path().join("artifacts");
12791 let build = |spec, seat| SeatJob {
12792 spec,
12793 seat,
12794 cwd: winner.worktree.clone(),
12795 prompt: "fixture".into(),
12796 timeout: Duration::from_secs(10),
12797 allow_write: true,
12798 sessions: false,
12799 artifacts: artifacts.clone(),
12800 stem: "fix".into(),
12801 handover: None,
12802 };
12803 let (_, _, out) = runner
12804 .ask_fixer_among(
12805 attempts.clone(),
12806 "graft",
12807 None,
12808 Some((&winner.worktree, &head)),
12809 build,
12810 )
12811 .await;
12812 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12813 assert!(leftover.exists(), "contention must prevent clean");
12814 assert!(!next.exists());
12815 assert!(runner.state.handovers.is_empty());
12816 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12818 from_run: "earlier".into(),
12819 generation: 1,
12820 mode: crate::recovery::Mode::VerifyRepair,
12821 reasons: Vec::new(),
12822 reviewed_head: None,
12823 rounds: Vec::new(),
12824 open_findings: Vec::new(),
12825 evidence: None,
12826 criteria_inherited: true,
12827 repair: crate::recovery::Repair::Started {
12828 before: head.clone(),
12829 },
12830 });
12831 assert!(
12832 runner
12833 .recovery_repair(&winner)
12834 .await
12835 .unwrap_err()
12836 .is::<crate::repo_lock::ResourceBlocked>()
12837 );
12838 assert!(matches!(
12839 runner.state.recovery.as_ref().unwrap().repair,
12840 crate::recovery::Repair::Started { .. }
12841 ));
12842 assert!(leftover.exists());
12843 drop(external);
12844 runner.recovery_repair(&winner).await.unwrap();
12845 assert!(!leftover.exists());
12846 assert!(matches!(
12847 runner.state.recovery.as_ref().unwrap().repair,
12848 crate::recovery::Repair::Done { .. }
12849 ));
12850 std::fs::remove_file(&lock_path).unwrap();
12852 std::fs::create_dir(&lock_path).unwrap();
12853 std::fs::write(&leftover, "untrusted edits").unwrap();
12854 let (_, _, out) = runner
12855 .ask_fixer_among(
12856 attempts,
12857 "graft",
12858 None,
12859 Some((&winner.worktree, &head)),
12860 build,
12861 )
12862 .await;
12863 assert!(matches!(out, AgentOutcome::Failed(_)));
12864 assert!(leftover.exists());
12865 assert!(!next.exists());
12866 assert_eq!(
12867 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12868 head
12869 );
12870 }
12871
12872 #[cfg(unix)]
12873 async fn registered_runner(root: &Path) -> Runner {
12874 ask_test_home();
12875 let repo = root.join("repo");
12876 std::fs::create_dir_all(&repo).unwrap();
12877 init_repo(&repo);
12878 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12879 let mut runner = runner_at(RunStatus::Prep);
12880 runner.state.repo = repo.clone();
12881 runner.state.config.coordination = crate::config::test_coordination();
12882 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12883 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12884 runner.state.config.graph.advise = false;
12885 runner.roles.implementers = vec![spec("implement")];
12886 runner.roles.judges = vec![spec("judge")];
12887 runner.prep().await.unwrap();
12888 runner
12889 }
12890
12891 #[tokio::test]
12892 #[cfg(unix)]
12893 async fn parked_implementer_persists_completed_work() {
12894 for registered in [false, true] {
12895 let tmp = tempfile::tempdir().unwrap();
12896 let mut runner = registered_runner(tmp.path()).await;
12897 if !registered {
12898 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12899 }
12900 let marker = tmp.path().join("started");
12901 let release = tmp.path().join("release");
12902 let mut implementer = spec("implement");
12903 implementer.command = vec![
12904 "sh".into(),
12905 "-c".into(),
12906 format!(
12907 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12908 marker.display(),
12909 release.display()
12910 ),
12911 ];
12912 runner.roles.implementers = vec![implementer.clone()];
12913 runner.state.config.agents = vec![implementer];
12914 let pause = runner.pause.clone();
12915 let (result, ()) = tokio::join!(runner.implement(), async {
12916 tokio::time::timeout(Duration::from_secs(10), async {
12917 while !marker.exists() {
12918 tokio::time::sleep(Duration::from_millis(10)).await;
12919 }
12920 })
12921 .await
12922 .unwrap();
12923 pause.park();
12924 std::fs::write(&release, "go").unwrap();
12925 });
12926 result.unwrap();
12927 let candidate = &runner.state.candidates[0];
12928 assert!(candidate.commits > 0, "{candidate:?}");
12929 assert!(candidate.summary.contains("saved-result"));
12930 assert_eq!(candidate.files, 1);
12931 let before = candidate.commits;
12932 runner.implement().await.unwrap();
12933 assert_eq!(runner.state.candidates[0].commits, before);
12934 }
12935 }
12936
12937 #[tokio::test]
12938 #[cfg(unix)]
12939 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12940 let tmp = tempfile::tempdir().unwrap();
12941 let mut runner = registered_runner(tmp.path()).await;
12942 let winner = runner.state.candidates[0].clone();
12943 let cache = tmp.path().join("cache");
12944 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12945 runner.state.config.graph.timeout_resource_wait = 0;
12946 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12947 let crate::cache::AcquireOutcome::Acquired(held) =
12948 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12949 else {
12950 panic!("cache must be free")
12951 };
12952 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12953 from_run: "earlier".into(),
12954 generation: 1,
12955 mode: crate::recovery::Mode::VerifyRepair,
12956 reasons: Vec::new(),
12957 reviewed_head: None,
12958 rounds: Vec::new(),
12959 open_findings: Vec::new(),
12960 evidence: None,
12961 criteria_inherited: true,
12962 repair: crate::recovery::Repair::NotStarted,
12963 });
12964 runner.recovery_repair(&winner).await.unwrap();
12965 assert!(matches!(
12966 runner.state.recovery.as_ref().unwrap().repair,
12967 crate::recovery::Repair::WaitingForRestore { .. }
12968 ));
12969 drop(held);
12970 runner.state.resource_block = None;
12971 runner.state.config.graph.timeout_resource_wait = 1;
12972 runner.recovery_repair(&winner).await.unwrap();
12973 assert!(matches!(
12974 runner.state.recovery.as_ref().unwrap().repair,
12975 crate::recovery::Repair::Done { .. }
12976 ));
12977 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12978 }
12979
12980 #[tokio::test]
12981 #[cfg(unix)]
12982 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12983 use std::os::fd::AsRawFd as _;
12984 for graft in [false, true] {
12985 let tmp = tempfile::tempdir().unwrap();
12986 let mut runner = registered_runner(tmp.path()).await;
12987 runner.state.config.graph.timeout_resource_wait = 1;
12988 let winner = runner.state.candidates[0].clone();
12989 let marker = tmp.path().join("fixer-started");
12990 let mut fixer = spec("fixer");
12991 fixer.command = vec![
12992 "sh".into(),
12993 "-c".into(),
12994 format!("echo started >> '{}'; echo done", marker.display()),
12995 ];
12996 runner.roles.fixer = Some(vec![fixer]);
12997 if graft {
12998 let mut loser = winner.clone();
12999 loser.label = 'B';
13000 loser.index = 1;
13001 runner.state.candidates.push(loser);
13002 runner.state.config.graph.graft_runner_up = true;
13003 runner.state.tally = Some(Tally {
13004 first_choice: BTreeMap::new(),
13005 borda: BTreeMap::new(),
13006 winner: winner.label,
13007 rankings: 1,
13008 unanimous_initial: true,
13009 deliberated: false,
13010 changed_votes: 0,
13011 unanimous_final: true,
13012 tie_break: None,
13013 judges: 1,
13014 present: 1,
13015 quorum: 1,
13016 met_quorum: true,
13017 uncontested: None,
13018 });
13019 runner.state.judgements.push(crate::run::Judgement {
13020 judge: 1,
13021 seat: "judge-1".into(),
13022 agent: "judge".into(),
13023 ranking: vec![winner.label, 'B'],
13024 reasons: BTreeMap::new(),
13025 confidence: None,
13026 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13027 order: vec![0, 1],
13028 failed: None,
13029 duration_ms: 0,
13030 });
13031 } else {
13032 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13033 from_run: "earlier".into(),
13034 generation: 1,
13035 mode: crate::recovery::Mode::VerifyRepair,
13036 reasons: Vec::new(),
13037 reviewed_head: None,
13038 rounds: Vec::new(),
13039 open_findings: Vec::new(),
13040 evidence: None,
13041 criteria_inherited: true,
13042 repair: crate::recovery::Repair::NotStarted,
13043 });
13044 }
13045 let reader = crate::repo_lock::SeatLease::for_checkout(
13046 &winner.worktree,
13047 false,
13048 crate::config::test_coordination().as_ref(),
13049 )
13050 .await
13051 .unwrap();
13052 let pause = runner.pause.clone();
13053 let (result, ()) = tokio::join!(
13054 async {
13055 if graft {
13056 runner.graft().await
13057 } else {
13058 runner.recovery_repair(&winner).await
13059 }
13060 },
13061 async {
13062 tokio::time::sleep(Duration::from_millis(100)).await;
13063 pause.park();
13064 }
13065 );
13066 result.unwrap();
13067 assert!(!marker.exists(), "a blocked seat must not spawn");
13068 runner.state = RunState::load(&runner.state.id).unwrap();
13070 if graft {
13071 assert_eq!(
13072 runner.state.graft.as_ref().unwrap().outcome,
13073 GraftOutcome::WaitingForRestore
13074 );
13075 } else {
13076 assert!(matches!(
13077 runner.state.recovery.as_ref().unwrap().repair,
13078 crate::recovery::Repair::WaitingForRestore { .. }
13079 ));
13080 }
13081 drop(reader);
13082 runner.pause = Pause::new();
13083 runner.state.resource_block = None;
13084 let external =
13086 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13087 assert_eq!(
13088 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13089 0
13090 );
13091 let leftover = winner.worktree.join("leftover");
13092 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13093 let result = if graft {
13094 runner.graft().await
13095 } else {
13096 runner.recovery_repair(&winner).await
13097 };
13098 assert!(
13099 result
13100 .unwrap_err()
13101 .is::<crate::repo_lock::ResourceBlocked>()
13102 );
13103 assert!(leftover.exists());
13104 assert!(!marker.exists());
13105 drop(external);
13106 runner.state.resource_block = None;
13107 let result = if graft {
13108 runner.graft().await
13109 } else {
13110 runner.recovery_repair(&winner).await
13111 };
13112 result.unwrap();
13113 assert!(!leftover.exists(), "restore must precede retry");
13114 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13115 if graft {
13116 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13117 } else {
13118 assert!(matches!(
13119 runner.state.recovery.as_ref().unwrap().repair,
13120 crate::recovery::Repair::Done { .. }
13121 ));
13122 }
13123 }
13124 }
13125
13126 #[tokio::test]
13127 #[cfg(unix)]
13128 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13129 use std::os::fd::AsRawFd as _;
13130 let tmp = tempfile::tempdir().unwrap();
13131 let mut runner = registered_runner(tmp.path()).await;
13132 runner.state.config.graph.timeout_resource_wait = 1;
13133 let winner = runner.state.candidates[0].clone();
13134 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13135 runner.state.tally = Some(Tally {
13136 first_choice: BTreeMap::new(),
13137 borda: BTreeMap::new(),
13138 winner: winner.label,
13139 rankings: 0,
13140 unanimous_initial: false,
13141 deliberated: false,
13142 changed_votes: 0,
13143 unanimous_final: false,
13144 tie_break: None,
13145 judges: 0,
13146 present: 0,
13147 quorum: 0,
13148 met_quorum: true,
13149 uncontested: Some("fixture".into()),
13150 });
13151 runner.state.graft = Some(GraftRecord {
13152 pre: pre.clone(),
13153 post: None,
13154 lines: None,
13155 limit: None,
13156 agent: None,
13157 outcome: GraftOutcome::Running,
13158 requested: Vec::new(),
13159 adoption: None,
13160 adoption_error: None,
13161 });
13162 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13163 git::commit_all(&winner.worktree, "unfinished graft")
13164 .await
13165 .unwrap();
13166 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13167 let leftover = winner.worktree.join("leftover");
13168 std::fs::write(&leftover, "untrusted edits").unwrap();
13169 let path = runner.state.repo.join(".git/updater.lock");
13170 let external = std::fs::File::create(&path).unwrap();
13171 assert_eq!(
13172 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13173 0
13174 );
13175 assert!(
13176 runner
13177 .graft()
13178 .await
13179 .unwrap_err()
13180 .is::<crate::repo_lock::ResourceBlocked>()
13181 );
13182 assert!(matches!(
13183 runner.state.graft.as_ref().unwrap().outcome,
13184 GraftOutcome::Running
13185 ));
13186 assert_eq!(
13187 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13188 unfinished
13189 );
13190 assert!(leftover.exists());
13191 drop(external);
13192 std::fs::remove_file(&path).unwrap();
13193 std::fs::create_dir(&path).unwrap();
13194 assert!(runner.graft().await.is_err());
13195 assert!(matches!(
13196 runner.state.graft.as_ref().unwrap().outcome,
13197 GraftOutcome::Running
13198 ));
13199 assert_eq!(
13200 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13201 unfinished
13202 );
13203 assert!(leftover.exists());
13204 std::fs::remove_dir(&path).unwrap();
13205 runner.graft().await.unwrap();
13206 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13207 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13208 assert!(!leftover.exists());
13209 }
13210
13211 #[tokio::test]
13212 #[cfg(unix)]
13213 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13214 use std::os::fd::AsRawFd as _;
13215 let tmp = tempfile::tempdir().unwrap();
13216 let mut runner = registered_runner(tmp.path()).await;
13217 runner.state.prep_complete = Some(false);
13218 runner.state.config.graph.timeout_resource_wait = 1;
13219 runner.state.save().unwrap();
13220 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13221 assert_eq!(
13222 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13223 0
13224 );
13225 runner.execute().await.unwrap();
13226 let disk = RunState::load(&runner.state.id).unwrap();
13227 assert!(disk.driver_exited);
13228 assert_eq!(disk.driver_pid, None);
13229 assert_eq!(runner.state.status, RunStatus::Blocked);
13230 assert_eq!(
13231 runner.state.resource_block.as_ref().unwrap().kind,
13232 "repository coordination"
13233 );
13234 assert!(runner.state.prep_failure.is_none());
13235 assert!(runner.state.seats.is_empty());
13236 runner.pause.park();
13237 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13240 .await
13241 .unwrap()
13242 .unwrap();
13243 assert!(runner.state.parked);
13244 assert!(runner.state.prep_failure.is_none());
13245 let disk = RunState::load(&runner.state.id).unwrap();
13246 assert!(disk.driver_exited);
13247 assert_eq!(disk.driver_pid, None);
13248 }
13249
13250 #[tokio::test]
13251 #[cfg(unix)]
13252 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13253 use std::os::fd::AsRawFd as _;
13254 let tmp = tempfile::tempdir().unwrap();
13255 let mut runner = registered_runner(tmp.path()).await;
13256 runner.state.prep_complete = Some(false);
13257 runner.state.config.graph.timeout_resource_wait = 30;
13258 runner.state.save().unwrap();
13259 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13260 assert_eq!(
13261 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13262 0
13263 );
13264 assert!(
13265 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13266 .await
13267 .is_err()
13268 );
13269 let disk = RunState::load(&runner.state.id).unwrap();
13270 assert!(disk.driver_exited);
13271 assert_eq!(disk.driver_pid, None);
13272 assert!(disk.status.resumable());
13273 assert!(runner.state.driver_exited);
13274 assert_eq!(runner.state.driver_pid, None);
13275 runner.state.save().unwrap();
13276 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13277 }
13278
13279 #[tokio::test]
13280 #[cfg(unix)]
13281 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13282 let tmp = tempfile::tempdir().unwrap();
13283 let mut runner = registered_runner(tmp.path()).await;
13284 let winner = runner.state.candidates[0].clone();
13285 let cache = tmp.path().join("cache");
13286 let started = tmp.path().join("pre-gate-started");
13287 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13288 runner.state.config.verify.gate =
13289 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13290 let home = crate::run::home();
13291 let owner =
13292 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13293 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13294 crate::cache::AcquireOutcome::Acquired(g) => g,
13295 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13296 };
13297 let mut gate = Box::pin(runner.run_gate(&winner));
13298 assert!(
13299 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13300 .await
13301 .is_err()
13302 );
13303 assert!(!started.exists(), "pre-gate must wait for the cache too");
13304 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13308 let editor = crate::repo_lock::SeatLease::checkout_with(
13309 &winner.worktree,
13310 true,
13311 crate::config::test_coordination().as_ref(),
13312 &wait,
13313 )
13314 .await
13315 .unwrap();
13316 drop(editor);
13317 guard.release();
13318 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13319 .await
13320 .unwrap()
13321 .unwrap();
13322 drop(gate);
13323 assert!(outcomes.iter().all(CommandOutcome::ok));
13324 assert!(started.exists());
13325 }
13326
13327 #[tokio::test]
13328 #[cfg(unix)]
13329 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13330 use std::os::fd::AsRawFd as _;
13331
13332 ask_test_home();
13333 let tmp = tempfile::tempdir().unwrap();
13334 let repo = tmp.path().join("repo");
13335 std::fs::create_dir_all(&repo).unwrap();
13336 init_repo(&repo);
13337 let common = repo.join(".git");
13338 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13339 let mut runner = runner_at(RunStatus::Prep);
13340 runner.state.repo = repo.clone();
13341 runner.state.config.coordination = crate::config::test_coordination();
13342 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13343 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13344 runner.state.config.graph.advise = false;
13345 runner.roles.implementers = vec![spec("implement")];
13346 runner.roles.judges = vec![spec("judge")];
13347 runner.prep().await.unwrap();
13348 let winner = runner.state.candidates[0].clone();
13349 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13352 let completed = tmp.path().join("seat-completed");
13353 runner.roles.implementers[0].command = vec![
13354 "sh".into(),
13355 "-c".into(),
13356 format!("touch '{}'; echo completed", completed.display()),
13357 ];
13358 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13359 assert_eq!(
13360 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13361 0
13362 );
13363 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13364 let mut implement = Box::pin(runner.implement());
13365 tokio::select! {
13366 result = &mut implement => panic!("rescue must wait: {result:?}"),
13367 result = tokio::time::timeout(Duration::from_secs(10), async {
13368 while !completed.exists() {
13369 tokio::time::sleep(Duration::from_millis(20)).await;
13370 }
13371 }) => result.unwrap(),
13372 }
13373 assert!(
13374 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13375 .await
13376 .is_err()
13377 );
13378 assert_eq!(
13379 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13380 before
13381 );
13382 assert_eq!(
13383 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13384 0
13385 );
13386 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13387 .await
13388 .unwrap()
13389 .unwrap();
13390 drop(implement);
13391 assert_ne!(
13392 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13393 before
13394 );
13395 assert!(git::is_clean(&winner.worktree).await.unwrap());
13396
13397 let started = tmp.path().join("pre-gate-started");
13398 let proceed = tmp.path().join("pre-gate-proceed");
13399 let checked = tmp.path().join("gate-ran");
13400 runner.state.config.verify.pre_gate = vec![format!(
13401 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13402 started.display(),
13403 proceed.display()
13404 )];
13405 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13406 assert_eq!(
13407 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13408 0
13409 );
13410 let mut gate = Box::pin(runner.run_gate(&winner));
13411 assert!(
13412 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13413 .await
13414 .is_err()
13415 );
13416 assert!(!started.exists());
13417 assert!(!checked.exists());
13418 assert!(git::is_clean(&winner.worktree).await.unwrap());
13419 assert_eq!(
13420 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13421 0
13422 );
13423 tokio::select! {
13424 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13425 result = tokio::time::timeout(Duration::from_secs(10), async {
13426 while !started.exists() {
13427 tokio::time::sleep(Duration::from_millis(20)).await;
13428 }
13429 }) => result.unwrap(),
13430 }
13431 assert_eq!(
13432 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13433 -1,
13434 "mechanical edits must hold the real lock"
13435 );
13436 std::fs::write(proceed, "ready").unwrap();
13437 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13438 .await
13439 .unwrap()
13440 .unwrap();
13441 drop(gate);
13442 assert!(outcomes.iter().all(CommandOutcome::ok));
13443 assert!(checked.exists());
13444 assert!(runner.state.pre_gate_commit.is_some());
13445 assert!(git::is_clean(&winner.worktree).await.unwrap());
13446 assert_eq!(
13447 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13448 0,
13449 "gate completion must release coordination"
13450 );
13451 drop(outsider);
13452 std::fs::remove_file(common.join("updater.lock")).unwrap();
13453 std::fs::create_dir(common.join("updater.lock")).unwrap();
13454 std::fs::remove_file(&started).unwrap();
13455 std::fs::remove_file(&checked).unwrap();
13456 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13457 let error = runner.run_gate(&winner).await.unwrap_err();
13458 assert!(
13459 error
13460 .to_string()
13461 .contains("acquire mandatory repository lock")
13462 );
13463 assert!(
13464 !started.exists(),
13465 "failed acquisition must prevent pre-gate"
13466 );
13467 assert!(!checked.exists(), "failed acquisition must prevent gate");
13468 assert!(git::is_clean(&winner.worktree).await.unwrap());
13469 assert_eq!(
13470 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13471 head
13472 );
13473 }
13474
13475 #[tokio::test]
13476 #[cfg(unix)]
13477 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13478 use std::os::fd::AsRawFd as _;
13479 use std::time::Duration;
13480
13481 ask_test_home();
13482 let tmp = tempfile::tempdir().unwrap();
13483 let repo = tmp.path().join("repo");
13484 std::fs::create_dir_all(&repo).unwrap();
13485 init_repo(&repo);
13486 let common = repo.join(".git");
13487 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13488 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13489 assert_eq!(
13490 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13491 0
13492 );
13493 let mut runner = runner_at(RunStatus::Prep);
13494 runner.state.repo = repo.clone();
13495 runner.state.config.coordination = crate::config::test_coordination();
13496 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13497 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13498 runner.state.config.graph.advise = true;
13499 runner.state.config.graph.advisors = 1;
13500 runner.roles.implementers = vec![spec("implement")];
13501 runner.roles.judges = vec![spec("judge")];
13502 let root = runner.state.worktree_root();
13503 let started = tmp.path().join("setup-started");
13504 let proceed = tmp.path().join("setup-proceed");
13505 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13506 run: Some(format!(
13507 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13508 started.display(),
13509 proceed.display()
13510 )),
13511 ..Default::default()
13512 }];
13513 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13514 let config = git::git(&repo, &["config", "--local", "--list"])
13515 .await
13516 .unwrap();
13517 let mut prep = Box::pin(runner.prep());
13518 assert!(
13519 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13520 .await
13521 .is_err()
13522 );
13523 assert!(
13524 !root.exists(),
13525 "contention must prevent even checkout creation"
13526 );
13527 assert!(!started.exists(), "contention must prevent setup");
13528 assert_eq!(
13529 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13530 branches
13531 );
13532 assert_eq!(
13533 git::git(&repo, &["config", "--local", "--list"])
13534 .await
13535 .unwrap(),
13536 config
13537 );
13538 drop(lock);
13539 tokio::select! {
13540 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13541 result = tokio::time::timeout(Duration::from_secs(10), async {
13542 while !started.exists() {
13543 tokio::time::sleep(Duration::from_millis(20)).await;
13544 }
13545 }) => result.unwrap(),
13546 }
13547 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13548 assert_eq!(
13549 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13550 -1
13551 );
13552 assert!(
13553 tokio::time::timeout(
13554 Duration::from_millis(100),
13555 crate::repo_lock::SeatLease::for_checkout(
13556 &repo,
13557 true,
13558 crate::config::test_coordination().as_ref()
13559 )
13560 )
13561 .await
13562 .is_err(),
13563 "preparation must also exclude local editors"
13564 );
13565 std::fs::write(proceed, "ready").unwrap();
13566 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13567 .await
13568 .unwrap()
13569 .unwrap();
13570 drop(prep);
13571 assert_eq!(runner.state.prep_complete, Some(true));
13572 assert!(root.join("judge-1").exists());
13573 assert!(root.join("advisor-1").exists());
13574 assert_eq!(
13575 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13576 0
13577 );
13578 }
13579
13580 #[tokio::test]
13581 #[cfg(unix)]
13582 async fn preparation_lock_failure_prevents_repository_changes() {
13583 ask_test_home();
13584 let tmp = tempfile::tempdir().unwrap();
13585 let repo = tmp.path().join("repo");
13586 std::fs::create_dir_all(&repo).unwrap();
13587 init_repo(&repo);
13588 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13589 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13592 let mut runner = runner_at(RunStatus::Prep);
13593 runner.state.repo = repo.clone();
13594 runner.state.config.coordination = crate::config::test_coordination();
13595 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13596 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13597 runner.roles.implementers = vec![spec("implement")];
13598 let error = runner.prep().await.unwrap_err();
13599 assert!(
13600 error
13601 .to_string()
13602 .contains("acquire mandatory repository lock")
13603 );
13604 assert!(!runner.state.worktree_root().exists());
13605 assert!(runner.state.candidates.is_empty());
13606 assert!(!repo.join(".git/config.worktree").exists());
13607 assert_eq!(
13608 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13609 "* main"
13610 );
13611 }
13612
13613 #[tokio::test]
13614 async fn preparation_records_creation_intent_before_git_can_fail() {
13615 ask_test_home();
13616 let tmp = tempfile::tempdir().unwrap();
13617 let repo = tmp.path().join("repo");
13618 std::fs::create_dir_all(&repo).unwrap();
13619 init_repo(&repo);
13620 let mut runner = runner_at(RunStatus::Prep);
13621 runner.state.repo = repo.clone();
13622 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13623 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13624 runner.state.config.blind.commit_msg_hook = false;
13625 runner.roles.implementers = vec![spec("one")];
13626 let label = blind::assign_labels(1, runner.state.seed)[0];
13627 let branch = runner.state.branch_for(label);
13628 git::git(&repo, &["branch", &branch]).await.unwrap();
13631 assert!(runner.prep().await.is_err());
13632 let saved = RunState::load(&runner.state.id).unwrap();
13633 assert!(saved.candidates.is_empty());
13634 let event = saved
13635 .events
13636 .iter()
13637 .find(|e| e.node == "prep-intent")
13638 .unwrap();
13639 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13640 assert_eq!(intent["branch"], branch);
13641 assert_eq!(intent["start"], runner.state.base_commit);
13642 assert_eq!(
13643 intent["worktree"],
13644 runner
13645 .state
13646 .worktree_root()
13647 .join(format!("cand-{label}"))
13648 .to_string_lossy()
13649 .as_ref()
13650 );
13651 assert_eq!(intent["picks"], serde_json::json!([]));
13652 }
13653
13654 #[tokio::test]
13655 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13656 ask_test_home();
13657 let tmp = tempfile::tempdir().unwrap();
13658 let repo = tmp.path().join("repo");
13659 std::fs::create_dir_all(&repo).unwrap();
13660 init_repo(&repo);
13661 let mut runner = runner_at(RunStatus::Prep);
13662 runner.state.repo = repo.clone();
13663 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13664 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13665 runner.state.sources.pick_commits.push(refs::Input {
13666 token: "missing".into(),
13667 sha: "0".repeat(40),
13668 provenance: refs::Provenance::ExplicitCli,
13669 });
13670 assert!(runner.prep().await.is_err());
13671 assert!(runner.state.candidates.is_empty());
13672 assert!(!runner.state.worktree_root().exists());
13673 }
13674
13675 #[tokio::test]
13676 async fn prose_and_legacy_inference_never_import_into_candidates() {
13677 ask_test_home();
13678 let tmp = tempfile::tempdir().unwrap();
13679 let repo = tmp.path().join("repo");
13680 std::fs::create_dir_all(&repo).unwrap();
13681 init_repo(&repo);
13682 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13683 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13684 .await
13685 .unwrap();
13686 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13687 git::git(&repo, &["add", "."]).await.unwrap();
13688 git::git(&repo, &["commit", "-m", "historical implementation"])
13689 .await
13690 .unwrap();
13691 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13692 git::git(&repo, &["checkout", "main"]).await.unwrap();
13693 for instruction in [
13694 format!("Example: magi/history/A at {source}"),
13695 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13696 format!("Never import magi/history/A or {source}"),
13697 format!("Conversation answer: I mentioned {source} only as background"),
13698 ] {
13699 let mut runner = runner_at(RunStatus::Prep);
13700 runner.state.repo = repo.clone();
13701 runner.state.base_commit = base.clone();
13702 runner.state.instruction = instruction.clone();
13703 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13704 runner.state.config.blind.commit_msg_hook = false;
13705 runner.state.seeds = vec![refs::Seed {
13706 token: "magi/history/A".into(),
13707 sha: source.clone(),
13708 kind: refs::SeedKind::Unmerged,
13709 branch: true,
13710 detail: String::new(),
13711 }];
13712 runner.roles.implementers = vec![spec("one")];
13713 assert_eq!(seeded_instruction(&runner.state), instruction);
13714 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13716 legacy.as_object_mut().unwrap().remove("sources");
13717 runner.state = serde_json::from_value(legacy).unwrap();
13718 assert_eq!(runner.state.sources, refs::Sources::default());
13719 runner.state.save().unwrap();
13720 runner.state = RunState::load(&runner.state.id).unwrap();
13721 runner.prep().await.unwrap();
13722 for candidate in &runner.state.candidates {
13723 assert_eq!(
13724 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13725 base
13726 );
13727 assert!(!candidate.worktree.join("unwanted").exists());
13728 }
13729 }
13730 }
13731
13732 #[tokio::test]
13733 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13734 ask_test_home();
13735 let tmp = tempfile::tempdir().unwrap();
13736 let repo = tmp.path().join("repo");
13737 std::fs::create_dir_all(&repo).unwrap();
13738 init_repo(&repo);
13739 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13740 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13741 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13742 git::git(&repo, &["add", "."]).await.unwrap();
13743 git::git(&repo, &["commit", "-m", "required seed"])
13744 .await
13745 .unwrap();
13746 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13747 git::git(&repo, &["checkout", "main"]).await.unwrap();
13748 let mut runner = runner_at(RunStatus::Prep);
13749 runner.state.repo = repo.clone();
13750 runner.state.base_commit = base.clone();
13751 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13752 runner.state.config.graph.advise = false;
13753 runner.state.config.blind.commit_msg_hook = false;
13754 runner.state.sources =
13755 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13756 .await
13757 .unwrap();
13758 runner.roles.implementers = vec![spec("one"), spec("two")];
13759 runner.roles.judges = vec![spec("judge")];
13760 let label = blind::assign_labels(2, runner.state.seed)[0];
13761 let branch = runner.state.branch_for(label);
13762 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13763 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13764 .await
13765 .unwrap();
13766 runner.state.candidates.push(Candidate {
13767 index: 0,
13768 label,
13769 agent: "one".into(),
13770 branch,
13771 worktree,
13772 summary: String::new(),
13773 stat: String::new(),
13774 files: 0,
13775 commits: 0,
13776 empty: false,
13777 failed: None,
13778 verified_noop: None,
13779 duration_ms: 0,
13780 folded: false,
13781 });
13782 runner.state.save().unwrap();
13783 runner.state = RunState::load(&runner.state.id).unwrap();
13785 assert_eq!(runner.state.prep_complete, Some(false));
13786 runner.prep().await.unwrap();
13787 assert_eq!(runner.state.prep_complete, Some(true));
13788 assert_eq!(runner.state.candidates.len(), 2);
13789 for candidate in &runner.state.candidates {
13790 assert_eq!(
13791 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13792 git::tree_of(&repo, &seed).await.unwrap()
13793 );
13794 assert_eq!(
13795 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13796 "required commit"
13797 );
13798 }
13799 assert!(runner.state.worktree_root().join("judge-1").exists());
13800 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13801 runner.prep().await.unwrap();
13802 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13803
13804 runner.state.prep_complete = Some(false);
13805 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13806 run: Some(format!(
13807 "test -f '{}'",
13808 tmp.path().join("dependency-ready").display()
13809 )),
13810 ..Default::default()
13811 }];
13812 assert!(runner.prep().await.is_err());
13813 assert!(runner.state.prep_failure.is_some());
13814 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13815 runner.state = RunState::load(&runner.state.id).unwrap();
13816 assert!(
13817 runner.prep().await.is_err(),
13818 "correcting the cause alone must not auto-retry"
13819 );
13820 let failed_at = runner.state.events.last().unwrap().at;
13821 assert!(
13822 runner.retry_preparation(failed_at).is_err(),
13823 "old approvals cannot unlock the hold"
13824 );
13825 let mut task = crate::queue::Task::new(
13826 "retry".into(),
13827 "retry setup".into(),
13828 repo.clone(),
13829 crate::queue::Source::Human,
13830 );
13831 task.start(runner.state.id.clone());
13832 task.hold_machine(Some("setup failed".into()));
13833 task.release_operator("cli", None);
13834 runner
13835 .retry_preparation(task.resumption.as_ref().unwrap().at)
13836 .unwrap();
13837 runner.prep().await.unwrap();
13838 assert!(runner.state.prep_failure.is_none());
13839 assert_eq!(runner.state.prep_complete, Some(true));
13840 assert_eq!(runner.state.candidates.len(), 2);
13841 assert!(runner.state.worktree_root().join("judge-1").exists());
13842 for candidate in &runner.state.candidates {
13843 assert_eq!(
13844 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13845 git::tree_of(&repo, &seed).await.unwrap()
13846 );
13847 }
13848 }
13849
13850 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13851 let mut heads = Vec::new();
13852 for candidate in candidates {
13853 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13854 }
13855 heads
13856 }
13857
13858 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13859 ask_test_home();
13860 let tmp = tempfile::tempdir().unwrap();
13861 let repo = tmp.path().join("repo");
13862 std::fs::create_dir(&repo).unwrap();
13863 init_repo(&repo);
13864 let mut runner = runner_at(RunStatus::Reviewing);
13865 runner.state.repo = repo.clone();
13866 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13867 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13868 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13869 let branch = format!("candidate-{label}");
13870 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13871 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13872 .await
13873 .unwrap();
13874 if label == 'B' {
13875 std::fs::write(worktree.join("change"), "winner").unwrap();
13876 git::rescue_commit(&worktree, "candidate change")
13877 .await
13878 .unwrap();
13879 }
13880 runner.state.candidates.push(Candidate {
13881 index,
13882 label,
13883 agent: "alpha".into(),
13884 branch,
13885 worktree,
13886 summary: "completed reply".into(),
13887 stat: String::new(),
13888 files: usize::from(label == 'B'),
13889 commits: usize::from(label == 'B'),
13890 empty: label == 'A',
13891 failed: None,
13892 verified_noop: None,
13893 duration_ms: 1,
13894 folded: false,
13895 });
13896 }
13897 runner.state.judge_skipped = true;
13898 runner.state.tally = Some(Tally {
13899 first_choice: BTreeMap::new(),
13900 borda: BTreeMap::new(),
13901 winner: 'B',
13902 rankings: 0,
13903 unanimous_initial: false,
13904 deliberated: false,
13905 changed_votes: 0,
13906 unanimous_final: false,
13907 tie_break: None,
13908 judges: 0,
13909 present: 0,
13910 quorum: 0,
13911 met_quorum: true,
13912 uncontested: Some("only B produced a change".into()),
13913 });
13914 runner.state.save().unwrap();
13915 (tmp, runner)
13916 }
13917
13918 #[tokio::test]
13919 #[cfg(unix)]
13920 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13921 for (solo, failed_checkpoint) in
13922 [(true, false), (false, false), (true, true), (false, true)]
13923 {
13924 let (_tmp, mut runner) = preservation_resume_fixture().await;
13925 if solo {
13926 runner.state.candidates.retain(|c| c.label == 'A');
13927 }
13928 runner.state.candidates[0].empty = false;
13929 runner.state.candidates[0].summary.clear();
13930 runner.state.tally = None;
13931 runner.state.judge_skipped = false;
13932 runner.state.status = RunStatus::Implementing;
13933 runner.state.config.disk.min_free_bytes = u64::MAX;
13934 let mut implementer = spec("alpha");
13935 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13936 runner.roles.implementers = vec![implementer];
13937 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13938 if failed_checkpoint {
13939 std::fs::create_dir_all(&patch).unwrap();
13940 }
13941 runner.state.save().unwrap();
13942 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13943 runner.state = RunState::load(&runner.state.id).unwrap();
13944 assert_eq!(runner.state.status, RunStatus::Blocked);
13945 assert!(runner.state.interrupted_implementations.contains(&'A'));
13946 assert_eq!(runner.state.candidates[0].commits, 0);
13947 assert!(!runner.state.candidates[0].empty);
13948 assert!(runner.state.candidates[0].failed.is_none());
13949 if failed_checkpoint {
13950 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13951 std::fs::remove_dir(patch).unwrap();
13952 }
13953 runner.state.resource_block = None;
13954 runner.state.config.disk.min_free_bytes = 0;
13955 runner.roles.implementers[0].command = vec![
13956 "sh".into(),
13957 "-c".into(),
13958 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13959 ];
13960 runner.implement().await.unwrap();
13961 assert!(runner.state.interrupted_implementations.is_empty());
13962 assert!(runner.state.candidate_preservation.is_empty());
13963 assert_eq!(runner.state.candidates[0].commits, 1);
13964 assert!(!runner.state.candidates[0].empty);
13965 assert!(
13966 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13967 .unwrap()
13968 .contains("retried empty interrupted seat")
13969 );
13970 if !solo {
13971 assert_eq!(runner.state.candidates[1].commits, 1);
13972 }
13973 }
13974 }
13975
13976 #[tokio::test]
13977 #[cfg(unix)]
13978 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13979 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13980 let (_tmp, mut runner) = preservation_resume_fixture().await;
13981 runner.state.candidates.retain(|c| c.label == 'A');
13982 runner.state.candidates[0].empty = false;
13983 runner.state.candidates[0].summary.clear();
13984 runner.state.tally = None;
13985 runner.state.judge_skipped = false;
13986 runner.state.status = RunStatus::Implementing;
13987 runner.state.config.disk.min_free_bytes = u64::MAX;
13990 let wt = runner.state.candidates[0].worktree.clone();
13991 std::fs::write(
13992 wt.join("interrupted-source"),
13993 "retain implementation work\n",
13994 )
13995 .unwrap();
13996 if committed {
13997 git::rescue_commit(&wt, "implementation checkpoint")
13998 .await
13999 .unwrap();
14000 }
14001 let lock = git::git(
14002 &wt,
14003 &[
14004 "rev-parse",
14005 "--path-format=absolute",
14006 "--git-path",
14007 "index.lock",
14008 ],
14009 )
14010 .await
14011 .unwrap();
14012 if block_rescue {
14013 std::fs::write(&lock, "fixture-owned lock").unwrap();
14014 }
14015 let mut implementer = spec("alpha");
14016 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14017 runner.roles.implementers = vec![implementer];
14018 runner.state.save().unwrap();
14019 let result = runner.implement().await;
14020 assert_eq!(result.is_err(), block_rescue);
14021 assert_eq!(
14022 runner.state.resource_block.as_ref().unwrap().kind,
14023 "disk pressure"
14024 );
14025 assert!(
14026 runner
14027 .state
14028 .events
14029 .iter()
14030 .any(|e| e.message.contains("agent execution interrupted"))
14031 );
14032 assert!(
14033 runner
14034 .state
14035 .dir()
14036 .join("artifacts/candidate-A-recovery/staged.patch")
14037 .exists()
14038 );
14039 if block_rescue {
14040 let saved = RunState::load(&runner.state.id).unwrap();
14041 assert!(saved.candidate_preservation.contains_key(&'A'));
14042 assert!(!saved.candidates[0].empty);
14043 assert_eq!(
14044 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14045 "retain implementation work\n"
14046 );
14047 std::fs::remove_file(lock).unwrap();
14048 runner.state = saved;
14049 } else {
14050 assert_eq!(runner.state.status, RunStatus::Blocked);
14051 assert_eq!(runner.state.candidates[0].commits, 1);
14052 }
14053 runner.roles.implementers.clear();
14056 runner.state.config.disk.min_free_bytes = 0;
14057 runner.state.resource_block = None;
14058 runner.implement().await.unwrap();
14059 assert!(runner.state.candidate_preservation.is_empty());
14060 assert_eq!(runner.state.candidates[0].commits, 1);
14061 assert!(!runner.state.candidates[0].empty);
14062 assert!(git::is_clean(&wt).await.unwrap());
14063 assert!(
14064 crate::run::read_artifact(&runner.state, "cand-A.patch")
14065 .unwrap()
14066 .contains("retain implementation work")
14067 );
14068 }
14069 }
14070
14071 #[tokio::test]
14072 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14073 let (_tmp, mut runner) = preservation_resume_fixture().await;
14074 runner.implement().await.unwrap();
14076 assert!(runner.state.candidate_preservation.is_empty());
14077 assert!(
14078 !runner
14079 .state
14080 .dir()
14081 .join("artifacts/candidate-A-recovery")
14082 .exists()
14083 );
14084 runner.fold_losers().await.unwrap();
14085 assert!(!runner.state.candidates[0].worktree.exists());
14086 for schema in [14, crate::run::SCHEMA] {
14087 runner.state.schema = schema;
14088 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14089 runner.state.save().unwrap();
14090 runner.state = RunState::load(&runner.state.id).unwrap();
14091 runner.implement().await.unwrap();
14092 assert!(runner.state.candidate_preservation.is_empty());
14093 assert!(runner.state.candidates[0].folded);
14094 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14095 }
14096 }
14097
14098 #[tokio::test]
14099 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14100 let (_tmp, mut runner) = preservation_resume_fixture().await;
14101 let wt = runner.state.candidates[0].worktree.clone();
14102 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14103 runner.state.schema = 14;
14104 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14105 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14106 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14107 for _ in 0..2 {
14108 assert!(runner.implement().await.is_err());
14109 assert_eq!(runner.state.status, RunStatus::Blocked);
14110 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14111 assert!(
14112 runner.state.candidates[0].empty,
14113 "old verdict must remain unusable"
14114 );
14115 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14116 assert!(runner.state.judge_skipped);
14117 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14118 assert!(
14119 crate::run::read_artifact(&runner.state, "cand-A.patch")
14120 .unwrap()
14121 .contains("owned edits")
14122 );
14123 assert!(wt.exists());
14124 assert_eq!(
14125 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14126 .await
14127 .unwrap(),
14128 1
14129 );
14130 runner.state = RunState::load(&runner.state.id).unwrap();
14131 }
14132 }
14133
14134 #[tokio::test]
14135 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14136 let (_tmp, mut runner) = preservation_resume_fixture().await;
14137 let wt = runner.state.candidates[0].worktree.clone();
14138 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14139 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14140 let index_lock = PathBuf::from(
14141 git::git(
14142 &wt,
14143 &[
14144 "rev-parse",
14145 "--path-format=absolute",
14146 "--git-path",
14147 "index.lock",
14148 ],
14149 )
14150 .await
14151 .unwrap(),
14152 );
14153 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14154 runner.state.schema = 14;
14155 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14156 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14157 assert!(runner.implement().await.is_err());
14158 assert!(runner.after_implement().is_err());
14159 assert!(runner.fold_losers().await.is_err());
14160 assert_eq!(
14161 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14162 "failed seat edits"
14163 );
14164 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14165 runner.state = RunState::load(&runner.state.id).unwrap();
14166 std::fs::remove_file(index_lock).unwrap();
14167 assert!(runner.implement().await.is_err());
14169 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14170 assert!(runner.state.candidates[0].empty);
14171 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14172 assert!(
14173 crate::run::read_artifact(&runner.state, "cand-A.patch")
14174 .unwrap()
14175 .contains("failed seat edits")
14176 );
14177 assert!(wt.exists());
14178 }
14179
14180 #[tokio::test]
14181 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14182 let (_tmp, mut runner) = preservation_resume_fixture().await;
14183 let repo = runner.state.repo.clone();
14184 let common = git::git(
14185 &repo,
14186 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14187 )
14188 .await
14189 .unwrap();
14190 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14191 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14192 assert!(runner.fold_losers().await.is_err());
14193 assert!(!runner.state.candidates[0].worktree.exists());
14194 assert!(!runner.state.candidates[0].folded);
14195 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14196 runner.state = RunState::load(&runner.state.id).unwrap();
14197 std::fs::remove_file(branch_lock).unwrap();
14198 let judge = runner.state.worktree_root().join("judge-1");
14199 git::worktree_add_detached(&repo, &judge, "main")
14200 .await
14201 .unwrap();
14202 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14203 runner.roles.judges = vec![spec("judge")];
14204 assert!(runner.fold_losers().await.is_err());
14205 runner.state = RunState::load(&runner.state.id).unwrap();
14206 assert!(
14207 runner.state.candidates[0].folded,
14208 "candidate progress saved before judge failure"
14209 );
14210 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14211 assert!(judge.join("uncommitted").exists());
14212 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14213 runner.fold_losers().await.unwrap();
14214 assert!(!judge.exists());
14215 runner.fold_losers().await.unwrap();
14216 }
14217
14218 #[tokio::test]
14219 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14220 let (_tmp, mut runner) = preservation_resume_fixture().await;
14221 runner.state.candidates.retain(|c| c.label == 'B');
14222 runner.state.tally = None;
14223 runner.state.judge_skipped = false;
14224 runner.state.prep_complete = Some(true);
14225 runner.state.config.graph.advise = false;
14226 runner.state.config.graph.selected_implementer = Some("alpha".into());
14227 let wt = runner.state.candidates[0].worktree.clone();
14228 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14229 runner
14230 .state
14231 .candidate_preservation
14232 .insert('B', "awaiting persistence".into());
14233 let index_lock = git::git(
14234 &wt,
14235 &[
14236 "rev-parse",
14237 "--path-format=absolute",
14238 "--git-path",
14239 "index.lock",
14240 ],
14241 )
14242 .await
14243 .unwrap();
14244 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14245 runner.state.save().unwrap();
14246 assert!(runner.execute_graph().await.is_err());
14247 assert_eq!(runner.state.status, RunStatus::Blocked);
14248 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14249 std::fs::remove_file(index_lock).unwrap();
14250 for _ in 0..2 {
14253 runner.state = RunState::load(&runner.state.id).unwrap();
14254 runner.pause.park();
14255 runner.execute_graph().await.unwrap();
14256 assert_eq!(runner.state.status, RunStatus::Implementing);
14257 assert!(runner.state.candidate_preservation.is_empty());
14258 assert_eq!(runner.state.candidates[0].commits, 2);
14259 assert!(runner.state.tally.is_none());
14260 }
14261 runner.implement().await.unwrap();
14262 assert_eq!(runner.state.status, RunStatus::Judging);
14263 runner.state.status = RunStatus::Blocked;
14265 runner.state.candidates[0].failed = Some("implementation failed".into());
14266 runner.state.save().unwrap();
14267 let error = runner.execute_graph().await.unwrap_err();
14268 assert!(
14269 error
14270 .to_string()
14271 .contains("selected implementation is held")
14272 );
14273 }
14274
14275 #[tokio::test]
14276 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14277 let (_tmp, mut runner) = preservation_resume_fixture().await;
14278 let repo = runner.state.repo.clone();
14279 let candidate = runner.state.candidates[0].clone();
14280 let registered = git::worktree_holding(&repo, &candidate.branch)
14283 .await
14284 .unwrap()
14285 .unwrap();
14286 assert_eq!(
14287 registered.canonicalize().unwrap(),
14288 candidate.worktree.canonicalize().unwrap()
14289 );
14290 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14291 assert_eq!(
14292 git::worktree_holding(&repo, &candidate.branch)
14293 .await
14294 .unwrap(),
14295 Some(registered)
14296 );
14297 assert!(
14298 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14299 "the stale registration initially keeps the branch checked out"
14300 );
14301 runner.fold_losers().await.unwrap();
14302 assert!(runner.state.candidates[0].folded);
14303 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14304 assert_eq!(
14305 git::worktree_holding(&repo, &candidate.branch)
14306 .await
14307 .unwrap(),
14308 None
14309 );
14310 runner.state = RunState::load(&runner.state.id).unwrap();
14311 runner.fold_losers().await.unwrap();
14312 }
14313
14314 #[tokio::test]
14315 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14316 let (_tmp, mut runner) = preservation_resume_fixture().await;
14317 let repo = runner.state.repo.clone();
14318 let candidate = runner.state.candidates[0].clone();
14319 let winner = runner.state.candidates[1].clone();
14320 let registered = git::worktree_holding(&repo, &candidate.branch)
14321 .await
14322 .unwrap()
14323 .unwrap();
14324 assert_eq!(
14325 registered.canonicalize().unwrap(),
14326 candidate.worktree.canonicalize().unwrap()
14327 );
14328 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14329 assert_eq!(
14330 git::worktree_holding(&repo, &candidate.branch)
14331 .await
14332 .unwrap(),
14333 Some(registered)
14334 );
14335 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14336
14337 for _ in 0..2 {
14338 fold_run(&mut runner.state, false, &crate::run::home())
14339 .await
14340 .unwrap();
14341 runner.state = RunState::load(&runner.state.id).unwrap();
14342 assert!(runner.state.candidates[0].folded);
14343 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14344 assert_eq!(
14345 git::worktree_holding(&repo, &candidate.branch)
14346 .await
14347 .unwrap(),
14348 None
14349 );
14350 assert!(!runner.state.candidates[1].folded);
14351 assert!(winner.worktree.exists());
14352 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14353 }
14354 }
14355
14356 #[tokio::test]
14357 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14358 ask_test_home();
14359 let tmp = tempfile::tempdir().unwrap();
14360 let repo = tmp.path().join("repo");
14361 std::fs::create_dir(&repo).unwrap();
14362 init_repo(&repo);
14363 let mut runner = runner_at(RunStatus::Implementing);
14364 runner.state.repo = repo.clone();
14365 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14366 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14367 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14368 let branch = format!("candidate-{label}");
14369 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14370 git::worktree_add_branch(&repo, &wt, &branch, "main")
14371 .await
14372 .unwrap();
14373 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14374 git::git(&wt, &["add", "README.md"]).await.unwrap();
14375 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14376 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14377 runner.state.candidates.push(Candidate {
14378 index,
14379 label,
14380 agent: "alpha".to_owned(),
14381 branch,
14382 worktree: wt,
14383 summary: "completed implementation".to_owned(),
14384 stat: String::new(),
14385 files: 0,
14386 commits: 0,
14387 empty: false,
14388 failed: None,
14389 verified_noop: None,
14390 duration_ms: 42,
14391 folded: false,
14392 });
14393 runner
14394 .state
14395 .candidate_preservation
14396 .insert(label, "awaiting persistence".to_owned());
14397 }
14398 let wt = runner.state.candidates[0].worktree.clone();
14399 let index_lock = git::git(
14400 &wt,
14401 &[
14402 "rev-parse",
14403 "--path-format=absolute",
14404 "--git-path",
14405 "index.lock",
14406 ],
14407 )
14408 .await
14409 .unwrap();
14410 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14411 runner.state.save().unwrap();
14412 assert!(runner.persist_candidates().await.is_err());
14413 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14414 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14415 assert_eq!(
14416 runner.state.candidates[1].commits, 1,
14417 "other results are still collected"
14418 );
14419 assert!(
14420 !runner.state.candidates[0].empty,
14421 "failed rescue is not no change"
14422 );
14423 assert_eq!(
14424 runner.state.candidates[0].summary,
14425 "completed implementation"
14426 );
14427 assert!(runner.after_implement().is_err());
14428 let home = tmp.path().join("fold-home");
14429 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14430 assert!(wt.join("new-file").exists());
14431 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14432 assert!(recovery.join("staged.patch").exists());
14433 assert!(recovery.join("unstaged.patch").exists());
14434 assert_eq!(
14435 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14436 "untracked edits\n"
14437 );
14438 runner.state.begin_pass();
14439 runner.state.end_pass_parked(ParkCause::Upgrade {
14440 from: "1.0".into(),
14441 to: Some("1.1".into()),
14442 });
14443 runner.state.save().unwrap();
14444 runner.state = RunState::load(&runner.state.id).unwrap();
14445 assert!(matches!(
14446 runner.state.passes.last().unwrap().end,
14447 Some(crate::run::PassEnd::Parked {
14448 cause: ParkCause::Upgrade { .. }
14449 })
14450 ));
14451 std::fs::remove_file(index_lock).unwrap();
14452 runner.implement().await.unwrap();
14454 assert!(runner.state.candidate_preservation.is_empty());
14455 assert!(
14456 runner
14457 .state
14458 .candidates
14459 .iter()
14460 .all(|c| c.commits == 1 && !c.empty)
14461 );
14462 assert!(git::is_clean(&wt).await.unwrap());
14463 assert!(
14464 crate::run::read_artifact(&runner.state, "cand-A.patch")
14465 .unwrap()
14466 .contains("later tracked edits")
14467 );
14468 }
14469
14470 #[tokio::test]
14471 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14472 ask_test_home();
14473 let tmp = tempfile::tempdir().unwrap();
14474 let repo = tmp.path().join("repo");
14475 std::fs::create_dir(&repo).unwrap();
14476 init_repo(&repo);
14477 let wt = tmp.path().join("candidate");
14478 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14479 .await
14480 .unwrap();
14481 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14482 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14483 assert!(wt.join("uncommitted").exists());
14484 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14485 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14486 let mut runner = runner_at(RunStatus::Implementing);
14487 runner.state.repo = repo.clone();
14488 runner.state.base_commit = "missing-base".to_owned();
14489 runner.state.candidates.push(Candidate {
14490 index: 0,
14491 label: 'A',
14492 agent: "alpha".into(),
14493 branch: "candidate".into(),
14494 worktree: wt.clone(),
14495 summary: "completed".into(),
14496 stat: String::new(),
14497 files: 0,
14498 commits: 0,
14499 empty: false,
14500 failed: None,
14501 verified_noop: None,
14502 duration_ms: 1,
14503 folded: false,
14504 });
14505 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14506 assert!(
14507 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14508 .await
14509 .is_err()
14510 );
14511 assert!(!runner.state.candidates[0].folded);
14512 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14513 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14514 runner
14515 .state
14516 .candidate_preservation
14517 .insert('A', "pending".into());
14518 assert!(runner.persist_candidates().await.is_err());
14519 assert!(!runner.state.candidates[0].empty);
14520 assert!(!runner.state.candidates[0].folded);
14521 assert!(wt.exists());
14522 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14523 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14524 std::fs::create_dir_all(&patch_path).unwrap();
14525 assert!(
14526 runner.persist_candidates().await.is_err(),
14527 "artifact failure stays pending"
14528 );
14529 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14530 assert!(!runner.state.candidates[0].empty);
14531 std::fs::remove_dir(patch_path).unwrap();
14532 runner.persist_candidates().await.unwrap();
14533 assert!(
14534 runner.state.candidates[0].empty,
14535 "only successful inspection confirms no change"
14536 );
14537 }
14538
14539 fn init_repo(dir: &Path) {
14540 let run = |args: &[&str]| {
14541 let out = std::process::Command::new("git")
14542 .args(args)
14543 .current_dir(dir)
14544 .quiet()
14545 .output()
14546 .expect("spawn git");
14547 assert!(
14548 out.status.success(),
14549 "git {args:?} failed: {}",
14550 String::from_utf8_lossy(&out.stderr)
14551 );
14552 };
14553 run(&["init", "-b", "main"]);
14554 run(&["config", "user.name", "magi test"]);
14555 run(&["config", "user.email", "magi@example.com"]);
14556 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14557 run(&["add", "-A"]);
14558 run(&["commit", "-m", "init"]);
14559 }
14560
14561 fn ask_test_home() {
14569 crate::run::pin_test_home();
14570 }
14571
14572 fn runner_at(status: RunStatus) -> Runner {
14575 let mut state = RunState::new(
14576 PathBuf::from("/nonexistent/repo"),
14577 "main".to_owned(),
14578 "deadbeef".to_owned(),
14579 "task".to_owned(),
14580 Config::default(),
14581 );
14582 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14585 state.id = format!(
14586 "20000101-000000-{:04x}",
14587 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14588 );
14589 state.status = status;
14590 Runner {
14591 state,
14592 roles: ResolvedRoles {
14593 implementers: Vec::new(),
14594 judges: Vec::new(),
14595 reviewers: Vec::new(),
14596 fixer: None,
14597 conductor: conductor(),
14598 implementer_roster: Vec::new(),
14599 judge_roster: Vec::new(),
14600 reviewer_roster: Vec::new(),
14601 },
14602 sem: Arc::new(Semaphore::new(1)),
14603 pause: Pause::new(),
14604 interrupt: Pause::new(),
14605 }
14606 }
14607
14608 #[tokio::test]
14609 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14610 crate::run::pin_test_home();
14611 let mut runner = runner_at(RunStatus::Gating);
14612 runner.state.save().unwrap();
14613 let id = runner.state.id.clone();
14614 let passes = runner.state.passes.len();
14615 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14616 let mut claiming = Box::pin(runner.claim_driver());
14617 std::future::poll_fn(|cx| {
14618 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14619 std::task::Poll::Ready(())
14620 })
14621 .await;
14622 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14623 drop(owner);
14624 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14625 .await
14626 .unwrap()
14627 .unwrap();
14628 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14629 assert!(execution.runner.state.resource_block.is_none());
14630 assert_eq!(
14631 execution.runner.state.liveness(false),
14632 crate::run::Liveness::Live
14633 );
14634 drop(execution);
14635 assert!(RunState::load(&id).unwrap().driver_exited);
14636 }
14637
14638 #[tokio::test]
14639 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14640 crate::run::pin_test_home();
14641 let mut runner = runner_at(RunStatus::Gating);
14642 runner.state.save().unwrap();
14643 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14646 assert!(runner.claim_driver().await.is_err());
14647 assert!(runner.state.driver_exited);
14648 assert_eq!(runner.state.driver_pid, None);
14649 let disk = RunState::load(&runner.state.id).unwrap();
14650 assert!(disk.driver_exited);
14651 assert_eq!(disk.driver_pid, None);
14652 assert_eq!(disk.status, RunStatus::Gating);
14653 }
14654
14655 #[tokio::test]
14656 async fn an_early_graph_error_releases_its_driver() {
14657 crate::run::pin_test_home();
14658 let mut runner = runner_at(RunStatus::Blocked);
14659 runner.state.config.graph.selected_implementer = Some("selected".into());
14660 runner.state.save().unwrap();
14661 assert!(runner.execute().await.is_err());
14662 let disk = RunState::load(&runner.state.id).unwrap();
14663 assert!(disk.driver_exited);
14664 assert_eq!(disk.driver_pid, None);
14665 assert_eq!(disk.status, RunStatus::Blocked);
14666 }
14667
14668 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14669 crate::run::MergeRepair {
14670 question: "repair-question".into(),
14671 head: "old-head".into(),
14672 pr: "https://github.com/o/r/pull/42".into(),
14673 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14674 gate: vec![outcome(Some(0), false)],
14675 gate_head: Some("old-head".into()),
14676 gate_fixes: Vec::new(),
14677 pre_gate: Vec::new(),
14678 phase,
14679 failure: None,
14680 }
14681 }
14682
14683 #[tokio::test]
14684 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14685 crate::run::pin_test_home();
14686 let mut runner = runner_at(RunStatus::Landing);
14687 runner.state.merge_repairs.push(merge_repair_request(
14688 crate::run::MergeRepairPhase::Requested,
14689 ));
14690 runner.state.save().unwrap();
14691 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14692 assert!(
14693 runner
14694 .execute()
14695 .await
14696 .unwrap_err()
14697 .to_string()
14698 .contains("already running")
14699 );
14700 assert_eq!(
14701 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14702 crate::run::MergeRepairPhase::Requested
14703 );
14704 drop(claim);
14705 let mut disk = runner.state.clone();
14706 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14707 disk.save().unwrap();
14708 assert!(
14709 runner
14710 .execute()
14711 .await
14712 .unwrap_err()
14713 .to_string()
14714 .contains("decision changed")
14715 );
14716 assert_eq!(
14717 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14718 crate::run::MergeRepairPhase::Complete
14719 );
14720 }
14721
14722 #[test]
14723 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14724 crate::run::pin_test_home();
14725 let mut runner = runner_at(RunStatus::Landing);
14726 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14727 runner.state.reviews = request.reviews.clone();
14728 runner.state.gate = request.gate.clone();
14729 runner.state.gate_ran = true;
14730 runner.state.land_approval = Some(crate::run::LandApproval {
14731 question: "old".into(),
14732 head: "old-head".into(),
14733 });
14734 runner.state.merge_repairs.push(request);
14735 runner.begin_merge_repair_review(0).unwrap();
14736 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14737 assert!(!runner.state.gate_status().ok());
14738 assert!(runner.state.land_approval.is_none());
14739 let disk = RunState::load(&runner.state.id).unwrap();
14740 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14741 assert!(disk.merge_repairs[0].gate[0].ok());
14742 assert_eq!(
14743 disk.merge_repairs[0].phase,
14744 crate::run::MergeRepairPhase::Reviewing
14745 );
14746 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14747 assert!(disk.reviews.is_empty());
14748 }
14749
14750 #[tokio::test]
14751 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14752 crate::run::pin_test_home();
14753 for phase in [
14754 crate::run::MergeRepairPhase::Fixing,
14755 crate::run::MergeRepairPhase::Requested,
14756 ] {
14757 let mut runner = runner_at(RunStatus::Landing);
14758 runner.state.config.graph.land_rounds = 0;
14759 runner.state.merge_repairs.push(merge_repair_request(phase));
14760 runner.state.save().unwrap();
14761 assert!(!runner.repair_merge_review().await.unwrap());
14762 assert_eq!(runner.state.status, RunStatus::Blocked);
14763 assert_eq!(
14764 runner.state.merge_repairs[0].phase,
14765 crate::run::MergeRepairPhase::Failed
14766 );
14767 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14768 assert!(
14769 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14770 "interrupted"
14771 } else {
14772 "budget exhausted"
14773 })
14774 );
14775 assert!(runner.state.merge.is_none());
14776 }
14777 }
14778
14779 #[test]
14783 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14784 crate::run::pin_test_home();
14785 let mut runner = runner_at(RunStatus::Implementing);
14786 let interrupt = Pause::new();
14787 runner.watch_interrupt(interrupt.clone());
14788
14789 interrupt.park_because("task a1b2 asked to run first");
14790
14791 assert!(runner.park_here().expect("park_here"));
14792 assert!(runner.state.parked);
14793 let last = runner.state.events.last().expect("a park event");
14794 assert_eq!(last.node, "park");
14795 assert!(
14796 last.message.contains("task a1b2 asked to run first"),
14797 "expected the interrupt reason in {:?}",
14798 last.message
14799 );
14800 }
14801
14802 #[test]
14805 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14806 crate::run::pin_test_home();
14807 let mut runner = runner_at(RunStatus::Implementing);
14808 runner.state.begin_pass();
14809 let shutdown = Pause::new();
14810 runner.on_pause(shutdown.clone());
14811 let upgrade = ParkCause::Upgrade {
14812 from: "0.121.0".to_owned(),
14813 to: Some("0.121.1".to_owned()),
14814 };
14815 shutdown.park_for(upgrade.clone());
14816 shutdown.park_for(ParkCause::Operator);
14817
14818 assert!(runner.park_here().expect("park_here"));
14819 assert_eq!(
14820 runner.state.passes[0].end,
14821 Some(crate::run::PassEnd::Parked { cause: upgrade })
14822 );
14823 let last = runner.state.events.last().expect("a park event");
14824 assert!(last.message.contains("0.121.0"), "{}", last.message);
14825 }
14826
14827 #[test]
14835 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14836 crate::run::pin_test_home();
14837 let mut runner = runner_at(RunStatus::Implementing);
14838 let shutdown = Pause::new();
14839 runner.on_pause(shutdown.clone());
14840 let interrupt = Pause::new();
14841 runner.watch_interrupt(interrupt.clone());
14842
14843 assert!(!runner.park_here().expect("park_here"));
14845 assert!(!runner.state.parked);
14846
14847 interrupt.park_because("test");
14849 assert!(!shutdown.parked());
14850 assert!(runner.park_here().expect("park_here"));
14851 }
14852
14853 #[tokio::test]
14867 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14868 crate::run::pin_test_home();
14869 let mut runner = runner_at(RunStatus::Implementing);
14870 let interrupt = Pause::new();
14871 runner.watch_interrupt(interrupt.clone());
14872
14873 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14874 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14875
14876 let node = async move {
14880 started_tx.send(()).expect("send started");
14881 finish_rx.await.expect("recv finish");
14882 "node finished"
14883 };
14884
14885 let interrupter = async move {
14886 started_rx.await.expect("recv started");
14887 interrupt.park_because("higher-priority task waiting");
14889 tokio::task::yield_now().await;
14893 finish_tx.send(()).expect("send finish");
14894 };
14895
14896 let (node_result, ()) = tokio::join!(node, interrupter);
14897 assert_eq!(
14898 node_result, "node finished",
14899 "the in-flight call ran to completion"
14900 );
14901
14902 assert!(runner.park_here().expect("park_here"));
14905 assert!(runner.state.parked);
14906 }
14907
14908 #[test]
14914 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14915 crate::run::pin_test_home();
14916 let mut runner = runner_at(RunStatus::Judging);
14917 runner.state.config.agents = vec![conductor()];
14921 runner.state.candidates = vec![Candidate {
14922 index: 0,
14923 label: 'A',
14924 agent: "alpha".to_owned(),
14925 branch: "magi/x/A".to_owned(),
14926 worktree: PathBuf::from("/nonexistent/worktree"),
14927 summary: "did the thing".to_owned(),
14928 stat: "1 file changed".to_owned(),
14929 files: 1,
14930 commits: 1,
14931 empty: false,
14932 failed: None,
14933 verified_noop: None,
14934 duration_ms: 1234,
14935 folded: false,
14936 }];
14937 let run_id = runner.state.id.clone();
14938
14939 let interrupt = Pause::new();
14940 runner.watch_interrupt(interrupt.clone());
14941 interrupt.park_because("task c3d4 asked to run first");
14942 assert!(runner.park_here().expect("park_here"));
14943
14944 let resumed = Runner::resume(&run_id).expect("resume");
14945 assert_eq!(resumed.state.candidates.len(), 1);
14946 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14947 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14948 assert_eq!(resumed.state.status, runner.state.status);
14949 assert!(
14950 resumed.state.parked,
14951 "still parked until `execute` actually walks the graph again"
14952 );
14953 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14954 }
14955
14956 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14958 let mut q = ask::Question::new(
14959 run.to_owned(),
14960 "implement".to_owned(),
14961 "impl-A".to_owned(),
14962 "Which storage backend should the cache use?".to_owned(),
14963 String::new(),
14964 vec!["SQLite".to_owned(), "Redis".to_owned()],
14965 );
14966 store.put(&mut q).unwrap();
14967 q
14968 }
14969
14970 #[test]
14971 fn a_failed_runs_open_question_is_abandoned() {
14972 ask_test_home();
14973 let store = ask::Questions::open();
14974 let mut runner = runner_at(RunStatus::Failed);
14975 let run = runner.state.id.clone();
14976 let q = ask_open_question(&store, &run);
14977
14978 runner.settle_questions();
14979
14980 let back = store.get(&q.id).unwrap();
14981 assert!(
14982 !back.status.open(),
14983 "the seat that asked died with the run; nobody is left to read an answer"
14984 );
14985 assert!(
14986 back.detail.contains(&run) && back.detail.contains("failed"),
14987 "the reason names what the run became, not just that it is gone: {}",
14988 back.detail
14989 );
14990 }
14991
14992 #[test]
14993 fn a_merged_runs_open_question_is_abandoned_too() {
14994 ask_test_home();
14995 let store = ask::Questions::open();
14996 for status in [RunStatus::Merged, RunStatus::Ready] {
14999 let mut runner = runner_at(status);
15000 let run = runner.state.id.clone();
15001 let q = ask_open_question(&store, &run);
15002
15003 runner.settle_questions();
15004
15005 let back = store.get(&q.id).unwrap();
15006 assert!(
15007 !back.status.open(),
15008 "{status:?} run's question must not outlive the run"
15009 );
15010 }
15011 }
15012
15013 #[test]
15014 fn a_still_resumable_runs_open_question_is_left_alone() {
15015 ask_test_home();
15016 let store = ask::Questions::open();
15017 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15023 let mut runner = runner_at(status);
15024 let run = runner.state.id.clone();
15025 let q = ask_open_question(&store, &run);
15026
15027 runner.settle_questions();
15028
15029 let back = store.get(&q.id).unwrap();
15030 assert!(
15031 back.status.open(),
15032 "{status:?} is still alive; the question must still be waiting"
15033 );
15034 }
15035 }
15036
15037 #[test]
15038 fn settle_questions_never_touches_an_already_answered_question() {
15039 ask_test_home();
15040 let store = ask::Questions::open();
15041 let mut runner = runner_at(RunStatus::Failed);
15042 let run = runner.state.id.clone();
15043 let mut q = ask_open_question(&store, &run);
15044 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15045 .unwrap();
15046 store.put(&mut q).unwrap();
15047
15048 runner.settle_questions();
15053 runner.settle_questions();
15054
15055 let back = store.get(&q.id).unwrap();
15056 assert_eq!(
15057 back.status,
15058 ask::QuestionStatus::Answered,
15059 "a real answer is a decision on record, never overwritten by a sweep"
15060 );
15061 }
15062
15063 #[tokio::test]
15074 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15075 crate::run::pin_test_home();
15076 let tmp = tempfile::tempdir().expect("tempdir");
15077 let repo = tmp.path().join("repo");
15078 std::fs::create_dir_all(&repo).unwrap();
15079 init_repo(&repo);
15080
15081 let mut config = Config::default();
15082 config.graph.worktree_root = Some(tmp.path().join("wt"));
15083
15084 let mut state = RunState::new(
15085 repo.clone(),
15086 "main".to_owned(),
15087 "deadbeef".to_owned(),
15088 "task".to_owned(),
15089 config,
15090 );
15091 let root = state.worktree_root();
15092 let wt_a = root.join("cand-A");
15093 let wt_b = root.join("cand-B");
15094 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15095 .await
15096 .expect("worktree A");
15097 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15098 .await
15099 .expect("worktree B");
15100
15101 state.candidates = vec![
15102 Candidate {
15103 index: 0,
15104 label: 'A',
15105 agent: "alpha".to_owned(),
15106 branch: "magi/x/A".to_owned(),
15107 worktree: wt_a.clone(),
15108 summary: String::new(),
15109 stat: String::new(),
15110 files: 0,
15111 commits: 0,
15112 empty: false,
15113 failed: None,
15114 verified_noop: None,
15115 duration_ms: 0,
15116 folded: false,
15117 },
15118 Candidate {
15119 index: 1,
15120 label: 'B',
15121 agent: "beta".to_owned(),
15122 branch: "magi/x/B".to_owned(),
15123 worktree: wt_b.clone(),
15124 summary: String::new(),
15125 stat: String::new(),
15126 files: 0,
15127 commits: 0,
15128 empty: false,
15129 failed: None,
15130 verified_noop: None,
15131 duration_ms: 0,
15132 folded: false,
15133 },
15134 ];
15135 state.tally = Some(Tally {
15136 first_choice: BTreeMap::from([('A', 1)]),
15137 borda: BTreeMap::new(),
15138 winner: 'A',
15139 rankings: 1,
15140 unanimous_initial: true,
15141 deliberated: false,
15142 changed_votes: 0,
15143 unanimous_final: true,
15144 tie_break: None,
15145 judges: 1,
15146 present: 1,
15147 quorum: 1,
15148 met_quorum: true,
15149 uncontested: None,
15150 });
15151 state.status = RunStatus::Ready;
15152
15153 fold_run(&mut state, false, &crate::run::home())
15154 .await
15155 .expect("fold_run");
15156
15157 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15158 assert!(
15159 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15160 "the unmerged winner's branch survives"
15161 );
15162 assert!(
15163 !state.candidates[0].folded,
15164 "the winner is not marked folded"
15165 );
15166
15167 assert!(!wt_b.exists(), "the loser's worktree is removed");
15168 assert!(
15169 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15170 "the loser's branch is removed"
15171 );
15172 assert!(state.candidates[1].folded, "the loser is marked folded");
15173 }
15174
15175 #[tokio::test]
15176 #[cfg(unix)]
15177 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15178 use std::os::fd::AsRawFd as _;
15179 use std::time::Duration;
15180
15181 let tmp = tempfile::tempdir().unwrap();
15182 let repo = tmp.path().join("repo");
15183 std::fs::create_dir_all(&repo).unwrap();
15184 init_repo(&repo);
15185 let common = repo.join(".git");
15186 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15187 let home = tmp.path().join("home");
15188 let mut config = Config::default();
15189 config.coordination = crate::config::test_coordination();
15190 config.graph.worktree_root = Some(tmp.path().join("wt"));
15191 config.graph.timeout_resource_wait = 1;
15192 let mut state = RunState::new(
15193 repo.clone(),
15194 "main".into(),
15195 "deadbeef".into(),
15196 "task".into(),
15197 config,
15198 );
15199 let root = state.worktree_root();
15200 let candidate = root.join("cand-A");
15201 let reviewer = root.join("review-1");
15202 let branch = state.branch_for('A');
15203 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15204 .await
15205 .unwrap();
15206 git::worktree_add_detached(&repo, &reviewer, "main")
15207 .await
15208 .unwrap();
15209 git::acquire_worktree_config(&repo).await.unwrap();
15210 state.enabled_worktree_config = true;
15211 state.status = RunStatus::Ready;
15212 state.candidates.push(Candidate {
15213 index: 0,
15214 label: 'A',
15215 agent: "implement".into(),
15216 branch: branch.clone(),
15217 worktree: candidate.clone(),
15218 summary: String::new(),
15219 stat: String::new(),
15220 files: 0,
15221 commits: 0,
15222 empty: false,
15223 failed: None,
15224 verified_noop: None,
15225 duration_ms: 0,
15226 folded: false,
15227 });
15228 state.save_under(&home).unwrap();
15229 let config_before = std::fs::read(common.join("config")).unwrap();
15230 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15231 .await
15232 .unwrap();
15233 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15236 assert_eq!(
15237 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15238 0
15239 );
15240 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15241 assert!(
15242 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15243 .await
15244 .is_err()
15245 );
15246 assert!(candidate.exists());
15247 assert!(reviewer.exists());
15248 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15249 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15250 assert_eq!(
15251 git::git(&repo, &["worktree", "list", "--porcelain"])
15252 .await
15253 .unwrap(),
15254 worktrees_before
15255 );
15256 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15257 .await
15258 .unwrap()
15259 .unwrap_err();
15260 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15261 drop(fold);
15262 assert!(candidate.exists() && reviewer.exists());
15263 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15264 assert_eq!(
15265 state.resource_block.as_ref().unwrap().kind,
15266 "repository coordination"
15267 );
15268 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15269 drop(external);
15270 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15271 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15272 .await
15273 .unwrap()
15274 .unwrap();
15275 drop(fold);
15276 assert!(removed.contains(&branch));
15277 assert!(!root.exists());
15278 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15279 assert!(state.candidates[0].folded);
15280 assert!(!state.enabled_worktree_config);
15281 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15282 let enabled = git::git_raw(
15283 &repo,
15284 &["config", "--local", "--get", "extensions.worktreeConfig"],
15285 )
15286 .await
15287 .unwrap();
15288 assert!(!enabled.ok(), "the last config reference was released");
15289 }
15290
15291 #[tokio::test]
15294 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15295 let tmp = tempfile::tempdir().expect("tempdir");
15296 let repo = tmp.path().join("repo");
15297 std::fs::create_dir_all(&repo).unwrap();
15298 init_repo(&repo);
15299 let home = tmp.path().join("home");
15300
15301 let mut config = Config::default();
15302 config.graph.worktree_root = Some(tmp.path().join("wt"));
15303 let mut state = RunState::new(
15304 repo.clone(),
15305 "main".to_owned(),
15306 "deadbeef".to_owned(),
15307 "task".to_owned(),
15308 config,
15309 );
15310 git::git(&repo, &["branch", "magi/x/A", "main"])
15312 .await
15313 .expect("branch");
15314 state.candidates = vec![Candidate {
15315 index: 0,
15316 label: 'A',
15317 agent: "alpha".to_owned(),
15318 branch: "magi/x/A".to_owned(),
15319 worktree: state.worktree_root().join("cand-A"),
15320 summary: String::new(),
15321 stat: String::new(),
15322 files: 0,
15323 commits: 0,
15324 empty: false,
15325 failed: None,
15326 verified_noop: None,
15327 duration_ms: 0,
15328 folded: true,
15329 }];
15330 state.released_to = Some("20260901-000000-new1".to_owned());
15331 state.released_branches = vec!["magi/x/A".to_owned()];
15332
15333 fold_run(&mut state, true, &home).await.expect("fold_run");
15334 state.status = RunStatus::Merged;
15335 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15336 state.merge = Some(crate::run::MergeOutcome {
15337 mode: MergeMode::Pr,
15338 ok: true,
15339 detail: "confirmed merged".into(),
15340 empty: false,
15341 });
15342 fold_run(&mut state, true, &home)
15343 .await
15344 .expect("merged fold");
15345
15346 assert!(
15347 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15348 "the handed-over branch survives a fold"
15349 );
15350 }
15351
15352 #[tokio::test]
15356 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15357 let tmp = tempfile::tempdir().expect("tempdir");
15358 let repo = tmp.path().join("repo");
15359 std::fs::create_dir_all(&repo).unwrap();
15360 init_repo(&repo);
15361 let run = |args: &[&str]| {
15362 let out = std::process::Command::new("git")
15363 .quiet()
15364 .args(args)
15365 .current_dir(&repo)
15366 .output()
15367 .expect("spawn git");
15368 assert!(out.status.success(), "git {args:?}");
15369 };
15370 run(&["branch", "magi/x/A"]);
15371 run(&["checkout", "-q", "-b", "magi/x/B"]);
15372 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15373 run(&["add", "-A"]);
15374 run(&["commit", "-q", "-m", "work"]);
15375 run(&["checkout", "-q", "main"]);
15376
15377 let probe = RunState::new(
15381 repo.clone(),
15382 "main".to_owned(),
15383 "deadbeef".to_owned(),
15384 "task".to_owned(),
15385 Config::default(),
15386 );
15387 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15388 let bare = tmp.path().join("origin.git");
15389 let out = std::process::Command::new("git")
15390 .quiet()
15391 .args(["init", "-q", "--bare"])
15392 .arg(&bare)
15393 .output()
15394 .expect("spawn git");
15395 assert!(out.status.success(), "git init --bare");
15396 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15397 run(&["push", "-q", "origin", "main"]);
15398
15399 let mut state = RunState::new(
15400 repo.clone(),
15401 "main".to_owned(),
15402 "deadbeef".to_owned(),
15403 "task".to_owned(),
15404 Config::default(),
15405 );
15406 state.seeds = vec![refs::Seed {
15407 token: "magi/27b2/A".to_owned(),
15408 kind: refs::SeedKind::Unresolved,
15409 sha: String::new(),
15410 branch: true,
15411 detail: "no branch or commit named magi/27b2/A".to_owned(),
15412 }];
15413
15414 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15415 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15416 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15417 let detail = empty_candidate_detail(&state, "main");
15418 assert!(detail.starts_with("empty candidate"), "{detail}");
15419 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15420 assert!(detail.contains("--start-from"), "{detail}");
15421 }
15422
15423 #[tokio::test]
15432 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15433 let tmp = tempfile::tempdir().expect("tempdir");
15434 let repo = tmp.path().join("repo");
15435 std::fs::create_dir_all(&repo).unwrap();
15436 init_repo(&repo);
15437
15438 let mut config = Config::default();
15439 config.merge.mode = MergeMode::Local;
15440
15441 let mut state = RunState::new(
15442 repo.clone(),
15443 "main".to_owned(),
15444 "deadbeef".to_owned(),
15445 "task".to_owned(),
15446 config,
15447 );
15448 state.candidates = vec![Candidate {
15449 index: 0,
15450 label: 'A',
15451 agent: "alpha".to_owned(),
15452 branch: "does-not-exist".to_owned(),
15453 worktree: repo.clone(),
15454 summary: String::new(),
15455 stat: String::new(),
15456 files: 0,
15457 commits: 0,
15458 empty: false,
15459 failed: None,
15460 verified_noop: None,
15461 duration_ms: 0,
15462 folded: false,
15463 }];
15464 state.tally = Some(Tally {
15465 first_choice: BTreeMap::from([('A', 1)]),
15466 borda: BTreeMap::new(),
15467 winner: 'A',
15468 rankings: 1,
15469 unanimous_initial: true,
15470 deliberated: false,
15471 changed_votes: 0,
15472 unanimous_final: true,
15473 tie_break: None,
15474 judges: 0,
15475 present: 0,
15476 quorum: 0,
15477 met_quorum: true,
15478 uncontested: Some("only candidate A produced a change".to_owned()),
15479 });
15480 state.reviews = vec![ReviewRound {
15481 round: 1,
15482 head: "deadbeef".to_owned(),
15483 verified_head: None,
15484 verified_at: None,
15485 reviews: Vec::new(),
15486 e2e: Vec::new(),
15487 fix: None,
15488 blocking: 0,
15489 answered: 0,
15490 expected: 0,
15491 clean: true,
15492 verify_retried: false,
15493 e2e_deferred: false,
15494 e2e_defer_reason: None,
15495 progressed: false,
15496 vote_split: false,
15497 reconsideration: Vec::new(),
15498 verdict: None,
15499 }];
15500 state.gate = vec![CommandOutcome {
15501 command: "test".to_owned(),
15502 code: Some(0),
15503 output_tail: String::new(),
15504 duration_ms: 0,
15505 resource_blocked: false,
15506 }];
15507 state.gate_ran = true;
15508 state.status = RunStatus::Ready;
15513 state.merge = Some(MergeOutcome {
15514 mode: MergeMode::Local,
15515 ok: false,
15516 detail: "already concluded".to_owned(),
15517 empty: false,
15518 });
15519
15520 let mut runner = Runner {
15521 state,
15522 roles: ResolvedRoles {
15523 implementers: Vec::new(),
15524 judges: Vec::new(),
15525 reviewers: Vec::new(),
15526 fixer: None,
15527 conductor: conductor(),
15528 implementer_roster: Vec::new(),
15529 judge_roster: Vec::new(),
15530 reviewer_roster: Vec::new(),
15531 },
15532 sem: Arc::new(Semaphore::new(1)),
15533 pause: Pause::new(),
15534 interrupt: Pause::new(),
15535 };
15536
15537 runner.merge().await.expect("merge");
15538
15539 assert_eq!(
15540 runner.state.status,
15541 RunStatus::Ready,
15542 "a concluded run's status must not change on reentry"
15543 );
15544 assert_eq!(
15545 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15546 Some("already concluded"),
15547 "merge must not run again once the node already recorded an outcome"
15548 );
15549 }
15550
15551 #[tokio::test]
15560 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15561 let tmp = tempfile::tempdir().expect("tempdir");
15562 let repo = tmp.path().join("repo");
15563 std::fs::create_dir_all(&repo).unwrap();
15564 init_repo(&repo);
15565
15566 let mut config = Config::default();
15567 config.merge.mode = MergeMode::Local;
15568
15569 let mut state = RunState::new(
15570 repo.clone(),
15571 "main".to_owned(),
15572 "deadbeef".to_owned(),
15573 "task".to_owned(),
15574 config,
15575 );
15576 state.candidates = vec![Candidate {
15577 index: 0,
15578 label: 'A',
15579 agent: "alpha".to_owned(),
15580 branch: "does-not-exist".to_owned(),
15581 worktree: repo.clone(),
15582 summary: String::new(),
15583 stat: String::new(),
15584 files: 0,
15585 commits: 0,
15586 empty: false,
15587 failed: None,
15588 verified_noop: None,
15589 duration_ms: 0,
15590 folded: false,
15591 }];
15592 state.tally = Some(Tally {
15593 first_choice: BTreeMap::from([('A', 1)]),
15594 borda: BTreeMap::new(),
15595 winner: 'A',
15596 rankings: 1,
15597 unanimous_initial: true,
15598 deliberated: false,
15599 changed_votes: 0,
15600 unanimous_final: true,
15601 tie_break: None,
15602 judges: 0,
15603 present: 0,
15604 quorum: 0,
15605 met_quorum: true,
15606 uncontested: Some("only candidate A produced a change".to_owned()),
15607 });
15608 state.reviews = vec![ReviewRound {
15609 round: 1,
15610 head: "deadbeef".to_owned(),
15611 verified_head: None,
15612 verified_at: None,
15613 reviews: Vec::new(),
15614 e2e: Vec::new(),
15615 fix: None,
15616 blocking: 0,
15617 answered: 0,
15618 expected: 0,
15619 clean: true,
15620 verify_retried: false,
15621 e2e_deferred: false,
15622 e2e_defer_reason: None,
15623 progressed: false,
15624 vote_split: false,
15625 reconsideration: Vec::new(),
15626 verdict: None,
15627 }];
15628 state.gate = Vec::new();
15630 state.gate_ran = false;
15631 state.status = RunStatus::Gating;
15632
15633 let mut runner = Runner {
15634 state,
15635 roles: ResolvedRoles {
15636 implementers: Vec::new(),
15637 judges: Vec::new(),
15638 reviewers: Vec::new(),
15639 fixer: None,
15640 conductor: conductor(),
15641 implementer_roster: Vec::new(),
15642 judge_roster: Vec::new(),
15643 reviewer_roster: Vec::new(),
15644 },
15645 sem: Arc::new(Semaphore::new(1)),
15646 pause: Pause::new(),
15647 interrupt: Pause::new(),
15648 };
15649
15650 runner.merge().await.expect("merge");
15651
15652 assert!(
15653 runner.state.merge.is_none(),
15654 "an empty gate must never be read as a passing one: {:?}",
15655 runner.state.merge
15656 );
15657 }
15658
15659 #[tokio::test]
15666 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15667 let tmp = tempfile::tempdir().expect("tempdir");
15668 let repo = tmp.path().join("repo");
15669 std::fs::create_dir_all(&repo).unwrap();
15670 init_repo(&repo);
15671
15672 let config = Config::default();
15674
15675 let mut state = RunState::new(
15676 repo.clone(),
15677 "main".to_owned(),
15678 "deadbeef".to_owned(),
15679 "task".to_owned(),
15680 config,
15681 );
15682 state.candidates = vec![Candidate {
15683 index: 0,
15684 label: 'A',
15685 agent: "alpha".to_owned(),
15686 branch: "does-not-exist".to_owned(),
15687 worktree: repo.clone(),
15688 summary: String::new(),
15689 stat: String::new(),
15690 files: 0,
15691 commits: 0,
15692 empty: false,
15693 failed: None,
15694 verified_noop: None,
15695 duration_ms: 0,
15696 folded: false,
15697 }];
15698 state.tally = Some(Tally {
15699 first_choice: BTreeMap::from([('A', 1)]),
15700 borda: BTreeMap::new(),
15701 winner: 'A',
15702 rankings: 1,
15703 unanimous_initial: true,
15704 deliberated: false,
15705 changed_votes: 0,
15706 unanimous_final: true,
15707 tie_break: None,
15708 judges: 0,
15709 present: 0,
15710 quorum: 0,
15711 met_quorum: true,
15712 uncontested: Some("only candidate A produced a change".to_owned()),
15713 });
15714 state.reviews = vec![ReviewRound {
15715 round: 1,
15716 head: "deadbeef".to_owned(),
15717 verified_head: None,
15718 verified_at: None,
15719 reviews: Vec::new(),
15720 e2e: Vec::new(),
15721 fix: None,
15722 blocking: 0,
15723 answered: 0,
15724 expected: 0,
15725 clean: true,
15726 verify_retried: false,
15727 e2e_deferred: false,
15728 e2e_defer_reason: None,
15729 progressed: false,
15730 vote_split: false,
15731 reconsideration: Vec::new(),
15732 verdict: None,
15733 }];
15734
15735 let mut runner = Runner {
15736 state,
15737 roles: ResolvedRoles {
15738 implementers: Vec::new(),
15739 judges: Vec::new(),
15740 reviewers: Vec::new(),
15741 fixer: None,
15742 conductor: conductor(),
15743 implementer_roster: Vec::new(),
15744 judge_roster: Vec::new(),
15745 reviewer_roster: Vec::new(),
15746 },
15747 sem: Arc::new(Semaphore::new(1)),
15748 pause: Pause::new(),
15749 interrupt: Pause::new(),
15750 };
15751
15752 runner.gate().await.expect("gate");
15753 assert!(
15754 runner.state.gate_ran,
15755 "zero configured commands is still a real attempt, not an unrun gate"
15756 );
15757 assert!(runner.state.gate.is_empty());
15758 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15759 assert_ne!(
15760 runner.state.status,
15761 RunStatus::Blocked,
15762 "a gate with nothing to check must not read as failed"
15763 );
15764
15765 runner.merge().await.expect("merge");
15766 assert_eq!(
15767 runner.state.status,
15768 RunStatus::Ready,
15769 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15770 );
15771 }
15772
15773 #[tokio::test]
15784 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15785 crate::run::pin_test_home();
15786 let home = crate::run::home();
15787
15788 let tmp = tempfile::tempdir().expect("tempdir");
15789 let repo = tmp.path().join("repo");
15790 std::fs::create_dir_all(&repo).unwrap();
15791 init_repo(&repo);
15792 let cache_dir = tmp.path().join("target");
15795
15796 let mut config = Config::default();
15797 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15798 config.graph.timeout_verify = Some(2);
15801 config.graph.timeout_resource_wait = 2;
15802
15803 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15804 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15805 .expect("no io error acquiring directly")
15806 {
15807 crate::cache::AcquireOutcome::Acquired(g) => g,
15808 crate::cache::AcquireOutcome::Busy(b) => {
15809 panic!("expected the direct acquire to win the lease first: {b:?}")
15810 }
15811 };
15812
15813 let mut state = RunState::new(
15814 repo.clone(),
15815 "main".to_owned(),
15816 "deadbeef".to_owned(),
15817 "task".to_owned(),
15818 config,
15819 );
15820 state.candidates = vec![Candidate {
15821 index: 0,
15822 label: 'A',
15823 agent: "alpha".to_owned(),
15824 branch: "does-not-exist".to_owned(),
15825 worktree: repo.clone(),
15826 summary: String::new(),
15827 stat: String::new(),
15828 files: 0,
15829 commits: 0,
15830 empty: false,
15831 failed: None,
15832 verified_noop: None,
15833 duration_ms: 0,
15834 folded: false,
15835 }];
15836 state.tally = Some(Tally {
15837 first_choice: BTreeMap::from([('A', 1)]),
15838 borda: BTreeMap::new(),
15839 winner: 'A',
15840 rankings: 1,
15841 unanimous_initial: true,
15842 deliberated: false,
15843 changed_votes: 0,
15844 unanimous_final: true,
15845 tie_break: None,
15846 judges: 0,
15847 present: 0,
15848 quorum: 0,
15849 met_quorum: true,
15850 uncontested: Some("only candidate A produced a change".to_owned()),
15851 });
15852 state.reviews = vec![ReviewRound {
15853 round: 1,
15854 head: "deadbeef".to_owned(),
15855 verified_head: None,
15856 verified_at: None,
15857 reviews: Vec::new(),
15858 e2e: Vec::new(),
15859 fix: None,
15860 blocking: 0,
15861 answered: 0,
15862 expected: 0,
15863 clean: true,
15864 verify_retried: false,
15865 e2e_deferred: false,
15866 e2e_defer_reason: None,
15867 progressed: false,
15868 vote_split: false,
15869 reconsideration: Vec::new(),
15870 verdict: None,
15871 }];
15872
15873 let mut runner = Runner {
15874 state,
15875 roles: ResolvedRoles {
15876 implementers: Vec::new(),
15877 judges: Vec::new(),
15878 reviewers: Vec::new(),
15879 fixer: None,
15880 conductor: conductor(),
15881 implementer_roster: Vec::new(),
15882 judge_roster: Vec::new(),
15883 reviewer_roster: Vec::new(),
15884 },
15885 sem: Arc::new(Semaphore::new(1)),
15886 pause: Pause::new(),
15887 interrupt: Pause::new(),
15888 };
15889
15890 let started = std::time::Instant::now();
15891 runner.gate().await.expect("gate");
15892 assert!(
15893 started.elapsed() < Duration::from_secs(1),
15894 "a gate with nothing to run must never wait on a lease it never needed"
15895 );
15896 assert!(
15897 runner.state.gate_ran,
15898 "zero commands is still a real, immediate attempt"
15899 );
15900 assert!(runner.state.gate.is_empty());
15901 assert_ne!(
15902 runner.state.status,
15903 RunStatus::Blocked,
15904 "must not read as resource-blocked on a lease it never asked for"
15905 );
15906 }
15907
15908 #[cfg(unix)]
15909 #[tokio::test]
15910 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15911 crate::run::pin_test_home();
15912 let tmp = tempfile::tempdir().unwrap();
15913 let mut config = Config::default();
15914 config.disk.min_free_bytes = 0;
15915 let mut state = RunState::new(
15916 tmp.path().to_owned(),
15917 "main".into(),
15918 "base".into(),
15919 "verification evidence".into(),
15920 config,
15921 );
15922 let (outcomes, pids) = run_commands(
15923 &mut state,
15924 "gate",
15925 "gate",
15926 1,
15927 &["sh".into(), "-c".into()],
15928 &[
15929 "printf before-timeout; exec sleep 30".into(),
15930 "kill -TERM $$".into(),
15931 "exit 7".into(),
15932 "printf success".into(),
15933 ],
15934 tmp.path(),
15935 Duration::from_secs(1),
15936 )
15937 .await;
15938 wait_for_timed_out_children_to_die(&pids).await;
15939 assert_eq!(
15940 pids.len(),
15941 1,
15942 "unleased timeout retains its death-check PID"
15943 );
15944 assert_eq!(outcomes.len(), 4);
15945 assert_eq!(outcomes[0].code, None);
15946 assert!(outcomes[0].output_tail.contains("before-timeout"));
15947 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15948 assert_eq!(outcomes[1].code, None);
15949 assert!(outcomes[1].output_tail.contains("command terminated:"));
15950 assert!(!outcomes[1].output_tail.contains("timed out"));
15951 assert_eq!(outcomes[2].code, Some(7));
15952 assert!(!outcomes[2].output_tail.contains("timed out"));
15953 assert_eq!(outcomes[3].code, Some(0));
15954 assert_eq!(outcomes[3].output_tail, "success");
15955 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15956 }
15957
15958 #[tokio::test]
15968 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15969 crate::run::pin_test_home();
15970
15971 let tmp = tempfile::tempdir().expect("tempdir");
15972 let repo = tmp.path().join("repo");
15973 std::fs::create_dir_all(&repo).unwrap();
15974 init_repo(&repo);
15975
15976 let mut config = Config::default();
15977 config.verify.gate = vec![
15978 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15979 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15980 .to_owned(),
15981 ];
15982
15983 let mut state = RunState::new(
15984 repo.clone(),
15985 "main".to_owned(),
15986 "deadbeef".to_owned(),
15987 "task".to_owned(),
15988 config,
15989 );
15990 let run_id = state.id.clone();
15991 state.candidates = vec![Candidate {
15992 index: 0,
15993 label: 'A',
15994 agent: "alpha".to_owned(),
15995 branch: "does-not-exist".to_owned(),
15996 worktree: repo.clone(),
15997 summary: String::new(),
15998 stat: String::new(),
15999 files: 0,
16000 commits: 0,
16001 empty: false,
16002 failed: None,
16003 verified_noop: None,
16004 duration_ms: 0,
16005 folded: false,
16006 }];
16007 state.tally = Some(Tally {
16008 first_choice: BTreeMap::from([('A', 1)]),
16009 borda: BTreeMap::new(),
16010 winner: 'A',
16011 rankings: 1,
16012 unanimous_initial: true,
16013 deliberated: false,
16014 changed_votes: 0,
16015 unanimous_final: true,
16016 tie_break: None,
16017 judges: 0,
16018 present: 0,
16019 quorum: 0,
16020 met_quorum: true,
16021 uncontested: Some("only candidate A produced a change".to_owned()),
16022 });
16023 state.reviews = vec![ReviewRound {
16024 round: 1,
16025 head: "deadbeef".to_owned(),
16026 verified_head: None,
16027 verified_at: None,
16028 reviews: Vec::new(),
16029 e2e: Vec::new(),
16030 fix: None,
16031 blocking: 0,
16032 answered: 0,
16033 expected: 0,
16034 clean: true,
16035 verify_retried: false,
16036 e2e_deferred: false,
16037 e2e_defer_reason: None,
16038 progressed: false,
16039 vote_split: false,
16040 reconsideration: Vec::new(),
16041 verdict: None,
16042 }];
16043
16044 let mut runner = Runner {
16045 state,
16046 roles: ResolvedRoles {
16047 implementers: Vec::new(),
16048 judges: Vec::new(),
16049 reviewers: Vec::new(),
16050 fixer: None,
16051 conductor: conductor(),
16052 implementer_roster: Vec::new(),
16053 judge_roster: Vec::new(),
16054 reviewer_roster: Vec::new(),
16055 },
16056 sem: Arc::new(Semaphore::new(1)),
16057 pause: Pause::new(),
16058 interrupt: Pause::new(),
16059 };
16060
16061 let started_marker = repo.join("started.marker");
16062 let release_marker = repo.join("release.marker");
16063 let poller = tokio::spawn(async move {
16064 for _ in 0..100 {
16069 if started_marker.exists()
16070 && let Ok(s) = crate::run::RunState::load(&run_id)
16071 && let Some(a) = s.active.get("gate")
16072 {
16073 std::fs::write(&release_marker, b"go").expect("release marker");
16074 return Some(a.clone());
16075 }
16076 tokio::time::sleep(Duration::from_millis(50)).await;
16077 }
16078 None
16079 });
16080
16081 runner.gate().await.expect("gate");
16082 let captured = poller.await.expect("poller task");
16083 let captured = captured.expect(
16084 "the poller never saw a `gate` task entry in run.json while the command was \
16085 still blocked on its own release marker",
16086 );
16087
16088 assert_eq!(captured.task.as_deref(), Some("gate"));
16089 assert_eq!(captured.node, "gate");
16090 assert_eq!(captured.index, Some(1));
16091 assert_eq!(captured.total, Some(1));
16092 assert!(
16093 captured
16094 .command
16095 .as_deref()
16096 .is_some_and(|c| c.contains("started.marker")),
16097 "{captured:?}"
16098 );
16099
16100 assert!(
16101 runner.state.active.is_empty(),
16102 "the entry must be cleared once the command actually finished: {:?}",
16103 runner.state.active
16104 );
16105 assert!(runner.state.gate_ran);
16106 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16107 }
16108
16109 #[tokio::test]
16112 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16113 use crate::verdict::{Finding, ReviewVote, Severity};
16114 crate::run::pin_test_home();
16115 let tmp = tempfile::tempdir().expect("tempdir");
16116 let repo = tmp.path().join("repo");
16117 std::fs::create_dir_all(&repo).unwrap();
16118 init_repo(&repo);
16119
16120 for (severity, vote, expect) in [
16121 (Severity::Major, ReviewVote::Reject, true),
16122 (Severity::Minor, ReviewVote::Reject, false),
16123 (Severity::Major, ReviewVote::Approve, false),
16124 ] {
16125 let mut round = review_round(false, 1, 1, 1, false, true);
16126 round.reviews = vec![ReviewRecord {
16127 reviewer: 1,
16128 agent: "alpha".to_owned(),
16129 summary: String::new(),
16130 findings: vec![Finding {
16131 id: "R1-1-1".to_owned(),
16132 severity,
16133 file: None,
16134 line: None,
16135 title: "t".to_owned(),
16136 detail: String::new(),
16137 }],
16138 vote: Some(vote),
16139 failed: None,
16140 duration_ms: 0,
16141 attempts: 0,
16142 }];
16143 let mut state = RunState::new(
16144 repo.clone(),
16145 "main".to_owned(),
16146 "deadbeef".to_owned(),
16147 "task".to_owned(),
16148 Config::default(),
16149 );
16150 state.reviews = vec![round];
16151 let mut runner = Runner {
16152 state,
16153 roles: ResolvedRoles {
16154 implementers: Vec::new(),
16155 judges: Vec::new(),
16156 reviewers: Vec::new(),
16157 fixer: None,
16158 conductor: conductor(),
16159 implementer_roster: Vec::new(),
16160 judge_roster: Vec::new(),
16161 reviewer_roster: Vec::new(),
16162 },
16163 sem: Arc::new(Semaphore::new(1)),
16164 pause: Pause::new(),
16165 interrupt: Pause::new(),
16166 };
16167 let shell = runner.state.config.shell();
16168 runner
16169 .stop_reviewing("round budget spent", &shell, &repo)
16170 .await
16171 .expect("stop_reviewing");
16172 assert_eq!(runner.state.status, RunStatus::Gating);
16173 assert_eq!(
16174 runner.state.contested_handoff.is_some(),
16175 expect,
16176 "{severity:?} + {vote:?}"
16177 );
16178 }
16179 }
16180
16181 #[tokio::test]
16194 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16195 crate::run::pin_test_home();
16196 let home = crate::run::home();
16197
16198 let tmp = tempfile::tempdir().expect("tempdir");
16199 let repo = tmp.path().join("repo");
16200 std::fs::create_dir_all(&repo).unwrap();
16201 init_repo(&repo);
16202 let head = crate::git::rev_parse(&repo, "HEAD")
16203 .await
16204 .expect("rev-parse");
16205 let cache_dir = tmp.path().join("target");
16208
16209 let mut config = Config::default();
16210 config.verify.e2e = vec![format!(
16211 "CARGO_TARGET_DIR='{}' test -f README.md",
16212 cache_dir.display()
16213 )];
16214 config.graph.review_rounds = 1;
16215 config.graph.timeout_verify = Some(2);
16218 config.graph.timeout_resource_wait = 2;
16219
16220 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16221 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16222 .expect("no io error acquiring directly")
16223 {
16224 crate::cache::AcquireOutcome::Acquired(g) => g,
16225 crate::cache::AcquireOutcome::Busy(b) => {
16226 panic!("expected the direct acquire to win the lease first: {b:?}")
16227 }
16228 };
16229
16230 let mut state = RunState::new(
16231 repo.clone(),
16232 "main".to_owned(),
16233 head.clone(),
16234 "task".to_owned(),
16235 config,
16236 );
16237 state.candidates = vec![Candidate {
16238 index: 0,
16239 label: 'A',
16240 agent: "alpha".to_owned(),
16241 branch: "does-not-exist".to_owned(),
16242 worktree: repo.clone(),
16243 summary: String::new(),
16244 stat: String::new(),
16245 files: 0,
16246 commits: 0,
16247 empty: false,
16248 failed: None,
16249 verified_noop: None,
16250 duration_ms: 0,
16251 folded: false,
16252 }];
16253 state.tally = Some(Tally {
16254 first_choice: BTreeMap::from([('A', 1)]),
16255 borda: BTreeMap::new(),
16256 winner: 'A',
16257 rankings: 1,
16258 unanimous_initial: true,
16259 deliberated: false,
16260 changed_votes: 0,
16261 unanimous_final: true,
16262 tie_break: None,
16263 judges: 0,
16264 present: 0,
16265 quorum: 0,
16266 met_quorum: true,
16267 uncontested: Some("only candidate A produced a change".to_owned()),
16268 });
16269 state.reviews = vec![ReviewRound {
16273 round: 1,
16274 head: head.clone(),
16275 verified_head: None,
16276 verified_at: None,
16277 reviews: Vec::new(),
16278 e2e: Vec::new(),
16279 fix: None,
16280 blocking: 1,
16281 answered: 1,
16282 expected: 1,
16283 clean: false,
16284 verify_retried: false,
16285 e2e_deferred: true,
16286 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16287 progressed: false,
16288 vote_split: false,
16289 reconsideration: Vec::new(),
16290 verdict: None,
16291 }];
16292
16293 let mut runner = Runner {
16294 state,
16295 roles: ResolvedRoles {
16296 implementers: Vec::new(),
16297 judges: Vec::new(),
16298 reviewers: Vec::new(),
16299 fixer: None,
16300 conductor: conductor(),
16301 implementer_roster: Vec::new(),
16302 judge_roster: Vec::new(),
16303 reviewer_roster: Vec::new(),
16304 },
16305 sem: Arc::new(Semaphore::new(1)),
16306 pause: Pause::new(),
16307 interrupt: Pause::new(),
16308 };
16309
16310 let shell = runner.state.config.shell();
16311 runner
16312 .stop_reviewing("round budget spent", &shell, &repo)
16313 .await
16314 .expect("stop_reviewing");
16315
16316 let last = runner.state.reviews.last().expect("round record");
16317 assert_eq!(
16318 last.e2e_status(),
16319 E2eStatus::ResourceBlocked,
16320 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16321 failed: {last:?}"
16322 );
16323 assert_eq!(
16324 last.verified_head.as_deref(),
16325 Some(head.as_str()),
16326 "which commit this attempt targeted is known even though nothing finished checking \
16327 it"
16328 );
16329 let first_attempt_at = last
16330 .verified_at
16331 .expect("when this attempt ran is known too");
16332 assert_ne!(
16333 runner.state.status,
16334 RunStatus::Blocked,
16335 "contention is evidence about the machine, not the patch — it must not settle the \
16336 run as blocked: {:?}",
16337 runner.state.status
16338 );
16339 assert!(
16340 !runner
16341 .state
16342 .events
16343 .iter()
16344 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16345 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16346 runner.state.events
16347 );
16348
16349 runner
16354 .stop_reviewing("round budget spent", &shell, &repo)
16355 .await
16356 .expect("stop_reviewing retry");
16357 assert_eq!(
16358 runner.state.reviews.len(),
16359 1,
16360 "no new round was started: {:?}",
16361 runner.state.reviews
16362 );
16363 let last = runner.state.reviews.last().expect("round record");
16364 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16365 assert!(
16366 last.verified_at.expect("still known") > first_attempt_at,
16367 "a second reentry must be a fresh attempt, not a stale copy of the first"
16368 );
16369 assert_ne!(runner.state.status, RunStatus::Blocked);
16370
16371 held.release();
16372 }
16373
16374 #[tokio::test]
16386 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16387 crate::run::pin_test_home();
16388 let home = crate::run::home();
16389
16390 let tmp = tempfile::tempdir().expect("tempdir");
16391 let repo = tmp.path().join("repo");
16392 std::fs::create_dir_all(&repo).unwrap();
16393 init_repo(&repo);
16394 let head = crate::git::rev_parse(&repo, "HEAD")
16395 .await
16396 .expect("rev-parse");
16397 let cache_dir = tmp.path().join("target");
16398
16399 let mut config = Config::default();
16400 config.verify.e2e = vec![format!(
16401 "CARGO_TARGET_DIR='{}' test -f README.md",
16402 cache_dir.display()
16403 )];
16404 config.graph.review_rounds = 1;
16405 config.graph.timeout_verify = Some(2);
16406 config.graph.timeout_resource_wait = 2;
16407
16408 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16409 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16410 .expect("no io error acquiring directly")
16411 {
16412 crate::cache::AcquireOutcome::Acquired(g) => g,
16413 crate::cache::AcquireOutcome::Busy(b) => {
16414 panic!("expected the direct acquire to win the lease first: {b:?}")
16415 }
16416 };
16417
16418 let mut state = RunState::new(
16419 repo.clone(),
16420 "main".to_owned(),
16421 head.clone(),
16422 "task".to_owned(),
16423 config,
16424 );
16425 state.candidates = vec![Candidate {
16426 index: 0,
16427 label: 'A',
16428 agent: "alpha".to_owned(),
16429 branch: "does-not-exist".to_owned(),
16430 worktree: repo.clone(),
16431 summary: String::new(),
16432 stat: String::new(),
16433 files: 0,
16434 commits: 0,
16435 empty: false,
16436 failed: None,
16437 verified_noop: None,
16438 duration_ms: 0,
16439 folded: false,
16440 }];
16441 state.tally = Some(Tally {
16442 first_choice: BTreeMap::from([('A', 1)]),
16443 borda: BTreeMap::new(),
16444 winner: 'A',
16445 rankings: 1,
16446 unanimous_initial: true,
16447 deliberated: false,
16448 changed_votes: 0,
16449 unanimous_final: true,
16450 tie_break: None,
16451 judges: 0,
16452 present: 0,
16453 quorum: 0,
16454 met_quorum: true,
16455 uncontested: Some("only candidate A produced a change".to_owned()),
16456 });
16457 state.reviews = vec![ReviewRound {
16461 round: 1,
16462 head: head.clone(),
16463 verified_head: Some(head.clone()),
16464 verified_at: Some(jiff::Timestamp::now()),
16465 reviews: Vec::new(),
16466 e2e: vec![CommandOutcome {
16467 command: format!(
16468 "CARGO_TARGET_DIR='{}' test -f README.md",
16469 cache_dir.display()
16470 ),
16471 code: None,
16472 output_tail: "waiting for the shared build cache".to_owned(),
16473 duration_ms: 0,
16474 resource_blocked: true,
16475 }],
16476 fix: None,
16477 blocking: 1,
16478 answered: 1,
16479 expected: 1,
16480 clean: false,
16481 verify_retried: false,
16482 e2e_deferred: false,
16483 e2e_defer_reason: None,
16484 progressed: false,
16485 vote_split: false,
16486 reconsideration: Vec::new(),
16487 verdict: None,
16488 }];
16489
16490 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16491 let mut runner = Runner {
16492 state,
16493 roles: ResolvedRoles {
16494 implementers: Vec::new(),
16495 judges: Vec::new(),
16496 reviewers: Vec::new(),
16497 fixer: None,
16498 conductor: conductor(),
16499 implementer_roster: Vec::new(),
16500 judge_roster: Vec::new(),
16501 reviewer_roster: Vec::new(),
16502 },
16503 sem: Arc::new(Semaphore::new(1)),
16504 pause: Pause::new(),
16505 interrupt: Pause::new(),
16506 };
16507
16508 runner.review_loop().await.expect("review_loop");
16513
16514 assert_eq!(
16515 runner.state.reviews.len(),
16516 1,
16517 "no new round was started on top of the unresolved one: {:?}",
16518 runner.state.reviews
16519 );
16520 let last = &runner.state.reviews[0];
16521 assert_eq!(
16522 last.e2e_status(),
16523 E2eStatus::ResourceBlocked,
16524 "still contended: {last:?}"
16525 );
16526 assert!(
16527 last.verified_at.expect("still known") > first_attempt_at,
16528 "review_loop must have actually retried the check, not left it exactly as found"
16529 );
16530 assert_ne!(
16531 runner.state.status,
16532 RunStatus::Blocked,
16533 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16534 runner.state.status
16535 );
16536
16537 held.release();
16538 }
16539
16540 #[tokio::test]
16541 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16542 crate::run::pin_test_home();
16543 for status in [RunStatus::Blocked, RunStatus::Gating] {
16544 let mut runner = runner_at(status);
16545 runner.state.config.graph.land = true;
16546 runner.state.gate_ran = true;
16547 runner.state.merge = Some(MergeOutcome {
16548 mode: MergeMode::Pr,
16549 ok: false,
16550 detail: "the fixer produced no commit".into(),
16551 empty: false,
16552 });
16553 let url = "https://example.invalid/o/r/pull/1";
16554 runner.state.pr = Some(crate::run::PrRecord {
16555 url: url.into(),
16556 number: 1,
16557 state: "open".into(),
16558 checks: "red".into(),
16559 round: 1,
16560 rounds: 1,
16561 red_at_merge: Vec::new(),
16562 });
16563 runner.state.save().unwrap();
16564 runner.execute_graph().await.unwrap();
16568 assert_eq!(runner.state.status, RunStatus::Blocked);
16569 assert!(runner.state.land_sync_pending);
16570 assert!(runner.state.events.iter().any(|event|
16571 event.node == "land" && event.message == format!("watching {url}")));
16572 assert_eq!(
16573 runner.state.merge.as_ref().unwrap().detail,
16574 "the fixer produced no commit"
16575 );
16576 }
16577 }
16578
16579 #[tokio::test]
16580 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16581 crate::run::pin_test_home();
16582 let tmp = tempfile::tempdir().expect("tempdir");
16583 let repo = tmp.path().join("repo");
16584 std::fs::create_dir_all(&repo).unwrap();
16585 init_repo(&repo);
16586
16587 let mut config = Config::default();
16588 config.merge.mode = MergeMode::Pr;
16589 config.graph.land = true;
16590 config.graph.land_approval = false;
16591
16592 let mut state = RunState::new(
16593 repo.clone(),
16594 "main".to_owned(),
16595 "deadbeef".to_owned(),
16596 "task".to_owned(),
16597 config,
16598 );
16599 state.candidates = vec![Candidate {
16600 index: 0,
16601 label: 'A',
16602 agent: "alpha".to_owned(),
16603 branch: "does-not-exist".to_owned(),
16604 worktree: repo.clone(),
16605 summary: String::new(),
16606 stat: String::new(),
16607 files: 0,
16608 commits: 0,
16609 empty: false,
16610 failed: None,
16611 verified_noop: None,
16612 duration_ms: 0,
16613 folded: false,
16614 }];
16615 state.tally = Some(Tally {
16616 first_choice: BTreeMap::from([('A', 1)]),
16617 borda: BTreeMap::new(),
16618 winner: 'A',
16619 rankings: 1,
16620 unanimous_initial: true,
16621 deliberated: false,
16622 changed_votes: 0,
16623 unanimous_final: true,
16624 tie_break: None,
16625 judges: 0,
16626 present: 0,
16627 quorum: 0,
16628 met_quorum: true,
16629 uncontested: Some("only candidate A produced a change".to_owned()),
16630 });
16631 state.reviews = vec![ReviewRound {
16632 round: 1,
16633 head: "deadbeef".to_owned(),
16634 verified_head: None,
16635 verified_at: None,
16636 reviews: Vec::new(),
16637 e2e: Vec::new(),
16638 fix: None,
16639 blocking: 0,
16640 answered: 0,
16641 expected: 0,
16642 clean: true,
16643 verify_retried: false,
16644 e2e_deferred: false,
16645 e2e_defer_reason: None,
16646 progressed: false,
16647 vote_split: false,
16648 reconsideration: Vec::new(),
16649 verdict: None,
16650 }];
16651 state.gate = vec![CommandOutcome {
16652 command: "test".to_owned(),
16653 code: Some(0),
16654 output_tail: String::new(),
16655 duration_ms: 0,
16656 resource_blocked: false,
16657 }];
16658 state.gate_ran = true;
16659 state.status = RunStatus::Landing;
16663 state.merge = Some(MergeOutcome {
16664 mode: MergeMode::Pr,
16665 ok: true,
16666 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16667 empty: false,
16668 });
16669
16670 ask_test_home();
16674 state.save().expect("save landing run");
16677 let store = ask::Questions::open();
16678 let q = ask_open_question(&store, &state.id);
16679
16680 let mut runner = Runner {
16681 state,
16682 roles: ResolvedRoles {
16683 implementers: Vec::new(),
16684 judges: Vec::new(),
16685 reviewers: Vec::new(),
16686 fixer: None,
16687 conductor: conductor(),
16688 implementer_roster: Vec::new(),
16689 judge_roster: Vec::new(),
16690 reviewer_roster: Vec::new(),
16691 },
16692 sem: Arc::new(Semaphore::new(1)),
16693 pause: Pause::new(),
16694 interrupt: Pause::new(),
16695 };
16696
16697 runner.execute().await.expect("execute");
16702
16703 assert_eq!(
16704 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16705 Some("https://example.invalid/x/y/pull/1"),
16706 "reentry must not push again or open a second pull request over the \
16707 one `land` is already watching"
16708 );
16709 assert_ne!(
16710 runner.state.status,
16711 RunStatus::Landing,
16712 "land could not actually reach the fake pull request, so it must \
16713 have given up rather than left the run silently parked forever"
16714 );
16715 assert_eq!(runner.state.status, RunStatus::Blocked);
16719 assert!(
16720 store.get(&q.id).unwrap().status.open(),
16721 "Blocked is still alive; settle_questions must have been a no-op here"
16722 );
16723 }
16724
16725 fn state_with_round(round: ReviewRound) -> RunState {
16726 let mut s = RunState::new(
16727 PathBuf::from("/repo"),
16728 "main".to_owned(),
16729 "abc1234".to_owned(),
16730 "add retries".to_owned(),
16731 Config::default(),
16732 );
16733 s.reviews = vec![round];
16734 s
16735 }
16736
16737 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16738 crate::verdict::Finding {
16739 id: id.to_owned(),
16740 severity,
16741 file: None,
16742 line: None,
16743 title: title.to_owned(),
16744 detail: String::new(),
16745 }
16746 }
16747
16748 #[test]
16749 fn pr_body_names_open_findings_and_declined_ones() {
16750 let round = ReviewRound {
16751 round: 2,
16752 head: "deadbee".to_owned(),
16753 verified_head: None,
16754 verified_at: None,
16755 reviews: vec![ReviewRecord {
16756 attempts: 0,
16757 reviewer: 1,
16758 agent: "alpha".to_owned(),
16759 summary: String::new(),
16760 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16761 vote: None,
16762 failed: None,
16763 duration_ms: 0,
16764 }],
16765 e2e: vec![CommandOutcome {
16766 command: "cargo test".to_owned(),
16767 code: Some(0),
16768 output_tail: String::new(),
16769 duration_ms: 0,
16770 resource_blocked: false,
16771 }],
16772 verify_retried: false,
16773 e2e_deferred: false,
16774 e2e_defer_reason: None,
16775 fix: Some(FixRecord {
16776 agent: "alpha".to_owned(),
16777 addressed: Vec::new(),
16778 rejected: vec![crate::verdict::Rejection {
16779 id: "R1-1-1".to_owned(),
16780 why: "not reachable from any caller".to_owned(),
16781 }],
16782 notes: String::new(),
16783 committed: true,
16784 failed: None,
16785 duration_ms: 0,
16786 continuation: None,
16787 }),
16788 blocking: 0,
16789 answered: 1,
16790 expected: 1,
16791 clean: false,
16792 progressed: true,
16793 vote_split: false,
16794 reconsideration: Vec::new(),
16795 verdict: None,
16796 };
16797 let state = state_with_round(round);
16798 let body = pr_message(&state, 'A').body;
16799
16800 assert!(body.contains("add retries"), "the task must still be there");
16801 assert!(body.contains("R2-1-1"), "{body}");
16802 assert!(!body.contains("unused import"), "{body}");
16803 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16804 assert!(
16805 !body.contains("not reachable from any caller"),
16806 "the reason it was declined: {body}"
16807 );
16808 }
16809
16810 #[test]
16811 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16812 let round = ReviewRound {
16813 round: 1,
16814 head: "deadbee".to_owned(),
16815 verified_head: None,
16816 verified_at: None,
16817 reviews: vec![ReviewRecord {
16818 attempts: 0,
16819 reviewer: 1,
16820 agent: "alpha".to_owned(),
16821 summary: String::new(),
16822 findings: Vec::new(),
16823 vote: None,
16824 failed: None,
16825 duration_ms: 0,
16826 }],
16827 e2e: Vec::new(),
16828 verify_retried: false,
16829 e2e_deferred: false,
16830 e2e_defer_reason: None,
16831 fix: None,
16832 blocking: 0,
16833 answered: 1,
16834 expected: 1,
16835 clean: true,
16836 progressed: false,
16837 vote_split: false,
16838 reconsideration: Vec::new(),
16839 verdict: None,
16840 };
16841 let state = state_with_round(round);
16842 let body = pr_message(&state, 'A').body;
16843 assert!(!body.contains("Open review findings"), "{body}");
16844 assert!(!body.contains("Declined"), "{body}");
16845 }
16846
16847 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16848 let mut state = RunState::new(
16849 PathBuf::from("/repo"),
16850 "main".to_owned(),
16851 "abc1234".to_owned(),
16852 instruction.to_owned(),
16853 Config::default(),
16854 );
16855 state.candidates.push(Candidate {
16856 index: 0,
16857 label: 'A',
16858 agent: "alpha".to_owned(),
16859 branch: "magi/x/A".to_owned(),
16860 worktree: PathBuf::from("/wt"),
16861 summary: summary.to_owned(),
16862 stat: String::new(),
16863 files: 1,
16864 commits: 1,
16865 empty: false,
16866 failed: None,
16867 verified_noop: None,
16868 folded: false,
16869 duration_ms: 0,
16870 });
16871 state
16872 }
16873
16874 fn review_state(subjects: &[&str]) -> RunState {
16875 let mut state = state_with_summary(
16876 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16877 "",
16878 );
16879 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16880 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16881 state
16882 }
16883
16884 #[tokio::test]
16887 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16888 ask_test_home();
16889 let tmp = tempfile::tempdir().unwrap();
16890 let repo = tmp.path().join("repo");
16891 std::fs::create_dir_all(&repo).unwrap();
16892 init_repo(&repo);
16893 let origin = tmp.path().join("origin.git");
16894 let g = |dir: &Path, args: &[&str]| {
16895 let out = std::process::Command::new("git")
16896 .args(args)
16897 .current_dir(dir)
16898 .quiet()
16899 .output()
16900 .expect("spawn git");
16901 assert!(
16902 out.status.success(),
16903 "git {args:?}: {}",
16904 String::from_utf8_lossy(&out.stderr)
16905 );
16906 };
16907 g(
16908 tmp.path(),
16909 &[
16910 "clone",
16911 "--bare",
16912 "-q",
16913 repo.to_str().unwrap(),
16914 origin.to_str().unwrap(),
16915 ],
16916 );
16917 g(
16918 &repo,
16919 &["remote", "add", "origin", origin.to_str().unwrap()],
16920 );
16921 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16922
16923 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16925 g(&repo, &["add", "-A"]);
16926 g(
16927 &repo,
16928 &["commit", "-q", "-m", "chore(deps): update a crate"],
16929 );
16930 g(&repo, &["push", "-q", "origin", "main"]);
16931 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16932 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16933 g(&repo, &["add", "-A"]);
16934 g(
16935 &repo,
16936 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16937 );
16938 g(&repo, &["checkout", "-q", "main"]);
16939
16940 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16941 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16942 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16944 assert_eq!(fallback, c1);
16945
16946 let mut state = review_state(&[
16947 "chore(deps): update a crate",
16948 "fix(daemon): apply a chosen action",
16949 ]);
16950 state.repo = repo.clone();
16951 state.base_branch = "main".to_owned();
16952 state.base_commit = c1;
16953 refresh_reviewed_commits(&mut state, "feat/own").await;
16954 assert_eq!(
16955 state.reviewed_commits,
16956 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16957 );
16958 assert_eq!(
16959 review_title(&state).as_deref(),
16960 Some("fix(daemon): apply a chosen action")
16961 );
16962 assert_eq!(
16963 leaked_subjects(&state, "feat/own").await,
16964 Some(vec!["chore(deps): update a crate".to_owned()])
16965 );
16966 state.config.merge.remote = "nowhere".to_owned();
16969 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16970 }
16971
16972 struct SummaryFixture {
16976 _tmp: tempfile::TempDir,
16977 repo: PathBuf,
16978 reply: PathBuf,
16979 runner: Runner,
16980 }
16981
16982 fn summary_fixture() -> SummaryFixture {
16983 ask_test_home();
16984 let tmp = tempfile::tempdir().unwrap();
16985 let repo = tmp.path().join("repo");
16986 std::fs::create_dir_all(&repo).unwrap();
16987 init_repo(&repo);
16988 let origin = tmp.path().join("origin.git");
16989 let g = |args: &[&str]| {
16990 let out = std::process::Command::new("git")
16991 .args(args)
16992 .current_dir(&repo)
16993 .quiet()
16994 .output()
16995 .expect("spawn git");
16996 assert!(
16997 out.status.success(),
16998 "git {args:?}: {}",
16999 String::from_utf8_lossy(&out.stderr)
17000 );
17001 };
17002 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17003 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17004 g(&["push", "-q", "origin", "main"]);
17005 g(&["checkout", "-q", "-b", "feat/own"]);
17006 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17007 g(&["add", "-A"]);
17008 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17009 let reply = tmp.path().join("reply.txt");
17010 std::fs::write(
17011 &reply,
17012 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17013 )
17014 .unwrap();
17015 let mut runner = runner_at(RunStatus::Gating);
17016 let mut state = review_state(&["feat(own): add the own file"]);
17017 state.repo = repo.clone();
17018 state.base_branch = "main".to_owned();
17019 state.candidates[0].branch = "feat/own".to_owned();
17020 state.candidates[0].worktree = repo.clone();
17021 let mut author = spec("alpha");
17022 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17023 author
17024 .env
17025 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17026 state.config.agents = vec![author];
17027 runner.state = state;
17028 SummaryFixture {
17029 _tmp: tmp,
17030 repo,
17031 reply,
17032 runner,
17033 }
17034 }
17035
17036 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17037 let winner = f.runner.state.candidates[0].clone();
17038 f.runner
17039 .guarded_pr_message(&winner, None, true)
17040 .await
17041 .unwrap()
17042 .expect("an English description never parks")
17043 }
17044
17045 fn clean_round_with_minor_findings() -> ReviewRound {
17046 let mut round = passing_round(2, None);
17047 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17048 round.reviews = vec![ReviewRecord {
17049 attempts: 0,
17050 reviewer: 1,
17051 agent: "alpha".into(),
17052 summary: "内部レビュー".into(),
17053 findings: vec![
17054 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17055 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17056 ],
17057 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17058 failed: None,
17059 duration_ms: 0,
17060 }];
17061 round
17062 }
17063
17064 #[tokio::test]
17065 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17066 for review_only in [false, true] {
17067 let mut f = summary_fixture();
17068 if !review_only {
17069 f.runner.state.candidates[0].agent = "alpha".into();
17070 }
17071 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17072 assert!(f.runner.state.reviews[0].clean);
17073 assert!(f.runner.state.reviews[0].fix.is_none());
17074 assert!(f.runner.state.open_findings().is_empty());
17075 assert_eq!(
17076 crate::pr_outcome::required_finding_ids(&f.runner.state),
17077 ["R2-1-1", "R2-1-2"]
17078 );
17079 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17080 let evidence = crate::pr_outcome::summary_evidence(
17081 &f.runner.state,
17082 crate::pr_outcome::FinalHead::At(&head),
17083 );
17084 assert!(
17085 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17086 );
17087 let omitted = posted(&mut f).await;
17089 assert!(omitted.body.contains("No description"));
17090 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17091 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17092 assert!(f.runner.state.change_summary.is_none());
17093 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17094 let described = posted(&mut f).await;
17095 assert!(described.body.contains("improve documentation for callers"));
17096 assert!(described.body.contains("remaining edge case"));
17097 assert!(described.body.contains("## Open review findings"));
17098 assert!(!described.body.contains("未修正"));
17099 }
17100 }
17101
17102 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17103 let mut f = summary_fixture();
17104 if !review_only {
17105 f.runner.state.candidates[0].agent = "alpha".into();
17106 }
17107 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17108 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17109 fixed
17110 .fix
17111 .as_mut()
17112 .unwrap()
17113 .rejected
17114 .push(crate::verdict::Rejection {
17115 id: "R1-1-1".into(),
17116 why: "呼び出されないコードのため辞退".into(),
17117 });
17118 let mut clean = clean_round_with_minor_findings();
17119 clean.verified_head = Some(head.clone());
17120 f.runner.state.reviews = vec![fixed, clean];
17121 ok_gate(&mut f.runner.state, Some(&head));
17122 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17123 f.runner.state.config.agents.push(scripted("beta", reply));
17124 f.runner.state.config.roles.language_judge =
17125 Some(crate::config::AgentChoice::One("beta".into()));
17126 f
17127 }
17128
17129 #[tokio::test]
17130 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17131 for review_only in [false, true] {
17132 let mut f = rewrite_fixture_with_findings(review_only,
17133 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17134 let m = posted(&mut f).await;
17135 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17136 let expected =
17137 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17138 assert!(m.body.contains(&expected));
17139 assert!(m.body.contains("## Review rounds"));
17140 assert!(
17141 m.body
17142 .contains("Gate (`verify.gate`): passed on the final commit")
17143 );
17144 assert!(
17145 m.body
17146 .contains("Tests (`verify.e2e`): passed on the final commit")
17147 );
17148 let c = f.runner.state.change_summary.as_ref().unwrap();
17149 assert!(
17150 m.body
17151 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17152 );
17153 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17154 assert!(crate::pr_outcome::covers_required_findings(
17155 &f.runner.state,
17156 &m.body
17157 ));
17158 assert!(m.body.contains("documentation remains incomplete"));
17159 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17160 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17161 assert!(f.runner.state.github_text.is_none());
17162 }
17163 }
17164
17165 #[tokio::test]
17166 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17167 for review_only in [false, true] {
17168 for reply in [
17169 "TITLE: fix: public description\nA short English description.",
17170 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17171 ] {
17172 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17173 let winner = f.runner.state.candidates[0].clone();
17174 let posted = f
17175 .runner
17176 .guarded_pr_message(&winner, None, true)
17177 .await
17178 .unwrap();
17179 assert!(posted.is_none());
17180 assert!(f.runner.state.parked);
17181 let withheld = f.runner.state.github_text.as_ref().unwrap();
17182 assert!(
17183 withheld.rewrite.is_none(),
17184 "an incomplete rewrite must not be selectable"
17185 );
17186 let q = ask::Questions::open()
17187 .get(withheld.question.as_ref().unwrap())
17188 .unwrap();
17189 assert!(
17190 !q.choices
17191 .contains(&crate::github_text::USE_REWRITE.to_owned())
17192 );
17193 assert!(
17194 q.detail.contains("R1-1-1")
17195 && q.detail.contains("R2-1-1")
17196 && q.detail.contains("R2-1-2")
17197 );
17198 }
17199 }
17200 }
17201
17202 #[tokio::test]
17203 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17204 for review_only in [false, true] {
17205 let mut f = summary_fixture();
17206 if !review_only {
17207 f.runner.state.candidates[0].agent = "alpha".into();
17208 }
17209 f.runner.state.instruction = "内部タスクの説明".into();
17210 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17211 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17212 f.runner.state.reviews = vec![
17213 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17214 round_with_notes(2, Some("二回目の内部修正報告です。")),
17215 passing_round(3, Some(&head)),
17216 ];
17217 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17218 ok_gate(&mut f.runner.state, Some(&head));
17219 let captured = f.repo.join("public-prompt.txt");
17220 let author = &mut f.runner.state.config.agents[0];
17221 author.command = vec![
17222 "sh".into(),
17223 "-c".into(),
17224 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17225 ];
17226 author.env.insert(
17227 "PROMPT_FILE".into(),
17228 captured.to_string_lossy().into_owned(),
17229 );
17230 let m = posted(&mut f).await;
17231 assert!(m.body.contains("Adds `own.txt`"));
17232 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17233 assert!(m.body.contains("R1-1-1"));
17234 assert!(
17235 m.body
17236 .contains("Gate (`verify.gate`): passed on the final commit")
17237 );
17238 assert!(
17239 m.body
17240 .contains("Tests (`verify.e2e`): passed on the final commit")
17241 );
17242 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17243 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17244 let prompt = std::fs::read_to_string(captured).unwrap();
17245 assert!(prompt.contains(&format!(
17246 "git diff {} {head}",
17247 f.runner.state.change_summary.as_ref().unwrap().base
17248 )));
17249 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17250 assert_eq!(
17251 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17252 "二回目の内部修正報告です。"
17253 );
17254 }
17255 }
17256
17257 #[tokio::test]
17258 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17259 for review_only in [false, true] {
17260 let mut f = summary_fixture();
17261 if !review_only {
17262 f.runner.state.candidates[0].agent = "alpha".into();
17263 }
17264 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17265 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17266 std::fs::write(&f.reply, "no usable description").unwrap();
17267 let failed = posted(&mut f).await;
17268 assert!(
17269 failed
17270 .body
17271 .contains("No description of what this change does could be generated")
17272 );
17273 assert!(failed.body.is_ascii());
17274 assert!(f.runner.state.github_text.is_none());
17275 assert!(f.runner.state.change_summary.is_none());
17276 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17277 let evidence = crate::pr_outcome::summary_evidence(
17278 &f.runner.state,
17279 crate::pr_outcome::FinalHead::At(&head),
17280 );
17281 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17282 assert!(
17283 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17284 .exists()
17285 );
17286 f.runner.state =
17287 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17288 std::fs::write(
17289 &f.reply,
17290 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17291 )
17292 .unwrap();
17293 let retried = posted(&mut f).await;
17294 assert!(retried.body.contains("Adds the committed own file."));
17295 assert_eq!(
17296 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17297 "日本語の修正報告"
17298 );
17299 }
17300 }
17301
17302 #[tokio::test]
17303 async fn same_head_verification_and_findings_invalidate_public_prose() {
17304 let mut f = summary_fixture();
17305 posted(&mut f).await;
17306 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17307 let original = f
17308 .runner
17309 .state
17310 .change_summary
17311 .as_ref()
17312 .unwrap()
17313 .evidence
17314 .clone();
17315 ok_gate(&mut f.runner.state, Some(&head));
17316 std::fs::write(
17317 &f.reply,
17318 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17319 )
17320 .unwrap();
17321 let verified = posted(&mut f).await;
17322 assert!(verified.body.contains("now verified"));
17323 assert_ne!(
17324 f.runner.state.change_summary.as_ref().unwrap().evidence,
17325 original
17326 );
17327 let mut round = round_with_notes(1, Some("内部報告"));
17328 round.reviews.push(ReviewRecord {
17329 attempts: 0,
17330 reviewer: 1,
17331 agent: "alpha".into(),
17332 summary: "内部レビュー".into(),
17333 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17334 vote: None,
17335 failed: None,
17336 duration_ms: 0,
17337 });
17338 round
17339 .fix
17340 .as_mut()
17341 .unwrap()
17342 .rejected
17343 .push(crate::verdict::Rejection {
17344 id: "R1-1-1".into(),
17345 why: "呼び出し元から到達できない".into(),
17346 });
17347 f.runner.state.reviews = vec![round];
17348 let omitted = posted(&mut f).await;
17350 assert!(omitted.body.contains("No description"));
17351 assert!(omitted.body.contains("R1-1-1"));
17352 assert!(!omitted.body.contains("到達"));
17353 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17354 let described = posted(&mut f).await;
17355 assert!(described.body.contains("unreachable from callers"));
17356 assert!(described.body.contains("R1-1-1"));
17357 assert!(!described.body.contains("到達"));
17358 let evidence = f
17359 .runner
17360 .state
17361 .change_summary
17362 .as_ref()
17363 .unwrap()
17364 .evidence
17365 .clone();
17366 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17367 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17368 assert!(posted(&mut f).await.body.contains("rationale updated"));
17369 assert_ne!(
17370 f.runner.state.change_summary.as_ref().unwrap().evidence,
17371 evidence
17372 );
17373 f.runner
17375 .state
17376 .change_summary
17377 .as_mut()
17378 .unwrap()
17379 .evidence
17380 .clear();
17381 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17382 assert!(posted(&mut f).await.body.contains("current specification"));
17383 }
17384
17385 #[tokio::test]
17386 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17387 let mut f = summary_fixture();
17388 let m = posted(&mut f).await;
17389 assert_eq!(m.title, "feat(own): add the own file");
17390 assert!(
17391 m.body
17392 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17393 "{}",
17394 m.body
17395 );
17396 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17397 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17398 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17399 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17400 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17401 let kept = f.runner.state.change_summary.as_ref().unwrap();
17402 assert_eq!(
17403 (kept.head.as_str(), kept.base.as_str()),
17404 (head.as_str(), base.as_str())
17405 );
17406 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17407 }
17408
17409 #[tokio::test]
17410 async fn review_only_summary_receives_the_configured_github_language() {
17411 let mut f = summary_fixture();
17412 f.runner.state.config.graph.github_language = "ja".into();
17413 let captured = f.repo.join("summary-prompt.txt");
17414 let author = &mut f.runner.state.config.agents[0];
17415 author.command = vec![
17416 "sh".into(),
17417 "-c".into(),
17418 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17419 ];
17420 author.env.insert(
17421 "PROMPT_FILE".into(),
17422 captured.to_string_lossy().into_owned(),
17423 );
17424 std::fs::write(
17425 &f.reply,
17426 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17427 )
17428 .unwrap();
17429 let m = posted(&mut f).await;
17430 let prompt = std::fs::read_to_string(captured).unwrap();
17431 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17432 assert!(!prompt.contains("in English"));
17433 assert_eq!(m.title, "feat(own): ファイルを追加");
17434 assert!(m.body.contains("own.txt を追加します。"));
17435 }
17436
17437 #[tokio::test]
17438 async fn review_fixes_come_after_the_whole_change_summary() {
17439 let mut f = summary_fixture();
17440 f.runner.state.reviews = vec![
17441 round_with_notes(1, Some("handled the PATH case")),
17442 round_with_notes(2, None),
17443 ];
17444 let m = posted(&mut f).await;
17445 let summary = m.body.find("## Summary").expect("summary");
17446 let fixes = m.body.find("## Review rounds").expect("review rounds");
17447 assert!(summary < fixes, "{}", m.body);
17448 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17449 assert!(m.body.contains("## Final verification"), "{}", m.body);
17450 }
17451
17452 #[tokio::test]
17453 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17454 let mut f = summary_fixture();
17455 let first = posted(&mut f).await;
17456 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17457 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17459 let again = posted(&mut f).await;
17460 assert_eq!((again.title, again.body), (first.title, first.body));
17461
17462 let g = |args: &[&str]| {
17463 let out = std::process::Command::new("git")
17464 .args(args)
17465 .current_dir(&f.repo)
17466 .quiet()
17467 .output()
17468 .expect("spawn git");
17469 assert!(out.status.success(), "git {args:?}");
17470 };
17471 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17472 g(&["add", "-A"]);
17473 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17474 let second = posted(&mut f).await;
17475 let kept = f.runner.state.change_summary.clone().unwrap();
17476 assert_ne!(kept.head, first_head);
17477 assert!(second.body.contains("Not read yet."), "{}", second.body);
17478 assert!(!second.body.contains("so the first summary is read"));
17479 assert_eq!(second.title, "feat(own): other");
17480 }
17481
17482 #[tokio::test]
17483 async fn an_unavailable_summary_is_stated_and_never_invented() {
17484 let mut f = summary_fixture();
17485 std::fs::write(&f.reply, "no title line here").unwrap();
17486 let m = posted(&mut f).await;
17487 assert!(
17488 m.body
17489 .contains("No description of what this change does could be generated"),
17490 "{}",
17491 m.body
17492 );
17493 assert!(m.body.contains("internal reports remain in local records"));
17494 assert!(f.runner.state.change_summary.is_none());
17495 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17496 let winner = f.runner.state.candidates[0].clone();
17498 let preview = f
17499 .runner
17500 .guarded_pr_message(&winner, None, false)
17501 .await
17502 .unwrap()
17503 .unwrap();
17504 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17505 }
17506
17507 #[test]
17508 fn pr_message_review_single_commit_uses_its_subject() {
17509 let state = review_state(&["feat(nats): per-role user"]);
17510 let m = pr_message(&state, 'A');
17511 assert_eq!(m.title, "feat(nats): per-role user");
17512 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17513 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17514 }
17515
17516 #[test]
17517 fn pr_message_review_multi_commit_takes_the_oldest() {
17518 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17519 let m = pr_message(&state, 'A');
17520 assert_eq!(m.title, "feat: the change");
17521 for s in ["feat: the change", "fix: typo", "fix: again"] {
17522 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17523 }
17524 }
17525
17526 #[test]
17527 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17528 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17529 let mut state = review_state(&[first, "fix: later fixup"]);
17530 state.candidates[0].branch = "機能/ログイン".to_owned();
17531 let m = pr_message(&state, 'A');
17532 assert!(
17533 m.title.starts_with("chore: land candidate A of run"),
17534 "{}",
17535 m.title
17536 );
17537 }
17538 }
17539
17540 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17541 BranchFacts {
17542 commits: commits
17543 .iter()
17544 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17545 .collect(),
17546 stat: stat.to_owned(),
17547 }
17548 }
17549
17550 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17551 let mut r = review_round(true, 0, 1, 1, true, true);
17552 r.round = round;
17553 r.fix = notes.map(|n| FixRecord {
17554 agent: "fixer".to_owned(),
17555 addressed: Vec::new(),
17556 rejected: Vec::new(),
17557 notes: n.to_owned(),
17558 committed: true,
17559 failed: None,
17560 duration_ms: 0,
17561 continuation: None,
17562 });
17563 r
17564 }
17565
17566 #[test]
17567 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17568 let state = review_state(&["ignored"]);
17569 let f = facts(
17570 &[
17571 (
17572 "fix(login): resolve PATH on macOS",
17573 "Login shells skip rc files.",
17574 ),
17575 ("fix: address review", ""),
17576 ],
17577 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17578 );
17579 let m = pr_message_with(&state, 'A', Some(&f));
17580 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17581 assert!(
17582 m.body
17583 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17584 "{}",
17585 m.body
17586 );
17587 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17588 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17589 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17590 for banned in [
17591 "Review the work already",
17592 "no task statement",
17593 "Original task",
17594 ] {
17595 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17596 }
17597 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17598 }
17599
17600 #[test]
17601 fn pr_message_review_truncates_a_huge_first_commit_body() {
17602 let state = review_state(&["ignored"]);
17603 let f = facts(
17604 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17605 "s",
17606 );
17607 let m = pr_message_with(&state, 'A', Some(&f));
17608 assert!(m.body.len() < 30_000, "{}", m.body.len());
17609 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17610 assert!(
17611 m.body.contains("- ... 1 more commit(s)"),
17612 "{}",
17613 m.body.len()
17614 );
17615 assert!(m.body.ends_with("magi:candidate-a\n"));
17616 }
17617
17618 #[test]
17619 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17620 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17621 assert_eq!(m.title, "feat: x");
17622 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17623 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17624 assert!(!m.body.contains("no task statement"), "{}", m.body);
17625 }
17626
17627 #[test]
17628 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17629 let mut state = review_state(&["日本語の件名"]);
17630 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17631 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17632 state.candidates[0].branch = "機能/ログイン".to_owned();
17633 assert!(
17634 pr_message(&state, 'A')
17635 .title
17636 .starts_with("chore: land candidate A")
17637 );
17638 }
17639
17640 #[test]
17641 fn pr_message_review_fixes_survive_a_clean_final_round() {
17642 let mut state = review_state(&["feat: x"]);
17643 state.reviews = vec![
17644 round_with_notes(1, Some("handled the PATH case")),
17645 round_with_notes(2, None),
17646 ];
17647 let body = pr_message(&state, 'A').body;
17648 assert!(
17649 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17650 "{body}"
17651 );
17652 assert!(body.contains("### Round 1"), "{body}");
17653
17654 state.reviews = vec![
17655 round_with_notes(1, Some("first fix")),
17656 round_with_notes(2, Some("")),
17657 round_with_notes(3, Some("second fix")),
17658 round_with_notes(4, None),
17659 ];
17660 let body = pr_message(&state, 'A').body;
17661 assert!(
17662 body.contains("### Round 1") && !body.contains("first fix"),
17663 "{body}"
17664 );
17665 assert!(
17666 body.contains("### Round 3") && !body.contains("second fix"),
17667 "{body}"
17668 );
17669 assert!(body.contains("### Round 2"), "{body}");
17670 }
17671
17672 const FINAL: &str = "abcdef1234567890";
17673
17674 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17675 let mut r = round_with_notes(round, None);
17676 r.verified_head = verified.map(str::to_owned);
17677 r
17678 }
17679
17680 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17681 state.gate = vec![CommandOutcome {
17682 command: "gate".to_owned(),
17683 code: Some(0),
17684 output_tail: String::new(),
17685 duration_ms: 0,
17686 resource_blocked: false,
17687 }];
17688 state.gate_ran = true;
17689 state.gate_head = head.map(str::to_owned);
17690 }
17691
17692 fn body_at(state: &RunState) -> String {
17693 pr_message_raw(
17694 state,
17695 'A',
17696 None,
17697 crate::pr_outcome::FinalHead::At(FINAL),
17698 ChangeNote::NotAsked,
17699 )
17700 .body
17701 }
17702
17703 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17704
17705 #[test]
17706 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17707 let mut state = review_state(&["feat: x"]);
17708 state.reviews = vec![
17709 round_with_notes(1, Some(SANDBOXED)),
17710 passing_round(2, Some(FINAL)),
17711 ];
17712 ok_gate(&mut state, Some(FINAL));
17713 let body = body_at(&state);
17714 assert!(!body.contains("Handled the PATH case."), "{body}");
17715 assert!(!body.contains("could not commit"), "{body}");
17716 assert!(!body.contains("could not run"), "{body}");
17717 assert!(
17718 body.contains("The fix is committed on the branch"),
17719 "{body}"
17720 );
17721 assert!(!body.contains("sandbox"), "{body}");
17722 assert!(
17723 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17724 "{body}"
17725 );
17726 assert!(
17727 body.contains("Gate (`verify.gate`): passed on the final commit"),
17728 "{body}"
17729 );
17730 assert!(
17732 state.reviews[0]
17733 .fix
17734 .as_ref()
17735 .unwrap()
17736 .notes
17737 .contains("could not commit")
17738 );
17739 }
17740
17741 #[test]
17742 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17743 let mut state = review_state(&["feat: x"]);
17744 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17745 r1.fix.as_mut().unwrap().committed = false;
17746 let mut r2 = passing_round(2, Some(FINAL));
17747 r2.e2e[0].code = Some(1);
17748 state.reviews = vec![r1, r2];
17749 let body = body_at(&state);
17750 assert!(!body.contains("could not commit"), "{body}");
17751 assert!(!body.contains("could not run"), "{body}");
17752 assert!(body.contains("This round produced no commit"), "{body}");
17753 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17754 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17755 assert!(!body.contains("passed"), "{body}");
17756 }
17757
17758 #[test]
17759 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17760 let mut state = review_state(&["feat: x"]);
17761 state.reviews = vec![
17762 round_with_notes(1, Some(SANDBOXED)),
17763 passing_round(2, Some("0123456789abcdef")),
17764 ];
17765 ok_gate(&mut state, Some("0123456789abcdef"));
17766 let body = body_at(&state);
17767 assert!(!body.contains("HTTP tests could not run"), "{body}");
17768 assert!(!body.contains("could not commit"), "{body}");
17770 assert!(
17771 body.contains("passed on an earlier commit (`0123456`)"),
17772 "{body}"
17773 );
17774 assert!(!body.contains("passed on the final commit"), "{body}");
17775
17776 ok_gate(&mut state, None);
17778 let body = body_at(&state);
17779 assert!(
17780 body.contains("passed on a commit that was not recorded"),
17781 "{body}"
17782 );
17783 }
17784
17785 #[test]
17786 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17787 let state = state_with_summary(
17788 "add retries to the client",
17789 "TITLE: feat: retries\n\nDid it.",
17790 );
17791 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17792 assert_eq!(m.title, "feat: retries");
17793 assert!(
17794 m.body.contains("<summary>Original task</summary>"),
17795 "{}",
17796 m.body
17797 );
17798 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17799 assert!(
17800 m.body
17801 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17802 "{}",
17803 m.body
17804 );
17805 }
17806
17807 #[test]
17808 fn pr_message_review_bounds_a_long_english_subject() {
17809 let long = format!("feat: {}", "word ".repeat(100));
17810 let m = pr_message(&review_state(&[&long]), 'A');
17811 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17812 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17813 }
17814
17815 #[test]
17816 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17817 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17818 let m = pr_message(&state, 'A');
17819 assert_eq!(m.title, "add retries");
17820 assert!(m.body.contains("<summary>Original task</summary>"));
17821 assert!(!m.body.contains("Commits under review"));
17822 assert_eq!(landing_subject_source(&state), state.instruction);
17823 }
17824
17825 #[test]
17826 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17827 let state = review_state(&["feat: the change", "fix: typo"]);
17828 let source = landing_subject_source(&state);
17829 assert_eq!(land::merge_subject("", &source), "feat: the change");
17830 assert_eq!(
17831 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17832 "feat: the change"
17833 );
17834 assert_eq!(
17836 land::merge_subject("feat: renamed by hand", &source),
17837 "feat: renamed by hand"
17838 );
17839 let blank = review_state(&["日本語"]);
17840 assert!(
17841 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17842 );
17843 }
17844
17845 #[test]
17846 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17847 let state = review_state(&["feat: the change"]);
17848 let source = landing_subject_source(&state);
17849 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17850 assert_eq!(
17851 land::merge_subject(landing_title(&state, old), &source),
17852 "feat: the change"
17853 );
17854 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17855 let task = state_with_summary("add retries", "");
17856 assert_eq!(landing_title(&task, old), old);
17857 }
17858
17859 #[test]
17860 fn pr_message_describes_the_change_not_the_task() {
17861 let state = state_with_summary(
17862 "今回やってほしいこと: results projector を直す",
17863 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17864 );
17865 let m = pr_message(&state, 'A');
17866 assert_eq!(m.title, "fix(web): batch the runs list reads");
17867 assert!(
17868 m.body.starts_with("## Summary\n\n- reads run.json once"),
17869 "{}",
17870 m.body
17871 );
17872 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17873 let task_at = m.body.find("今回やってほしいこと").unwrap();
17874 let details_at = m.body.find("<details>").unwrap();
17875 assert!(
17876 details_at < task_at,
17877 "the task lives inside <details>: {}",
17878 m.body
17879 );
17880 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17881 assert!(m.body.contains("magi:candidate-a"));
17882 }
17883
17884 #[test]
17885 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17886 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17887 let m = pr_message(&state, 'A');
17888 assert_eq!(m.title, "add retries");
17889 assert!(
17890 m.body.contains("## Summary\n\n- did some things"),
17891 "{}",
17892 m.body
17893 );
17894
17895 let none = RunState::new(
17896 PathBuf::from("/repo"),
17897 "main".to_owned(),
17898 "abc1234".to_owned(),
17899 "add retries".to_owned(),
17900 Config::default(),
17901 );
17902 let m = pr_message(&none, 'A');
17903 assert_eq!(m.title, "add retries");
17904 assert!(!m.body.contains("## Summary"), "{}", m.body);
17905 }
17906
17907 #[test]
17908 fn pr_message_refuses_the_candidate_commit_subject() {
17909 for bad in [
17910 "TITLE: magi: candidate A (uncommitted work)",
17911 "TITLE: chore: stuff (uncommitted work)",
17912 "TITLE: ",
17913 ] {
17914 let state = state_with_summary("add retries", bad);
17915 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17916 }
17917 }
17918
17919 #[test]
17920 fn pr_message_bounds_a_very_long_task_and_title() {
17921 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17922 let state = state_with_summary(&long, "- nothing");
17923 let m = pr_message(&state, 'A');
17924 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17925 assert!(!m.title.contains('\n'));
17926
17927 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17928 let m = pr_message(&state, 'A');
17929 assert!(m.title.starts_with("feat: "));
17930 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17931 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17932 }
17933
17934 fn long_title_of(instruction: &str) -> String {
17935 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17936 }
17937
17938 #[test]
17939 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17940 let first = "Make the landing path keep a readable title for long tasks";
17941 let line = format!(
17942 "{first}. {}",
17943 "And then keep going with more words ".repeat(20)
17944 );
17945 let t = long_title_of(&line);
17946 assert_eq!(t, first);
17947 assert!(!t.starts_with("chore: land"));
17948 }
17949
17950 #[test]
17951 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17952 let line = "word ".repeat(200);
17953 let t = long_title_of(&line);
17954 assert!(t.ends_with("word..."), "{t}");
17955 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17956 }
17957
17958 #[test]
17959 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17960 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17961 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17962 }
17963 }
17964
17965 #[test]
17966 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17967 let mut state = state_with_summary("再試行を追加する", "- nothing");
17968 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17969 state.config.graph.github_language = "ja".into();
17970 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17971 }
17972
17973 #[test]
17974 fn pr_message_title_limit_is_exact() {
17975 let at = "a".repeat(PR_TITLE_MAX);
17976 assert_eq!(long_title_of(&at), at);
17977 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17978 assert!(over.ends_with("..."), "{over}");
17979 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17980 }
17981
17982 #[test]
17983 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17984 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17985 let t = long_title_of(&line);
17986 assert!(t.ends_with("..."), "{t}");
17987 assert!(t.is_ascii(), "{t}");
17988 }
17989
17990 #[test]
17991 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17992 let line = format!(
17993 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17994 "plus more filler words ".repeat(20)
17995 );
17996 let t = long_title_of(&line);
17997 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17998 }
17999
18000 #[test]
18001 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18002 let line = format!("\n\n# {}", "title words ".repeat(40));
18003 let state = state_with_summary(&line, "- nothing");
18004 let m = pr_message(&state, 'A');
18005 assert!(m.title.starts_with("title words"), "{}", m.title);
18006 assert_eq!(
18007 land::merge_subject(&m.title, &landing_subject_source(&state)),
18008 m.title
18009 );
18010 assert_eq!(
18012 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18013 "feat: renamed by hand"
18014 );
18015 }
18016
18017 #[tokio::test]
18018 async fn github_text_rewrite_is_bounded_and_falls_back() {
18019 crate::run::pin_test_home();
18020 for (reply, accepted) in [
18021 (
18022 "TITLE: fix: retries\nAdd retries for failed requests.",
18023 true,
18024 ),
18025 (
18026 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18027 false,
18028 ),
18029 (
18030 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18031 true,
18032 ),
18033 (
18034 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18035 true,
18036 ),
18037 (
18038 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18039 true,
18040 ),
18041 (
18042 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18043 false,
18044 ),
18045 ("TITLE: fix: retries\n", false),
18046 ] {
18047 let dir = tempfile::tempdir().unwrap();
18048 let mut runner = runner_at(RunStatus::Gating);
18049 runner.state = state_with_summary(
18050 "add retries",
18051 "TITLE: fix: retries\n日本語の説明を書きます。",
18052 );
18053 runner.state.repo = dir.path().to_owned();
18054 runner.state.candidates[0].worktree = dir.path().to_owned();
18055 let mut author = spec("alpha");
18056 author.command = vec![
18057 "sh".into(),
18058 "-c".into(),
18059 "printf '%s' \"$REWRITE_REPLY\"".into(),
18060 ];
18061 author.env.insert("REWRITE_REPLY".into(), reply.into());
18062 runner.state.config.agents = vec![author];
18063 seed_public_summary(&mut runner);
18064 let winner = runner.state.candidates[0].clone();
18065 let first = runner
18066 .guarded_pr_message(&winner, None, true)
18067 .await
18068 .unwrap();
18069 let message = if accepted {
18070 first.expect("a passing rewrite never asks")
18071 } else {
18072 assert!(first.is_none());
18074 assert!(runner.state.parked);
18075 let store = ask::Questions::open();
18076 let asked: Vec<_> = store
18077 .list()
18078 .into_iter()
18079 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18080 .collect();
18081 assert_eq!(asked.len(), 1);
18082 assert!(!asked[0].detail.contains("secret"));
18083 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18084 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18085 assert_eq!(
18086 asked[0]
18087 .choices
18088 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18089 rewrite.is_some()
18090 );
18091 store
18094 .update(&asked[0].id, |q| {
18095 q.abandon("test");
18096 Ok(())
18097 })
18098 .unwrap();
18099 let again = runner
18100 .guarded_pr_message(&winner, None, true)
18101 .await
18102 .unwrap()
18103 .expect("an abandoned question falls back");
18104 let n = store
18105 .list()
18106 .into_iter()
18107 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18108 .count();
18109 assert_eq!(n, 1, "asked once per rejected text");
18110 again
18111 };
18112 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18113 assert!(!message.body.contains("secret"));
18114 assert!(!message.body.contains("prefix"));
18115 assert!(!message.title.contains("secret"));
18116 assert_eq!(
18117 message.body.contains("Add retries for failed requests."),
18118 accepted
18119 );
18120 assert_eq!(
18121 runner.state.seats["github-text-rewrite"].turns, 1,
18122 "only one rewrite invocation"
18123 );
18124 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18125 if accepted {
18126 assert!(runner.state.github_text.is_none());
18127 assert!(!runner.state.parked);
18128 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18129 assert!(message.body.contains("## Final verification"));
18130 assert!(message.body.contains("Evidence: final diff"));
18131 assert!(
18132 message
18133 .body
18134 .contains(&format!("magi:run/{}", runner.state.id))
18135 );
18136 }
18137 if !accepted {
18138 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18139 assert!(
18140 message
18141 .body
18142 .contains(&format!("magi:run/{}", runner.state.id))
18143 );
18144 }
18145 }
18146 }
18147
18148 fn scripted(id: &str, reply: &str) -> AgentSpec {
18149 let mut a = spec(id);
18150 a.command = vec![
18151 "sh".into(),
18152 "-c".into(),
18153 "printf '%s' \"$REWRITE_REPLY\"".into(),
18154 ];
18155 a.env.insert("REWRITE_REPLY".into(), reply.into());
18156 a
18157 }
18158
18159 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18160 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18161
18162 fn seed_public_summary(runner: &mut Runner) {
18164 let repo = &runner.state.repo;
18165 let g = |args: &[&str]| {
18166 let out = std::process::Command::new("git")
18167 .args(args)
18168 .current_dir(repo)
18169 .quiet()
18170 .output()
18171 .unwrap();
18172 assert!(out.status.success(), "git {args:?}");
18173 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18174 };
18175 g(&["init", "-q", "-b", "main"]);
18176 g(&[
18177 "-c",
18178 "user.name=Test",
18179 "-c",
18180 "user.email=test@example.invalid",
18181 "commit",
18182 "-q",
18183 "--allow-empty",
18184 "-m",
18185 "feat: fixture",
18186 ]);
18187 let head = g(&["rev-parse", "HEAD"]);
18188 g(&["update-ref", "refs/remotes/origin/main", &head]);
18189 let evidence = crate::pr_outcome::summary_evidence(
18190 &runner.state,
18191 crate::pr_outcome::FinalHead::At(&head),
18192 );
18193 let generated = &runner.state.candidates[0].summary;
18194 runner.state.change_summary = Some(crate::run::ChangeSummary {
18195 base: head.clone(),
18196 head,
18197 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18198 title: summary_title(generated).unwrap(),
18199 body: summary_without_title(generated),
18200 });
18201 }
18202
18203 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18204 let mut runner = runner_at(RunStatus::Gating);
18205 runner.state = state_with_summary("add retries", JA_SUMMARY);
18206 if review_only {
18207 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18208 }
18209 runner.state.repo = dir.to_owned();
18210 runner.state.candidates[0].worktree = dir.to_owned();
18211 runner.state.config.agents = agents;
18212 seed_public_summary(&mut runner);
18213 runner
18214 }
18215
18216 fn gate_questions(runner: &Runner) -> usize {
18217 ask::Questions::open()
18218 .list()
18219 .into_iter()
18220 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18221 .count()
18222 }
18223
18224 #[tokio::test]
18225 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18226 crate::run::pin_test_home();
18227 let dir = tempfile::tempdir().unwrap();
18228 let mut runner = rewrite_runner(
18229 dir.path(),
18230 true,
18231 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18232 );
18233 runner.state.config.roles.language_judge =
18234 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18235 let winner = runner.state.candidates[0].clone();
18236 let msg = runner
18237 .guarded_pr_message(&winner, None, true)
18238 .await
18239 .unwrap()
18240 .expect("an English rewrite posts without asking");
18241 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18242 assert!(msg.body.contains("<details>kept</details>"));
18243 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18244 assert_eq!(gate_questions(&runner), 0);
18245 assert!(runner.state.github_text.is_none());
18246 assert!(!runner.state.seats.contains_key("impl-A"));
18247 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18248 }
18249
18250 #[tokio::test]
18251 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18252 crate::run::pin_test_home();
18253 let dir = tempfile::tempdir().unwrap();
18254 let mut runner = rewrite_runner(
18255 dir.path(),
18256 false,
18257 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18258 );
18259 runner.state.config.roles.language_judge =
18260 Some(crate::config::AgentChoice::One("beta".into()));
18261 runner.state.seats.insert(
18262 "impl-A".into(),
18263 SeatState::new("impl-A", "alpha", runner.state.seed),
18264 );
18265 let winner = runner.state.candidates[0].clone();
18266 let msg = runner
18267 .guarded_pr_message(&winner, None, true)
18268 .await
18269 .unwrap()
18270 .expect("rewritten");
18271 assert!(msg.body.contains("<details>kept</details>"));
18272 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18273 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18274 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18275 }
18276
18277 #[tokio::test]
18278 async fn rewrite_reuses_a_resumable_implementer_session() {
18279 crate::run::pin_test_home();
18280 let dir = tempfile::tempdir().unwrap();
18281 let mut runner = rewrite_runner(
18282 dir.path(),
18283 false,
18284 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18285 );
18286 runner.state.config.roles.language_judge =
18287 Some(crate::config::AgentChoice::One("beta".into()));
18288 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18289 seat.turns = 1;
18290 runner.state.seats.insert("impl-A".into(), seat);
18291 let winner = runner.state.candidates[0].clone();
18292 runner
18293 .guarded_pr_message(&winner, None, true)
18294 .await
18295 .unwrap()
18296 .expect("rewritten");
18297 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18298 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18299 }
18300
18301 #[tokio::test]
18302 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18303 crate::run::pin_test_home();
18306 let dir = tempfile::tempdir().unwrap();
18307 let mut exhausted = spec("alpha");
18308 exhausted.command = vec!["false".into()];
18309 let mut runner = rewrite_runner(
18310 dir.path(),
18311 false,
18312 vec![exhausted, scripted("beta", EN_REWRITE)],
18313 );
18314 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18316 "alpha".into(),
18317 "beta".into(),
18318 ]));
18319 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18320 seat.turns = 1;
18321 runner.state.seats.insert("impl-A".into(), seat);
18322 let winner = runner.state.candidates[0].clone();
18323 let msg = runner
18324 .guarded_pr_message(&winner, None, true)
18325 .await
18326 .unwrap()
18327 .expect("the role rewrites after the implementer fails");
18328 assert!(msg.body.contains("`src/net.rs`"));
18329 assert_eq!(
18330 runner.state.seats["impl-A"].turns, 2,
18331 "the implementer was tried once"
18332 );
18333 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18334 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18335 }
18336
18337 #[tokio::test]
18338 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18339 crate::run::pin_test_home();
18340 let dir = tempfile::tempdir().unwrap();
18341 let mut broken = spec("bad");
18342 broken.command = vec!["false".into()];
18343 let mut runner = rewrite_runner(
18344 dir.path(),
18345 true,
18346 vec![
18347 broken,
18348 scripted("noformat", "Add retries."),
18349 scripted("good", EN_REWRITE),
18350 ],
18351 );
18352 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18353 "bad".into(),
18354 "noformat".into(),
18355 "good".into(),
18356 ]));
18357 let winner = runner.state.candidates[0].clone();
18358 let msg = runner
18359 .guarded_pr_message(&winner, None, true)
18360 .await
18361 .unwrap()
18362 .expect("the third agent rewrites");
18363 assert!(msg.body.contains("`src/net.rs`"));
18364 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18365 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18366 }
18367
18368 #[tokio::test]
18369 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18370 crate::run::pin_test_home();
18371 for (agents, role, saved) in [
18373 (
18374 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18375 Some("beta"),
18376 true,
18377 ),
18378 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18379 (Vec::new(), None, false),
18380 ] {
18381 let dir = tempfile::tempdir().unwrap();
18382 let mut runner = rewrite_runner(dir.path(), true, agents);
18383 runner.state.config.roles.language_judge =
18384 role.map(|r| crate::config::AgentChoice::One(r.into()));
18385 let winner = runner.state.candidates[0].clone();
18386 let first = runner
18387 .guarded_pr_message(&winner, None, true)
18388 .await
18389 .unwrap();
18390 assert!(first.is_none(), "withheld text parks the run");
18391 assert!(runner.state.parked);
18392 assert_eq!(gate_questions(&runner), 1);
18393 let g = runner.state.github_text.as_ref().unwrap();
18394 assert_eq!(g.rewrite.is_some(), saved);
18395 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18396 }
18397 }
18398
18399 #[tokio::test]
18400 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18401 crate::run::pin_test_home();
18402 let dir = tempfile::tempdir().unwrap();
18403 let mut runner = runner_at(RunStatus::Gating);
18404 runner.state =
18405 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18406 runner.state.repo = dir.path().to_owned();
18407 runner.state.candidates[0].worktree = dir.path().to_owned();
18408 let mut author = spec("alpha");
18409 author.command = vec![
18410 "sh".into(),
18411 "-c".into(),
18412 "printf '%s' \"$REWRITE_REPLY\"".into(),
18413 ];
18414 author.env.insert(
18415 "REWRITE_REPLY".into(),
18416 format!(
18417 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18418 "English sentence. ".repeat(600)
18419 ),
18420 );
18421 runner.state.config.agents = vec![author];
18422 seed_public_summary(&mut runner);
18423 let winner = runner.state.candidates[0].clone();
18424 assert!(
18425 runner
18426 .guarded_pr_message(&winner, None, true)
18427 .await
18428 .unwrap()
18429 .is_none()
18430 );
18431 let record = runner.state.github_text.clone().unwrap();
18432 let rewrite = record.rewrite.clone().unwrap();
18433 assert_eq!(rewrite.failed, ["body-language"]);
18434 assert!(
18435 rewrite
18436 .body
18437 .ends_with(&format!("magi:run/{}", runner.state.id))
18438 );
18439 let store = ask::Questions::open();
18440 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18441 assert!(
18442 q.choices
18443 .contains(&crate::github_text::USE_REWRITE.to_owned())
18444 );
18445 for label in [
18446 "ORIGINAL",
18447 "FALLBACK",
18448 "REWRITE",
18449 "Rules still failed: body-language",
18450 "truncated",
18451 ] {
18452 assert!(q.detail.contains(label), "{label}");
18453 }
18454 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18455 &runner.state,
18456 &crate::github_text::artifact_name(&record.fingerprint),
18457 ))
18458 .unwrap();
18459 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18460 runner.state =
18462 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18463 store
18464 .update(&q.id, |q| {
18465 q.say("まだ日本語")?;
18466 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18467 })
18468 .unwrap();
18469 assert!(
18470 runner
18471 .guarded_pr_message(&winner, None, true)
18472 .await
18473 .unwrap()
18474 .is_none()
18475 );
18476 let retry = runner.state.github_text.as_ref().unwrap();
18477 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18478 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18479 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18480 assert!(
18481 q.posting_comparison
18482 .as_ref()
18483 .unwrap()
18484 .matches_detail(&q.detail)
18485 );
18486 store
18487 .update(&q.id, |q| {
18488 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18489 })
18490 .unwrap();
18491 let message = runner
18492 .guarded_pr_message(&winner, None, true)
18493 .await
18494 .unwrap()
18495 .unwrap();
18496 assert_eq!(message.title, rewrite.title);
18497 assert_eq!(message.body, rewrite.body);
18498 assert_eq!(
18499 runner.state.github_text.as_ref().unwrap().choice,
18500 Some(GithubTextChoice::Rewrite)
18501 );
18502 let again = runner
18503 .guarded_pr_message(&winner, None, true)
18504 .await
18505 .unwrap()
18506 .unwrap();
18507 assert_eq!(again.title, message.title);
18508 assert_eq!(again.body, message.body);
18509 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18510 assert!(
18511 !runner
18512 .state
18513 .events
18514 .iter()
18515 .any(|e| e.message.contains(&rewrite.title)
18516 || e.message.contains("English sentence."))
18517 );
18518 }
18519
18520 #[tokio::test]
18521 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18522 crate::run::pin_test_home();
18523 let dir = tempfile::tempdir().unwrap();
18524 let mut runner = runner_at(RunStatus::Gating);
18525 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18526 runner.state.repo = dir.path().to_owned();
18527 runner.state.candidates[0].worktree = dir.path().to_owned();
18528 seed_public_summary(&mut runner);
18529 let winner = runner.state.candidates[0].clone();
18530 assert!(
18531 runner
18532 .guarded_pr_message(&winner, None, true)
18533 .await
18534 .unwrap()
18535 .is_none()
18536 );
18537 let record = runner.state.github_text.as_ref().unwrap();
18538 assert!(record.rewrite.is_none());
18539 let mut old = serde_json::to_value(record).unwrap();
18541 old.as_object_mut().unwrap().remove("rewrite");
18542 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18543 assert!(old.rewrite.is_none());
18544 let id = record.question.clone().unwrap();
18545 ask::Questions::open()
18546 .update(&id, |q| {
18547 q.choices.push(crate::github_text::USE_REWRITE.into());
18548 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18549 })
18550 .unwrap();
18551 let message = runner
18552 .guarded_pr_message(&winner, None, true)
18553 .await
18554 .unwrap()
18555 .unwrap();
18556 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18557 assert_eq!(
18558 runner.state.github_text.as_ref().unwrap().choice,
18559 Some(GithubTextChoice::Fallback)
18560 );
18561 }
18562
18563 #[tokio::test]
18564 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18565 crate::run::pin_test_home();
18566 let dir = tempfile::tempdir().unwrap();
18567 let mut runner = runner_at(RunStatus::Gating);
18568 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18569 runner.state.repo = dir.path().to_owned();
18570 runner.state.candidates[0].worktree = dir.path().to_owned();
18571 seed_public_summary(&mut runner);
18572 let winner = runner.state.candidates[0].clone();
18573 assert!(
18574 runner
18575 .guarded_pr_message(&winner, None, true)
18576 .await
18577 .unwrap()
18578 .is_none()
18579 );
18580 let store = ask::Questions::open();
18581 let find = |run: &str| {
18582 store
18583 .list()
18584 .into_iter()
18585 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18586 .unwrap()
18587 };
18588 let run = runner.state.id.clone();
18589 let q = find(&run);
18590 let fp = runner
18591 .state
18592 .github_text
18593 .as_ref()
18594 .unwrap()
18595 .fingerprint
18596 .clone();
18597 let name = crate::github_text::artifact_name(&fp);
18598 let path = crate::run::artifact_path(&runner.state, &name);
18599 assert!(q.detail.contains(&path.display().to_string()));
18600 assert!(q.detail.contains("title-language: title"));
18601 let saved = std::fs::read_to_string(&path).expect("artifact written");
18602 assert!(saved.contains("日本語のタイトル"));
18603 assert!(q.cwd.is_none());
18604 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18605 let say = |id: &str, text: &str, choice: &str| {
18607 store
18608 .update(id, |q| {
18609 q.thread.push(ask::Turn {
18610 who: ask::Who::Operator,
18611 body: text.to_owned(),
18612 at: jiff::Timestamp::now(),
18613 note: None,
18614 });
18615 q.answer(ask::Answer::Choice(choice.to_owned()))
18616 })
18617 .unwrap();
18618 };
18619 say(&q.id, "まだ日本語", "use my text");
18620 assert!(
18621 runner
18622 .guarded_pr_message(&winner, None, true)
18623 .await
18624 .unwrap()
18625 .is_none()
18626 );
18627 let second = find(&run);
18628 assert_ne!(second.id, q.id);
18629 assert!(second.detail.contains("did not pass"));
18630 assert!(second.detail.contains("日本語のタイトル"));
18633 assert!(!second.detail.contains("まだ日本語"));
18634 assert!(second.detail.contains(&path.display().to_string()));
18635 say(&second.id, "fix: retry failed requests", "use my text");
18636 let message = runner
18637 .guarded_pr_message(&winner, None, true)
18638 .await
18639 .unwrap()
18640 .expect("a vetted replacement is posted");
18641 assert_eq!(message.title, "fix: retry failed requests");
18642 assert_eq!(
18643 runner
18644 .state
18645 .github_text
18646 .as_ref()
18647 .unwrap()
18648 .chosen_title
18649 .as_deref(),
18650 Some("fix: retry failed requests")
18651 );
18652 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18653 let again = runner
18655 .guarded_pr_message(&winner, None, true)
18656 .await
18657 .unwrap()
18658 .unwrap();
18659 assert_eq!(again.title, message.title);
18660 assert_eq!(
18661 store
18662 .list()
18663 .into_iter()
18664 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18665 .count(),
18666 2
18667 );
18668 }
18669
18670 #[test]
18671 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18672 let mut state = state_with_summary(
18676 "add retries",
18677 "TITLE: fix(web): batch reads\n- reads run.json once",
18678 );
18679 state.config.graph.language = "ja".to_owned();
18680 let m = pr_message(&state, 'A');
18681 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18682 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18683
18684 let task = "今回やってほしいこと: results projector を直す";
18687 let mut state = state_with_summary(task, "- no title line");
18688 state.config.graph.language = "ja".to_owned();
18689 let m = pr_message(&state, 'A');
18690 assert_eq!(
18691 m.title,
18692 format!("chore: land candidate A of run {}", state.id)
18693 );
18694 assert!(
18695 m.body.contains(&format!(
18696 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18697 )),
18698 "{}",
18699 m.body
18700 );
18701 }
18702
18703 #[test]
18704 fn pr_message_scrubs_home_paths_and_addresses() {
18705 let state = state_with_summary(
18706 "fix it in /Users/someone/src/x",
18707 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18708 );
18709 let m = pr_message(&state, 'A');
18710 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18711 assert!(!m.body.contains(leak), "{}", m.body);
18712 }
18713 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18714 }
18715
18716 #[test]
18717 fn pr_message_survives_a_task_that_closes_details() {
18718 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18719 let m = pr_message(&state, 'A');
18720 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18721 }
18722
18723 #[test]
18724 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18725 let cmd = manual_merge_command(
18726 MergeStyle::Squash,
18727 Path::new("/repo"),
18728 "b",
18729 "fix: \"quoted\" $(x) `y`\n\nbody",
18730 );
18731 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18732 }
18733
18734 #[test]
18735 fn manual_merge_command_matches_the_configured_style() {
18736 let repo = Path::new("/repo");
18737 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18738
18739 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18740 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18741
18742 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18743 assert_eq!(
18744 squash,
18745 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18746 \"Merge magi run 0832 (candidate A)\""
18747 );
18748
18749 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18750 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18751 }
18752
18753 #[test]
18754 fn a_nudge_gets_a_quarter_of_the_budget() {
18755 assert_eq!(retry_budget(secs(1200), true), secs(300));
18757 assert_eq!(retry_budget(secs(3600), true), secs(900));
18758 }
18759
18760 #[test]
18761 fn a_resent_prompt_keeps_the_whole_budget() {
18762 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18765 assert_eq!(retry_budget(secs(60), false), secs(60));
18766 }
18767
18768 #[test]
18769 fn the_floor_never_exceeds_the_original_budget() {
18770 assert_eq!(retry_budget(secs(60), true), secs(60));
18774 assert_eq!(retry_budget(secs(480), true), secs(120));
18775 assert_eq!(retry_budget(secs(0), true), secs(0));
18776 }
18777
18778 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18779 agent::CommandEvidence {
18780 id: "item1".to_owned(),
18781 description: "cargo test".to_owned(),
18782 exit_code,
18783 result_summary: String::new(),
18784 source: "codex".to_owned(),
18785 }
18786 }
18787
18788 #[test]
18789 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18790 assert!(!has_unconfirmed_command(&[]));
18794 }
18795
18796 #[test]
18797 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18798 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18802 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18803 assert!(!has_unconfirmed_command(&[
18804 evidence(Some(0)),
18805 evidence(Some(101))
18806 ]));
18807 }
18808
18809 #[test]
18810 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18811 assert!(has_unconfirmed_command(&[
18812 evidence(Some(0)),
18813 evidence(None)
18814 ]));
18815 }
18816
18817 #[test]
18818 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18819 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18820 assert_eq!(
18821 verified_noop_claim(true, &[], text).as_deref(),
18822 Some("already fixed by b32cfc4, on main.")
18823 );
18824 }
18825
18826 #[test]
18827 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18828 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18831 assert!(verified_noop_claim(false, &[], text).is_none());
18832 }
18833
18834 #[test]
18835 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18836 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18837 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18838 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18840 }
18841
18842 #[test]
18843 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18844 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18845 }
18846
18847 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18850 runner.state.candidates = shape
18851 .iter()
18852 .enumerate()
18853 .map(|(i, &(empty, verified))| Candidate {
18854 index: i,
18855 label: (b'A' + i as u8) as char,
18856 agent: "sonnet".to_owned(),
18857 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18858 worktree: PathBuf::from(format!("/wt/{i}")),
18859 summary: String::new(),
18860 stat: String::new(),
18861 files: 0,
18862 commits: 0,
18863 empty,
18864 failed: None,
18865 verified_noop: verified.map(str::to_owned),
18866 duration_ms: 0,
18867 folded: false,
18868 })
18869 .collect();
18870 }
18871
18872 #[test]
18873 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18874 ask_test_home();
18875 let mut runner = runner_at(RunStatus::Implementing);
18876 set_candidates(
18877 &mut runner,
18878 &[
18879 (true, Some("already on main at b32cfc4")),
18880 (true, Some("same fix, see the existing test")),
18881 ],
18882 );
18883
18884 runner
18885 .after_implement()
18886 .expect("a verified no-op is not an error");
18887
18888 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18889 }
18890
18891 #[test]
18892 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18893 ask_test_home();
18894 let mut runner = runner_at(RunStatus::Implementing);
18895 set_candidates(
18899 &mut runner,
18900 &[(true, Some("already on main at b32cfc4")), (true, None)],
18901 );
18902
18903 let err = runner
18904 .after_implement()
18905 .expect_err("an unverified empty candidate must still fail the run");
18906
18907 assert!(
18908 err.to_string().contains("no candidate produced a change"),
18909 "{err}"
18910 );
18911 assert_eq!(runner.state.status, RunStatus::Failed);
18912 }
18913
18914 #[test]
18915 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18916 ask_test_home();
18917 let mut runner = runner_at(RunStatus::Implementing);
18918 set_candidates(&mut runner, &[(true, None), (true, None)]);
18919
18920 let err = runner
18921 .after_implement()
18922 .expect_err("no candidate declared anything; this is an ordinary failure");
18923
18924 assert!(
18925 err.to_string().contains("no candidate produced a change"),
18926 "{err}"
18927 );
18928 assert_eq!(runner.state.status, RunStatus::Failed);
18929 }
18930}
18931
18932#[derive(Debug)]
18935struct GraftRestoreFailed(String);
18936
18937impl std::fmt::Display for GraftRestoreFailed {
18938 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18939 write!(
18940 f,
18941 "could not restore the winner to its pre-graft commit: {}",
18942 self.0
18943 )
18944 }
18945}
18946
18947impl std::error::Error for GraftRestoreFailed {}
18948
18949#[cfg(test)]
18953mod resource_wait_tests {
18954 use super::*;
18955 use std::collections::BTreeMap;
18956
18957 fn sh(id: &str, script: &str) -> AgentSpec {
18958 AgentSpec {
18959 id: id.to_owned(),
18960 kind: crate::config::AgentKind::Command,
18961 model: None,
18962 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18963 extra_args: Vec::new(),
18964 env: BTreeMap::new(),
18965 prompt_delivery: None,
18966 max_turn_secs: None,
18967 }
18968 }
18969
18970 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18971 SeatJob {
18972 seat: SeatState::new("impl-A", &spec.id, 1),
18973 spec,
18974 cwd: dir.to_path_buf(),
18975 prompt: "do it".to_owned(),
18976 timeout,
18977 allow_write: true,
18978 sessions: false,
18979 artifacts: dir.join("art"),
18980 stem: "impl-A".to_owned(),
18981 handover: None,
18982 }
18983 }
18984
18985 fn state(resource_wait_secs: u64) -> RunState {
18986 let mut config = Config::default();
18987 config.graph.timeout_resource_wait = resource_wait_secs;
18988 RunState::new(
18989 PathBuf::from("/repo"),
18990 "main".to_owned(),
18991 "abc1234".to_owned(),
18992 "task".to_owned(),
18993 config,
18994 )
18995 }
18996
18997 fn hold(cache: &Path) -> crate::cache::Guard {
18998 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18999 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19000 crate::cache::AcquireOutcome::Acquired(g) => g,
19001 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19002 }
19003 }
19004
19005 async fn run(
19006 jobs: Vec<SeatJob>,
19007 sem: Arc<Semaphore>,
19008 cache: Option<&Path>,
19009 st: &mut RunState,
19010 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19011 let prompts = Prompts::default();
19012 let ctx = WaveCtx {
19013 run: "r",
19014 node: "implement",
19015 prompts: &prompts,
19016 cache,
19017 round: None,
19018 carry_seats: false,
19019 pauses: None,
19020 };
19021 wave(jobs, sem, &ctx, st, 0).await
19022 }
19023
19024 #[tokio::test]
19025 async fn verification_cache_wait_preserves_the_command_budget() {
19026 crate::run::pin_test_home();
19027 let tmp = tempfile::tempdir().unwrap();
19028 let cache = tmp.path().join("target");
19029 let guard = hold(&cache);
19030 let release = tokio::spawn(async move {
19031 tokio::time::sleep(Duration::from_millis(100)).await;
19032 guard.release();
19033 });
19034 let mut st = state(10);
19035 let budget = Duration::from_millis(50);
19036 let wait = repository_wait(&st, None);
19037 let (outcomes, _) = with_cache_lease(
19038 &mut st,
19039 Some(&cache),
19040 "gate",
19041 "gate",
19042 tmp.path(),
19043 "abc1234",
19044 budget,
19045 "test gate",
19046 &wait,
19047 |_, available| async move {
19048 assert_eq!(
19049 available, budget,
19050 "contention must not shorten verification"
19051 );
19052 (Vec::new(), false, Vec::new())
19053 },
19054 )
19055 .await
19056 .unwrap();
19057 release.await.unwrap();
19058 assert!(outcomes.is_empty(), "{outcomes:?}");
19059 assert!(
19060 st.events
19061 .iter()
19062 .any(|event| event.message.contains("waiting for"))
19063 );
19064 assert!(st.resource_wait.is_none());
19065 hold(&cache).release();
19067 }
19068
19069 #[tokio::test]
19070 async fn verification_cache_wait_expires_without_starting_commands() {
19071 crate::run::pin_test_home();
19072 let tmp = tempfile::tempdir().unwrap();
19073 let cache = tmp.path().join("target");
19074 let _held = hold(&cache);
19075 let mut st = state(0);
19076 let wait = repository_wait(&st, None);
19077 let (outcomes, _) = with_cache_lease(
19078 &mut st,
19079 Some(&cache),
19080 "gate",
19081 "gate",
19082 tmp.path(),
19083 "abc1234",
19084 Duration::from_secs(60),
19085 "test gate",
19086 &wait,
19087 |_, _| async { panic!("commands must not start without the cache lease") },
19088 )
19089 .await
19090 .unwrap();
19091 assert_eq!(outcomes.len(), 1);
19092 assert!(outcomes[0].resource_blocked);
19093 assert!(st.resource_wait.is_none());
19094 }
19095
19096 #[tokio::test]
19097 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19098 crate::run::pin_test_home();
19099 let tmp = tempfile::tempdir().unwrap();
19100 let cache = tmp.path().join("target-a");
19101 let guard = hold(&cache);
19102 tokio::spawn(async move {
19104 tokio::time::sleep(Duration::from_millis(1500)).await;
19105 guard.release();
19106 });
19107 let mut st = state(60);
19108 let results = run(
19109 vec![job(
19110 sh("alpha", "echo ok"),
19111 tmp.path(),
19112 Duration::from_secs(1),
19113 )],
19114 Arc::new(Semaphore::new(1)),
19115 Some(&cache),
19116 &mut st,
19117 )
19118 .await;
19119 assert!(
19120 matches!(results[0].2, AgentOutcome::Ok(_)),
19121 "the wait must not be charged to the agent"
19122 );
19123 assert!(st.resource_block.is_none());
19124 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19125 assert!(st.resource_wait.is_none() && st.active.is_empty());
19126 }
19127
19128 #[tokio::test]
19129 #[cfg(unix)]
19130 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19131 use std::os::fd::AsRawFd as _;
19132 crate::run::pin_test_home();
19133 let tmp = tempfile::tempdir().unwrap();
19134 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19135 let common = tmp.path().join(".git");
19136 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19137 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19138 assert_eq!(
19139 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19140 0
19141 );
19142 let mut st = state(1);
19143 st.config.coordination = crate::config::test_coordination();
19144 let mut reader = job(
19145 sh("reader", "echo must-not-run"),
19146 tmp.path(),
19147 Duration::from_secs(30),
19148 );
19149 reader.allow_write = false;
19150 let results = run(
19151 vec![reader.clone()],
19152 Arc::new(Semaphore::new(1)),
19153 None,
19154 &mut st,
19155 )
19156 .await;
19157 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19158 assert_eq!(results[0].1.turns, 0);
19159 assert!(FailClass::of(&results[0].2).is_none());
19160 assert_eq!(
19161 st.resource_block.as_ref().unwrap().kind,
19162 "repository coordination"
19163 );
19164 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19165 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19166 drop(external);
19167 let editor = crate::repo_lock::SeatLease::for_checkout(
19168 tmp.path(),
19169 true,
19170 crate::config::test_coordination().as_ref(),
19171 )
19172 .await
19173 .unwrap();
19174 let pause = Pause::new();
19175 let interrupt = Pause::new();
19176 let prompts = Prompts::default();
19177 let ctx = WaveCtx {
19178 run: "r",
19179 node: "review",
19180 prompts: &prompts,
19181 cache: None,
19182 round: None,
19183 carry_seats: false,
19184 pauses: Some((&pause, &interrupt)),
19185 };
19186 let (results, ()) = tokio::join!(
19187 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19188 async {
19189 tokio::time::sleep(Duration::from_millis(50)).await;
19190 pause.park();
19191 }
19192 );
19193 assert!(
19194 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19195 );
19196 assert_eq!(results[0].1.turns, 0);
19197 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19198 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19199 drop(editor);
19200 }
19201
19202 #[tokio::test]
19203 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19204 crate::run::pin_test_home();
19205 let tmp = tempfile::tempdir().unwrap();
19206 let cache = tmp.path().join("target-b");
19207 let _held = hold(&cache);
19208 let mut st = state(1);
19209 let results = run(
19210 vec![job(
19211 sh("alpha", "echo ok"),
19212 tmp.path(),
19213 Duration::from_secs(1),
19214 )],
19215 Arc::new(Semaphore::new(1)),
19216 Some(&cache),
19217 &mut st,
19218 )
19219 .await;
19220 let (_, seat, out) = &results[0];
19221 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19222 assert!(wave_blocked(&results));
19223 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19224 assert_eq!(seat.turns, 0, "nothing was asked");
19225 assert!(
19226 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19227 "no agent was spawned"
19228 );
19229 assert!(FailClass::of(out).is_none(), "not a failure class");
19230 let block = st.resource_block.as_ref().expect("recorded for the resume");
19231 assert_eq!(block.kind, "shared build cache");
19232 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19233 assert!(st.active.is_empty());
19234 }
19235
19236 #[tokio::test]
19237 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19238 crate::run::pin_test_home();
19239 let tmp = tempfile::tempdir().unwrap();
19240 let mut st = state(1);
19241 let results = run(
19242 vec![job(
19243 sh("alpha", "echo ok"),
19244 tmp.path(),
19245 Duration::from_secs(1),
19246 )],
19247 Arc::new(Semaphore::new(0)),
19248 None,
19249 &mut st,
19250 )
19251 .await;
19252 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19253 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19254 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19255 }
19256
19257 #[tokio::test]
19258 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19259 crate::run::pin_test_home();
19260 let tmp = tempfile::tempdir().unwrap();
19261 let mut st = state(60);
19262 let results = run(
19263 vec![job(
19264 sh("alpha", "sleep 30"),
19265 tmp.path(),
19266 Duration::from_secs(1),
19267 )],
19268 Arc::new(Semaphore::new(1)),
19269 None,
19270 &mut st,
19271 )
19272 .await;
19273 let out = &results[0].2;
19274 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19275 let class = FailClass::of(out).expect("a failure");
19276 assert_eq!(class, FailClass::Timeout);
19277 assert!(should_hand_over(None, &class), "existing handover policy");
19278 assert!(st.resource_block.is_none());
19279 }
19280}