1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
488 let tracking = format!("{remote}/{base_branch}");
489 let fetched = git::fetch(repo, remote, base_branch).await;
490 if let Ok(out) = &fetched
491 && out.ok()
492 && git::rev_exists(repo, &tracking).await
493 {
494 return git::rev_parse(repo, &tracking).await;
495 }
496 let why = match &fetched {
497 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
498 Ok(_) => format!("{remote} has no {base_branch}"),
499 Err(e) => e.to_string(),
500 };
501 bail!(
504 "cannot read {tracking} ({why}); refusing to branch off the local \
505 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
506 base / remote in magi.toml"
507 )
508}
509
510struct FixClaim {
527 path: PathBuf,
528}
529
530impl FixClaim {
531 fn acquire(dir: &Path) -> Result<Self> {
532 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
533 let path = dir.join("fix.lock");
534 match Self::create(&path) {
535 Ok(claim) => Ok(claim),
536 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
537 if Self::reclaim_if_dead(&path) {
538 Self::create(&path).with_context(|| format!("lock {}", path.display()))
539 } else {
540 bail!(
541 "another repair or `magi fix` is already running for this run ({} exists)",
542 path.display()
543 )
544 }
545 }
546 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
547 }
548 }
549
550 fn create(path: &Path) -> std::io::Result<Self> {
551 let mut f = std::fs::OpenOptions::new()
552 .write(true)
553 .create_new(true)
554 .open(path)?;
555 use std::io::Write as _;
556 writeln!(f, "{}", std::process::id())?;
558 Ok(Self {
559 path: path.to_owned(),
560 })
561 }
562
563 fn reclaim_if_dead(path: &Path) -> bool {
567 let dead = std::fs::read_to_string(path)
568 .ok()
569 .and_then(|body| body.trim().parse::<u32>().ok())
570 .is_some_and(|pid| !crate::proc::pid_alive(pid));
571 dead && std::fs::remove_file(path).is_ok()
572 }
573}
574
575impl Drop for FixClaim {
576 fn drop(&mut self) {
577 let _ = std::fs::remove_file(&self.path);
578 }
579}
580
581impl Runner {
582 pub async fn start(
584 repo: &Path,
585 instruction: String,
586 config: Config,
587 origin: Origin,
588 ) -> Result<Self> {
589 Self::start_naming(repo, instruction, "", config, origin).await
590 }
591
592 pub async fn start_naming(
596 repo: &Path,
597 instruction: String,
598 also_scan: &str,
599 config: Config,
600 origin: Origin,
601 ) -> Result<Self> {
602 let repo = git::toplevel(repo).await?;
603 let missing = agent::missing_programs(&config.agents);
604 if !missing.is_empty() {
605 bail!(
606 "these agent programs are not on PATH: {}. Fix the roster in \
607 magi.toml or install them.",
608 missing.join(", ")
609 );
610 }
611 let base_branch =
612 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
613 .await?;
614 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
615 let roles = config.resolve_roles()?;
616 let max_parallel = config.graph.max_parallel.max(1);
617 let seeds = refs::resolve(
620 &repo,
621 &base_commit,
622 &config.merge.remote,
623 &format!("{also_scan}\n{instruction}"),
624 )
625 .await;
626 refs::plan(&repo, &seeds).await?;
627 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
628 state.origin = Some(origin);
632 for seed in &seeds {
633 state.event(
634 "seed",
635 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
636 );
637 }
638 state.seeds = seeds;
639 state.event("start", format!("run {} created", state.id));
640 state.save()?;
641 Ok(Self {
642 state,
643 roles,
644 sem: Arc::new(Semaphore::new(max_parallel)),
645 pause: Pause::new(),
646 interrupt: Pause::new(),
647 })
648 }
649
650 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
664 Self::review_taking_over(repo, branch, config, None, origin).await
665 }
666
667 pub async fn review_taking_over(
674 repo: &Path,
675 branch: &str,
676 config: Config,
677 takeover: Option<crate::handover::Takeover>,
678 origin: Origin,
679 ) -> Result<Self> {
680 let repo = git::toplevel(repo).await?;
681 let missing = agent::missing_programs(&config.agents);
682 if !missing.is_empty() {
683 bail!(
684 "these agent programs are not on PATH: {}. Fix the roster in \
685 magi.toml or install them.",
686 missing.join(", ")
687 );
688 }
689 let base_branch =
690 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
691 .await?;
692 if base_branch == branch {
693 bail!("`{branch}` is the base branch; there is nothing to review against");
694 }
695 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
696
697 let roles = config.resolve_roles()?;
698 let max_parallel = config.graph.max_parallel.max(1);
699 let mut state = RunState::new(
700 repo.clone(),
701 base_branch,
702 base_commit.clone(),
703 String::new(),
704 config,
705 );
706 state.origin = Some(origin);
707
708 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
715 earlier: Vec::new(),
716 home: crate::run::home(),
717 choice: None,
718 task_instruction: None,
719 });
720 let wait = repository_wait(&state, None);
721 let coordination = crate::repo_lock::SeatLease::operation_with(&repo, &wait).await?;
722 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
723 if let Some(released) = &released {
724 state.event(
725 "release",
726 format!(
727 "took `{branch}` over from run {}: its worktree was released: {}",
728 crate::run::short_of(&released.old_id),
729 released.audit
730 ),
731 );
732 }
733 if let Some(choice) = takeover.choice.as_ref()
737 && let Err(e) =
738 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
739 .await
740 {
741 if let Some(released) = &released {
742 released.restore(&repo, branch).await;
743 }
744 return Err(e.context("applying the owner's answer about the diverged branch"));
745 }
746 for id in takeover.earlier.iter().rev() {
752 if let Ok(old) = RunState::load_under(id, &takeover.home)
753 && let Some(head) = old.pr_head.clone()
754 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
755 {
756 state.pr_head = Some(head);
757 break;
758 }
759 }
760 drop(coordination);
761 let opened = Self::open_review(
762 &repo,
763 branch,
764 state,
765 roles,
766 max_parallel,
767 base_commit,
768 &takeover,
769 )
770 .await;
771 if opened.is_err()
772 && let Some(released) = &released
773 {
774 let _coordination = crate::repo_lock::SeatLease::operation_with(&repo, &wait).await?;
775 released.restore(&repo, branch).await;
776 }
777 opened
778 }
779
780 pub async fn recover_pr(
782 repo: &Path,
783 task: &str,
784 selection: crate::pr_recovery::Selection,
785 config: Config,
786 ) -> Result<Self> {
787 selection.check_task(task, &selection.instruction)?;
788 let base_branch =
789 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
790 .await?;
791 let base_commit = resolve_base(repo, &base_branch, &config.merge.remote).await?;
792 selection
793 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
794 .await?;
795 let roles = config.resolve_roles()?;
796 let max_parallel = config.graph.max_parallel.max(1);
797 let mut state = RunState::new(
798 repo.to_path_buf(),
799 base_branch,
800 base_commit.clone(),
801 selection.instruction.clone(),
802 config,
803 );
804 state.origin = Some(crate::run::Origin::queue(task));
805 state.pr_head = Some(selection.branch.clone());
806 state.pr_recovery = Some(selection.clone());
807 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
808 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
809 }
810
811 async fn open_selected(
812 repo: &Path,
813 state: RunState,
814 roles: ResolvedRoles,
815 max_parallel: usize,
816 base_commit: String,
817 selection: crate::pr_recovery::Selection,
818 ) -> Result<Self> {
819 let mut state = state;
820 let branch = state.branch_for('A');
821 let wait = repository_wait(&state, None);
822 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
823 git::git(repo, &["branch", &branch, &selection.head]).await?;
824 drop(coordination);
825 state.event(
826 "recovery",
827 format!(
828 "selected public implementation {} from {}",
829 selection.head, selection.url
830 ),
831 );
832 let takeover = crate::handover::Takeover {
833 earlier: Vec::new(),
834 home: crate::run::home(),
835 choice: None,
836 task_instruction: Some(selection.instruction.clone()),
837 };
838 state.save()?;
839 let id = state.id.clone();
840 let opened = Self::open_review(
841 repo,
842 &branch,
843 state,
844 roles,
845 max_parallel,
846 base_commit,
847 &takeover,
848 )
849 .await;
850 match opened {
851 Ok(runner) => Ok(runner),
852 Err(e) => {
853 if let Ok(mut partial) = RunState::load(&id) {
854 partial.status = RunStatus::Blocked;
855 partial.driver_exited = true;
856 partial.event(
857 "recovery",
858 format!("startup failed; retained source and candidate {branch}: {e:#}"),
859 );
860 partial.save()?;
861 }
862 Err(crate::pr_recovery::StartupFailed {
863 run: id,
864 cause: format!("{e:#}"),
865 }
866 .into())
867 }
868 }
869 }
870
871 async fn inherit(
876 repo: &Path,
877 branch: &str,
878 base_commit: &str,
879 takeover: &crate::handover::Takeover,
880 ) -> Option<crate::recovery::RecoveryRecord> {
881 use crate::recovery::{Relation, assess};
882 let newest = takeover.earlier.last()?;
883 let mut chosen = None;
884 for id in takeover.earlier.iter().rev() {
885 if let Ok(old) = RunState::load_under(id, &takeover.home)
886 && !old.reviews.is_empty()
887 && (old.winner().is_some_and(|w| w.branch == branch)
888 || old.pr_head.as_deref() == Some(branch))
889 {
890 chosen = Some(old);
891 break;
892 }
893 }
894 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
895 .await
896 .unwrap_or_default();
897 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
898 Some(last) if last.head == tip => Relation::Same,
899 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
900 _ => Relation::Diverged,
901 };
902 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
903 Some(assess(from, chosen.as_ref(), base_commit, relation))
904 }
905
906 async fn open_review(
909 repo: &Path,
910 branch: &str,
911 mut state: RunState,
912 roles: ResolvedRoles,
913 max_parallel: usize,
914 base_commit: String,
915 takeover: &crate::handover::Takeover,
916 ) -> Result<Self> {
917 let wait = repository_wait(&state, None);
918 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
919 if state.pr_recovery.is_none() {
920 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
921 }
922 let start = review_base(
925 repo,
926 &state.config.merge.remote,
927 &state.base_branch,
928 &base_commit,
929 branch,
930 )
931 .await;
932 let log = git::log_oneline(repo, &start, branch)
933 .await
934 .unwrap_or_default();
935 let log_text = if log.trim().is_empty() {
936 "(no commit messages)"
937 } else {
938 log.trim()
939 };
940 let mut instruction = format!(
941 "Review the work already on branch `{branch}`. There is no task \
942 statement: what the change claims to do is whatever its commits \
943 say.\n\n{log_text}"
944 );
945 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
949 if let Some(task) = takeover
950 .task_instruction
951 .as_deref()
952 .filter(|t| !t.trim().is_empty())
953 && rec.reviewed_head.is_some()
954 {
955 instruction = format!(
956 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
957 task.trim()
958 );
959 rec.criteria_inherited = true;
960 }
961 state.event("recovery", rec.summary());
962 state.recovery = Some(rec);
963 }
964 if let Some(selected) = &state.pr_recovery {
965 instruction = selected.instruction.clone();
966 }
967 state.task_instruction = takeover.task_instruction.clone();
968 state.instruction = instruction;
969 state.reviewed_commits = Some(
970 git::subjects(repo, &start, branch)
971 .await
972 .unwrap_or_default(),
973 );
974
975 let worktree = state.worktree_root().join("under-review");
978 if let Some(parent) = worktree.parent() {
979 tokio::fs::create_dir_all(parent).await.ok();
980 }
981 let path = worktree.to_string_lossy().to_string();
982 git::git(repo, &["worktree", "add", &path, branch])
983 .await
984 .with_context(|| {
985 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
986 })?;
987
988 let setup = match ensure_setup_config(&mut state, repo).await {
989 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
990 Err(e) => Err(e),
991 };
992 if let Err(e) = setup {
993 git::worktree_remove(repo, &worktree).await.ok();
994 return Err(e);
995 }
996 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
997 .await
998 .unwrap_or(0);
999 if commits == 0 {
1000 git::worktree_remove(repo, &worktree).await.ok();
1001 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1002 }
1003 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1004 .await
1005 .map(|f| f.len())
1006 .unwrap_or(0);
1007 if files == 0
1008 && let (Ok(head_tree), Ok(base_tree)) = (
1009 git::tree_of(&worktree, "HEAD").await,
1010 git::tree_of(&worktree, &base_commit).await,
1011 )
1012 && head_tree == base_tree
1013 {
1014 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1015 git::worktree_remove(repo, &worktree).await.ok();
1016 bail!(
1017 "`{branch}` at {} has a tree identical to base {}; this usually means \
1018 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1019 against `{}/{branch}`) rather than an empty change",
1020 short(&head),
1021 short(&base_commit),
1022 state.config.merge.remote
1023 );
1024 }
1025 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1026 .await
1027 .unwrap_or_default();
1028
1029 state.candidates.push(Candidate {
1030 index: 0,
1031 label: 'A',
1032 agent: EXISTING_BRANCH.to_owned(),
1035 branch: branch.to_owned(),
1036 worktree,
1037 summary: String::new(),
1038 stat,
1039 files,
1040 commits,
1041 empty: false,
1042 failed: None,
1043 verified_noop: None,
1044 duration_ms: 0,
1045 folded: false,
1046 });
1047 state.tally = Some(Tally {
1048 first_choice: BTreeMap::from([('A', 0)]),
1049 borda: BTreeMap::new(),
1050 winner: 'A',
1051 rankings: 0,
1052 unanimous_initial: false,
1053 deliberated: false,
1054 changed_votes: 0,
1055 unanimous_final: false,
1056 tie_break: None,
1057 judges: 0,
1061 present: 0,
1062 quorum: 0,
1063 met_quorum: true,
1064 uncontested: Some("review-only run: nothing competed".to_owned()),
1065 });
1066 state.status = RunStatus::Reviewing;
1067 state.prep_complete = Some(true);
1068 state.event(
1069 "start",
1070 format!(
1071 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1072 state.id
1073 ),
1074 );
1075 state.save()?;
1076 Ok(Self {
1077 state,
1078 roles,
1079 sem: Arc::new(Semaphore::new(max_parallel)),
1080 pause: Pause::new(),
1081 interrupt: Pause::new(),
1082 })
1083 }
1084
1085 pub fn resume(id: &str) -> Result<Self> {
1087 let state = RunState::load(id)?;
1088 crate::pr_head::ensure_owner(&state)?;
1089 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1090 bail!(
1091 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1092 );
1093 }
1094 if let Some(to) = &state.released_to {
1095 bail!(
1096 "run {} cannot be resumed: its worktree was released to run {}",
1097 state.short(),
1098 crate::run::short_of(to)
1099 );
1100 }
1101 let roles = state.config.resolve_roles()?;
1102 if let Some(id) = &state.config.graph.selected_implementer
1103 && state.candidates.iter().any(|c| c.agent != *id)
1104 {
1105 bail!(
1106 "replacement implementer pin does not match the saved candidates; task must stay held"
1107 );
1108 }
1109 let max_parallel = state.config.graph.max_parallel.max(1);
1110 Ok(Self {
1111 state,
1112 roles,
1113 sem: Arc::new(Semaphore::new(max_parallel)),
1114 pause: Pause::new(),
1115 interrupt: Pause::new(),
1116 })
1117 }
1118
1119 pub async fn execute(&mut self) -> Result<()> {
1126 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1129 || self.state.land_approval.is_some()
1130 || self.state.merge_decision.is_some()
1131 || !self.state.merge_repairs.is_empty()
1132 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1133 {
1134 let claim = FixClaim::acquire(&self.state.dir())?;
1135 let disk = RunState::load(&self.state.id)?;
1136 let checkpoint = |s: &RunState| {
1137 (
1138 s.merge_repairs.len(),
1139 s.merge_repairs
1140 .last()
1141 .map(|r| (r.question.clone(), r.phase)),
1142 s.land_approval
1143 .as_ref()
1144 .map(|a| (a.question.clone(), a.head.clone())),
1145 )
1146 };
1147 if checkpoint(&disk) != checkpoint(&self.state) {
1148 bail!(
1149 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1150 );
1151 }
1152 Some(claim)
1153 } else {
1154 None
1155 };
1156 let wait = self.repository_wait();
1157 let result = wait.scope(self.execute_graph()).await;
1158 let result = match result {
1159 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1160 record_repository_block(&mut self.state, &e);
1161 self.park_here()?;
1162 Ok(())
1163 }
1164 other => other,
1165 };
1166 self.mark_driver_exited();
1167 let ended = if result.is_err() {
1168 Some(crate::notices::run_stopped(&self.state.id, &self.state))
1169 } else {
1170 crate::notices::run_ended(&self.state)
1171 };
1172 if let Some(notice) = ended {
1173 crate::notices::raise(notice);
1174 }
1175 result
1176 }
1177
1178 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1179 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1180 }
1181
1182 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1183 let wait = self.repository_wait();
1184 coordinate_operation(&mut self.state, cwd, &wait).await
1185 }
1186
1187 fn mark_driver_exited(&mut self) {
1196 self.state.driver_exited = true;
1197 let pid = std::process::id();
1198 let Ok(_lock) = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) else {
1199 return;
1200 };
1201 let Ok(mut disk) = RunState::load(&self.state.id) else {
1202 return;
1203 };
1204 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
1205 return;
1206 }
1207 disk.driver_exited = true;
1208 if let Err(e) = disk.save() {
1209 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
1210 }
1211 }
1212
1213 fn claim_driver(&mut self) -> Result<()> {
1215 let _lock = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1216 let disk = RunState::load(&self.state.id)?;
1217 crate::pr_head::ensure_owner(&disk)?;
1218 self.state.parked = false;
1223 self.state.resource_wait = None;
1224 self.state.resource_block = None;
1225 self.state.clear_active();
1232 if let Some(to) = &disk.released_to {
1250 bail!(
1251 "run {} cannot continue: its worktree was released to run {}",
1252 self.state.short(),
1253 crate::run::short_of(to)
1254 );
1255 }
1256 let pid = std::process::id();
1257 self.state.driver_pid = Some(pid);
1258 self.state.driver_started_at = crate::proc::process_started_at(pid);
1259 self.state.driver_exited = false;
1260 self.state.begin_pass();
1261 self.state.save()?;
1262 Ok(())
1263 }
1264
1265 async fn execute_graph(&mut self) -> Result<()> {
1266 self.claim_driver()?;
1267 if self
1268 .state
1269 .merge_decision
1270 .as_ref()
1271 .is_some_and(|decision| !decision.closed)
1272 && self.state.status != RunStatus::Landing
1273 {
1274 if crate::merge_decision::resumable_hold(&self.state) {
1275 self.state.merge = Some(MergeOutcome {
1276 mode: MergeMode::Pr,
1277 ok: false,
1278 detail: self
1279 .state
1280 .merge_decision
1281 .as_ref()
1282 .unwrap()
1283 .target
1284 .pr
1285 .clone(),
1286 empty: false,
1287 });
1288 self.run_land().await?;
1289 self.settle_questions();
1290 return Ok(());
1291 }
1292 bail!(
1293 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1294 );
1295 }
1296 if self.state.config.graph.selected_implementer.is_some()
1297 && self.state.status == RunStatus::Blocked
1298 && self.state.tally.is_none()
1299 {
1300 bail!(
1301 "selected implementation is held; explicitly release the owning task to retry it"
1302 );
1303 }
1304
1305 if self.state.merge_repairs.last().is_some_and(|r| {
1306 matches!(
1307 r.phase,
1308 crate::run::MergeRepairPhase::Requested
1309 | crate::run::MergeRepairPhase::Fixing
1310 | crate::run::MergeRepairPhase::Reviewing
1311 )
1312 }) {
1313 if self.repair_merge_review().await? {
1314 self.run_land().await?;
1315 }
1316 self.settle_questions();
1317 return Ok(());
1318 }
1319 if self.state.pr_publication_pending {
1322 self.merge().await?;
1323 if self.resource_halt()? {
1324 return Ok(());
1325 }
1326 self.state.save()?;
1327 self.settle_questions();
1328 return Ok(());
1329 }
1330 if self.state.status == RunStatus::Stalled {
1343 if self.recover_stall().await? {
1344 self.finish_after_tally().await?;
1345 } else {
1346 self.state.save()?;
1348 }
1349 return Ok(());
1350 }
1351 if self.state.status == RunStatus::Landing {
1361 self.run_land().await?;
1362 self.settle_questions();
1367 return Ok(());
1368 }
1369 if self.state.github_text.is_some()
1374 && self.state.merge.is_none()
1375 && self.state.status == RunStatus::Gating
1376 {
1377 self.merge().await?;
1378 if self.resource_halt()? {
1379 return Ok(());
1380 }
1381 self.state.save()?;
1382 return Ok(());
1383 }
1384 if let Err(e) = self.prep().await {
1385 if e.is::<crate::repo_lock::ResourceBlocked>() {
1386 return Err(e);
1387 }
1388 self.state.status = RunStatus::Blocked;
1389 self.state
1390 .prep_failure
1391 .get_or_insert_with(|| format!("{e:#}"));
1392 self.state
1393 .event("prep", format!("preparation stopped: {e:#}"));
1394 self.state.save()?;
1395 return Err(e);
1396 }
1397 if self.park_here()? {
1398 return Ok(());
1399 }
1400 self.advise().await?;
1401 if self.park_here()? {
1402 return Ok(());
1403 }
1404 self.implement().await?;
1405 if self.park_here()? {
1406 return Ok(());
1407 }
1408 if self.state.status == RunStatus::VerifiedNoop {
1412 return Ok(());
1413 }
1414 self.judge().await?;
1415 if self.park_here()? {
1416 return Ok(());
1417 }
1418 self.deliberate().await?;
1419 if self.park_here()? {
1420 return Ok(());
1421 }
1422 self.vote().await?;
1423 if self.park_here()? {
1424 return Ok(());
1425 }
1426 self.tally()?;
1427 if self.state.status == RunStatus::Stalled {
1432 self.state.save()?;
1436 return Ok(());
1437 }
1438 self.finish_after_tally().await?;
1439 Ok(())
1440 }
1441
1442 fn stop_for_resource(&mut self) -> Result<()> {
1447 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1449 {
1450 self.state.status = RunStatus::Blocked;
1451 }
1452 self.state.save()
1453 }
1454
1455 fn resource_halt(&mut self) -> Result<bool> {
1457 if self.state.resource_block.is_none() {
1458 return Ok(false);
1459 }
1460 self.stop_for_resource()?;
1461 Ok(true)
1462 }
1463
1464 fn park_here(&mut self) -> Result<bool> {
1471 let parking = self.pause.parked() || self.interrupt.parked();
1472 if !parking && self.state.resource_block.is_some() {
1473 self.stop_for_resource()?;
1474 return Ok(true);
1475 }
1476 if !self.pause.parked() && !self.interrupt.parked() {
1481 return Ok(false);
1482 }
1483 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1484 let why = match cause.as_ref().map(ParkCause::label) {
1485 Some(reason) => format!(
1486 "parked after `{}` ({reason}) — resume to carry on from here",
1487 self.state.status.as_str()
1488 ),
1489 None => format!(
1490 "parked after `{}` — resume to carry on from here",
1491 self.state.status.as_str()
1492 ),
1493 };
1494 self.state.event("park", why);
1495 self.state
1496 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1497 self.state.parked = true;
1498 self.state.save()?;
1499 Ok(true)
1500 }
1501
1502 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1504 if self.state.prep_failure.is_none() {
1505 return Ok(());
1506 }
1507 let failed_at = self
1508 .state
1509 .events
1510 .iter()
1511 .rev()
1512 .find(|event| event.node == "prep")
1513 .map_or(self.state.updated_at, |event| event.at);
1514 anyhow::ensure!(
1515 approval_at > failed_at,
1516 "preparation retry requires a new explicit release after the failure"
1517 );
1518 self.state.prep_failure = None;
1519 self.state.prep_complete = Some(false);
1520 self.state
1521 .event("prep-retry", "operator approved retry of preparation");
1522 Ok(())
1523 }
1524
1525 pub fn on_pause(&mut self, pause: Pause) {
1527 self.pause = pause;
1528 }
1529
1530 pub fn watch_interrupt(&mut self, pause: Pause) {
1536 self.interrupt = pause;
1537 }
1538
1539 fn settle_questions(&mut self) {
1559 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1560 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1561 }
1562 }
1563
1564 async fn finish_after_tally(&mut self) -> Result<()> {
1567 self.graft().await?;
1570 if self.resource_halt()? {
1571 return Ok(());
1572 }
1573 self.fold_losers().await?;
1574 self.sync_to_base().await?;
1579 if self.state.status == RunStatus::AlreadyInBase {
1580 return Ok(());
1581 }
1582 self.review_loop().await?;
1583 if self.resource_halt()? {
1584 return Ok(());
1585 }
1586 self.sync_to_base().await?;
1587 if self.state.status == RunStatus::AlreadyInBase {
1588 return Ok(());
1589 }
1590 self.gate().await?;
1591 self.merge().await?;
1592 if self.resource_halt()? {
1593 return Ok(());
1594 }
1595 self.state.save()?;
1596 Ok(())
1597 }
1598
1599 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1606 let result = match ensure_setup_config(&mut self.state, repo).await {
1607 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1608 Err(e) => Err(e),
1609 };
1610 if let Err(e) = result {
1611 git::worktree_remove(repo, wt).await.ok();
1612 if in_prep {
1613 self.rollback_prep(repo).await;
1614 }
1615 self.state.status = RunStatus::Blocked;
1616 self.state.event(
1617 "setup",
1618 format!("worktree setup failed in {}: {e:#}", wt.display()),
1619 );
1620 self.state.save()?;
1621 return Err(e);
1622 }
1623 Ok(())
1624 }
1625
1626 async fn rollback_prep(&mut self, repo: &Path) {
1628 let root = self.state.worktree_root();
1629 for c in std::mem::take(&mut self.state.candidates) {
1630 git::worktree_remove(repo, &c.worktree).await.ok();
1631 git::branch_delete(repo, &c.branch).await.ok();
1632 }
1633 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1634 for j in 1..=seats {
1635 git::worktree_remove(repo, &root.join(format!("judge-{j}")))
1636 .await
1637 .ok();
1638 }
1639 for k in 1..=self.state.config.graph.advisors {
1640 git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
1641 .await
1642 .ok();
1643 }
1644 }
1645
1646 async fn prep(&mut self) -> Result<()> {
1647 if let Some(reason) = &self.state.prep_failure {
1648 bail!(
1649 "candidate preparation previously failed: {reason}; correct the source before retrying"
1650 );
1651 }
1652 if self
1653 .state
1654 .prep_complete
1655 .unwrap_or(!self.state.candidates.is_empty())
1656 {
1657 return Ok(());
1658 }
1659 let repo = self.state.repo.clone();
1660 let _coordination = self.coordinate(&repo).await?;
1663 self.rollback_prep(&repo).await;
1664 self.state.prep_complete = Some(false);
1665 self.state.status = RunStatus::Prep;
1666 self.state.save()?;
1667 let repo = self.state.repo.clone();
1668 let base = self.state.base_commit.clone();
1669 let plan = refs::plan(&repo, &self.state.seeds).await?;
1670 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1671 let root = self.state.worktree_root();
1672 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1673
1674 let hooks_dir = self.state.dir().join("hooks");
1677 if self.state.config.blind.commit_msg_hook {
1678 std::fs::create_dir_all(&hooks_dir)
1679 .with_context(|| format!("create {}", hooks_dir.display()))?;
1680 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1681 let path = hooks_dir.join("commit-msg");
1682 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1683 make_executable(&path)?;
1684 git::acquire_worktree_config(&repo).await?;
1692 self.state.enabled_worktree_config = true;
1693 }
1694
1695 ensure_setup_config(&mut self.state, &repo).await?;
1698
1699 for (index, (spec, label)) in self
1700 .roles
1701 .implementers
1702 .clone()
1703 .into_iter()
1704 .zip(labels)
1705 .enumerate()
1706 {
1707 let branch = self.state.branch_for(label);
1708 let worktree = root.join(format!("cand-{label}"));
1709 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1710 if self.state.config.blind.commit_msg_hook {
1711 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1712 }
1713 git::local_exclude(&worktree, "/.magi/").await?;
1714 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1715 git::worktree_remove(&repo, &worktree).await.ok();
1718 git::branch_delete(&repo, &branch).await.ok();
1719 self.rollback_prep(&repo).await;
1720 self.state.status = RunStatus::Blocked;
1721 self.state
1722 .event("prep", format!("worktree setup failed: {e:#}"));
1723 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1724 self.state.save()?;
1725 return Err(e);
1726 }
1727 self.state.candidates.push(Candidate {
1728 index,
1729 label,
1730 agent: spec.id.clone(),
1731 branch,
1732 worktree,
1733 summary: String::new(),
1734 stat: String::new(),
1735 files: 0,
1736 commits: 0,
1737 empty: false,
1738 failed: None,
1739 verified_noop: None,
1740 duration_ms: 0,
1741 folded: false,
1742 });
1743 self.state.save()?;
1744 for pick in &plan.picks {
1745 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1746 {
1747 self.state.status = RunStatus::Blocked;
1748 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1749 self.state
1750 .event("prep", format!("cannot apply referenced commit: {e}"));
1751 self.state.save()?;
1752 return Err(e);
1753 }
1754 }
1755 }
1756
1757 for j in 1..=self.roles.judges.len() {
1758 let wt = root.join(format!("judge-{j}"));
1759 if !wt.exists() {
1760 git::worktree_add_detached(&repo, &wt, &base).await?;
1761 self.setup_seat_worktree(&repo, &wt, true).await?;
1762 }
1763 }
1764
1765 if self.state.config.graph.advise {
1773 for k in 1..=self.state.config.graph.advisors {
1774 let wt = root.join(format!("advisor-{k}"));
1775 if !wt.exists() {
1776 git::worktree_add_detached(&repo, &wt, &base).await?;
1777 self.setup_seat_worktree(&repo, &wt, true).await?;
1778 }
1779 }
1780 }
1781
1782 let authors: Vec<&str> = self
1787 .roles
1788 .implementers
1789 .iter()
1790 .map(|a| a.id.as_str())
1791 .collect();
1792 let overlap: Vec<String> = self
1793 .roles
1794 .judges
1795 .iter()
1796 .enumerate()
1797 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1798 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1799 .collect();
1800 if !overlap.is_empty() {
1801 let note = format!(
1802 "{} also authored a candidate; blind, but the panel is less \
1803 independent than {} distinct agents would be",
1804 overlap.join(", "),
1805 self.roles.judges.len()
1806 );
1807 self.state.event("prep", note);
1808 }
1809
1810 self.state.event(
1811 "prep",
1812 format!(
1813 "{} candidates, {} judges, base {} ({})",
1814 self.state.candidates.len(),
1815 self.roles.judges.len(),
1816 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1817 self.state.base_branch
1818 ),
1819 );
1820 self.state.prep_complete = Some(true);
1821 self.state.status = RunStatus::Implementing;
1822 self.state.save()?;
1823 Ok(())
1824 }
1825
1826 async fn advise(&mut self) -> Result<()> {
1859 let implement_untouched = self
1860 .state
1861 .candidates
1862 .iter()
1863 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1864 if !self.state.config.graph.advise || self.state.advise_attempted {
1865 return Ok(());
1866 }
1867 if !implement_untouched {
1868 self.state.event(
1869 "advise",
1870 "skipping the design-deliberation stage: at least one \
1871 candidate already shows implementation progress, so this \
1872 run is past the point the stage exists to run before"
1873 .to_owned(),
1874 );
1875 self.state.advise_attempted = true;
1876 self.state.save()?;
1877 return Ok(());
1878 }
1879 let run_id = self.state.id.clone();
1880 let prompts = self.state.config.prompts.clone();
1881 let instruction = self.state.instruction.clone();
1882 let language = self.state.config.graph.language.clone();
1883 let root = self.state.worktree_root();
1884 let n = self.state.config.graph.advisors;
1885 let where_recorded = self.state.dir().join("run.json");
1886
1887 let seats = match self.state.config.advisors() {
1888 Ok(seats) if !seats.is_empty() => seats,
1889 Ok(_) => {
1890 self.state.event(
1891 "advise",
1892 format!(
1893 "[graph] advisors is 0; skipping the design-deliberation \
1894 stage and continuing without a synthesis brief (see {})",
1895 where_recorded.display()
1896 ),
1897 );
1898 self.state.advise_attempted = true;
1899 self.state.save()?;
1900 return Ok(());
1901 }
1902 Err(e) => {
1903 self.state.event(
1904 "advise",
1905 format!(
1906 "could not resolve advisor seats ({e:#}); continuing \
1907 without a design-deliberation brief (see {})",
1908 where_recorded.display()
1909 ),
1910 );
1911 self.state.advise_attempted = true;
1912 self.state.save()?;
1913 return Ok(());
1914 }
1915 };
1916
1917 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1918 let artifacts = agent::artifacts_dir(&self.state.dir());
1919 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1920
1921 let mut jobs = Vec::new();
1922 for (i, spec) in seats.iter().cloned().enumerate() {
1923 let seat_key = format!("advisor-{}", i + 1);
1924 let seat = self.seat(&seat_key, &spec.id);
1925 jobs.push(SeatJob {
1926 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1927 spec,
1928 seat,
1929 cwd: worktrees[i % worktrees.len()].clone(),
1930 timeout,
1931 allow_write: false,
1932 sessions: false,
1933 artifacts: artifacts.clone(),
1934 stem: seat_key,
1935 handover: None,
1936 });
1937 }
1938
1939 self.state.event(
1940 "advise",
1941 format!(
1942 "{} advisor seat(s) sketching a design in parallel",
1943 jobs.len()
1944 ),
1945 );
1946 let mut quota_losses = Vec::new();
1947 let cache = self.state.config.cache_dir();
1948 let ctx = WaveCtx {
1949 carry_seats: false,
1950 pauses: Some((&self.pause, &self.interrupt)),
1951 run: &run_id,
1952 node: "advise",
1953 prompts: &prompts,
1954 cache: cache.as_deref(),
1955 round: None,
1956 };
1957 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1958 let results = ask_json_wave::<Proposal>(
1959 jobs,
1960 Arc::clone(&self.sem),
1961 self.state.config.graph.retries,
1962 &advisor_roster,
1963 &ctx,
1964 &mut quota_losses,
1965 &mut self.state,
1966 &|p: &Proposal| p.validate(),
1967 )
1968 .await;
1969 if self.state.resource_block.is_some() {
1970 return self.stop_for_resource();
1971 }
1972 self.state.quota.extend(quota_losses);
1973
1974 let mut records = Vec::with_capacity(results.len());
1975 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1976 let agent_id = seat.agent.clone();
1977 self.state.seats.insert(seat.key.clone(), seat);
1978 match res {
1979 Ok((proposal, out)) => {
1980 self.state
1981 .event("advise", format!("advisor-{} proposed a design", i + 1));
1982 records.push(advise::AdvisorRecord::proposed(
1983 i + 1,
1984 agent_id,
1985 proposal,
1986 out.duration_ms,
1987 ));
1988 }
1989 Err(e) => {
1990 self.state.event(
1991 "advise",
1992 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1993 );
1994 records.push(advise::AdvisorRecord::failed(
1995 i + 1,
1996 agent_id,
1997 e.to_string(),
1998 ));
1999 }
2000 }
2001 }
2002
2003 let mut advice = advise::Advice {
2004 records,
2005 synthesis: None,
2006 };
2007 if advice.proposals().is_empty() {
2008 self.state.event(
2009 "advise",
2010 "no advisor produced a usable proposal; continuing without a \
2011 synthesis brief"
2012 .to_owned(),
2013 );
2014 } else {
2015 match self
2016 .synthesize_brief(
2017 &advice,
2018 &instruction,
2019 &language,
2020 &worktrees[0],
2021 &artifacts,
2022 &run_id,
2023 &prompts,
2024 cache.as_deref(),
2025 )
2026 .await
2027 {
2028 Ok(Some(text)) => {
2029 self.state.event(
2030 "advise",
2031 "synthesized a design brief for the implementer".to_owned(),
2032 );
2033 advice.synthesis = Some(text);
2034 }
2035 Ok(None) => {
2036 self.state.event(
2037 "advise",
2038 "the synthesis seat produced nothing usable; continuing \
2039 without a design brief"
2040 .to_owned(),
2041 );
2042 }
2043 Err(e) => {
2044 if e.is::<crate::repo_lock::ResourceBlocked>() {
2045 return Err(e);
2046 }
2047 self.state.event(
2048 "advise",
2049 format!("could not synthesize a design brief: {e:#}"),
2050 );
2051 }
2052 }
2053 }
2054 advise::apply_reflection(&mut advice);
2055
2056 self.state.advice = Some(advice);
2057 self.state.advise_attempted = true;
2058 self.state.save()?;
2059 Ok(())
2060 }
2061
2062 #[allow(clippy::too_many_arguments)]
2074 async fn synthesize_brief(
2075 &mut self,
2076 advice: &advise::Advice,
2077 instruction: &str,
2078 language: &str,
2079 cwd: &Path,
2080 artifacts: &Path,
2081 run_id: &str,
2082 prompts: &Prompts,
2083 cache: Option<&Path>,
2084 ) -> Result<Option<String>> {
2085 let chain = agent::pick_chain(
2086 &self.state.config.agents,
2087 self.state.config.roles.synthesizer.as_ref(),
2088 &agent::installed,
2089 "synthesizer",
2090 )?;
2091 let proposals = advice.proposals();
2092 let mut prompt = prompt::with_overlay(
2093 prompt::synthesize_brief(instruction, &proposals, language),
2094 prompts.overlay("advise"),
2095 );
2096 if cache.is_some() {
2097 prompt.push('\n');
2102 prompt.push_str(&prompt::build_cache_note("advise", false));
2103 }
2104 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2105 let mut last = None;
2110 for (n, spec) in chain.iter().enumerate() {
2111 if n > 0 {
2112 self.state
2113 .event("advise", format!("synthesis falling back to {}", spec.id));
2114 }
2115 let mut seat = self.seat("advise-synthesis", &spec.id);
2116 let wait = self.repository_wait();
2117 let outcome = wait
2118 .scope(agent::invoke(
2119 spec,
2120 &mut seat,
2121 &Invocation {
2122 cwd,
2123 prompt: &prompt,
2124 timeout,
2125 allow_write: false,
2126 unsandboxed: false,
2127 sessions: false,
2128 artifacts,
2129 stem: &if n == 0 {
2130 "advise-synthesis".to_owned()
2131 } else {
2132 format!("advise-synthesis-{}", spec.id)
2133 },
2134 run: run_id,
2135 node: "advise",
2136 cache_dir: None,
2137 attachments: &[],
2138 writable: &[],
2139 },
2140 ))
2141 .await;
2142 if outcome.is_ok() {
2143 self.state.seats.insert(seat.key.clone(), seat);
2144 }
2145 let advance = agent::chain_advances(&outcome);
2146 last = Some(outcome);
2147 if !advance {
2148 break;
2149 }
2150 }
2151 let out = last.expect("a chain holds at least one agent")?;
2154 if !out.usable() {
2155 return Ok(None);
2156 }
2157 let text =
2158 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2159 Ok((!text.trim().is_empty()).then_some(text))
2160 }
2161
2162 async fn implement(&mut self) -> Result<()> {
2165 let run_id = self.state.id.clone();
2170 let prompts = self.state.config.prompts.clone();
2171 let todo: Vec<usize> = self
2172 .state
2173 .candidates
2174 .iter()
2175 .enumerate()
2176 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2177 .map(|(i, _)| i)
2178 .collect();
2179 if todo.is_empty() {
2180 return self.after_implement();
2181 }
2182 self.state.status = RunStatus::Implementing;
2183
2184 let language = self.state.config.graph.language.clone();
2185 let github_language = self.state.config.graph.github_language.clone();
2186 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2187 let sessions = self.state.config.graph.sessions;
2188 let artifacts = agent::artifacts_dir(&self.state.dir());
2189 let brief = self
2193 .state
2194 .advice
2195 .as_ref()
2196 .and_then(|a| a.synthesis.as_deref())
2197 .map(str::to_owned);
2198 let attachments = self.state.attachments.clone();
2199
2200 let mut jobs = Vec::new();
2201 for &i in &todo {
2202 let (index, label, worktree) = {
2203 let c = &self.state.candidates[i];
2204 (c.index, c.label, c.worktree.clone())
2205 };
2206 let spec = self.roles.implementers[index].clone();
2207 let seat_key = format!("impl-{label}");
2208 let seat = self.seat(&seat_key, &spec.id);
2209 let instruction = seeded_instruction(&self.state);
2210 jobs.push(SeatJob {
2211 spec,
2212 seat,
2213 prompt: prompt::implement_in(
2214 &instruction,
2215 &worktree.to_string_lossy(),
2216 &language,
2217 &github_language,
2218 brief.as_deref(),
2219 &attachments,
2220 ),
2221 cwd: worktree,
2222 timeout,
2223 allow_write: true,
2224 sessions,
2225 artifacts: artifacts.clone(),
2226 stem: format!("impl-{label}"),
2227 handover: None,
2228 });
2229 }
2230
2231 self.state.event(
2232 "implement",
2233 format!("{} candidates in parallel", jobs.len()),
2234 );
2235 let mut sent = jobs.clone();
2241 let cache = self.state.config.cache_dir();
2242 let ctx = WaveCtx {
2243 carry_seats: false,
2244 pauses: Some((&self.pause, &self.interrupt)),
2245 run: &run_id,
2246 node: "implement",
2247 prompts: &prompts,
2248 cache: cache.as_deref(),
2249 round: None,
2250 };
2251 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2252 if !wave_blocked(&results) {
2253 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2254 .await;
2255 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2256 .await?;
2257 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2258 .await;
2259 }
2260 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2261
2262 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2263 && results
2264 .iter()
2265 .any(|(_, _, out)| FailClass::of(out).is_some());
2266 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2267 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2268 continue;
2269 }
2270 let seat_key = seat.key.clone();
2271 let agent = seat.agent.clone();
2280 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2281 self.state.seats.insert(seat.key.clone(), seat);
2282 let label = self.state.candidates[i].label;
2283 let worktree = self.state.candidates[i].worktree.clone();
2284 let base = self.state.base_commit.clone();
2285
2286 let (summary, duration, failed, verified_claim) = match out {
2287 AgentOutcome::Ok(o) => {
2288 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2289 let failed = (!o.usable()).then(|| {
2290 if o.timed_out {
2291 "agent timed out".to_owned()
2292 } else {
2293 format!("agent exited with {:?}", o.exit_code)
2294 }
2295 });
2296 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2297 (text, o.duration_ms, failed, verified_claim)
2298 }
2299 AgentOutcome::Dropped(o) => {
2305 let why = o
2306 .dropped
2307 .as_ref()
2308 .map(|d| d.why.as_str())
2309 .unwrap_or("the CLI ended the stream without delivering its answer");
2310 (
2311 String::new(),
2312 o.duration_ms,
2313 Some(format!("the CLI dropped the stream ({why})")),
2314 None,
2315 )
2316 }
2317 AgentOutcome::Quota(o) => {
2318 self.state.quota.push(QuotaLoss {
2319 seat: seat_key,
2320 node: "implement".to_owned(),
2321 at: Timestamp::now(),
2322 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2323 });
2324 (
2325 String::new(),
2326 o.duration_ms,
2327 Some("rate limited (quota); produced no change".to_owned()),
2328 None,
2329 )
2330 }
2331 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2332 (String::new(), 0, Some(e), None)
2333 }
2334 };
2335
2336 let wait = crate::repo_lock::WaitPolicy::new(
2341 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2342 || false,
2343 );
2344 let _coordination = coordinate_operation(&mut self.state, &worktree, &wait).await?;
2345 let rescued = match git::rescue_commit(
2346 &worktree,
2347 &format!("magi: candidate {label} (uncommitted work)"),
2348 )
2349 .await
2350 {
2351 Ok(r) => {
2352 self.state.note_withheld("implement", &r.withheld);
2353 r.committed
2354 }
2355 Err(_) => false,
2356 };
2357 let commits = git::commits_ahead(&worktree, &base, "HEAD")
2358 .await
2359 .unwrap_or(0);
2360 let patch = git::diff(&worktree, &base, "HEAD")
2361 .await
2362 .unwrap_or_default();
2363 let stat = git::diff_stat(&worktree, &base, "HEAD")
2364 .await
2365 .unwrap_or_default();
2366 let files = git::changed_files(&worktree, &base, "HEAD")
2367 .await
2368 .map(|f| f.len())
2369 .unwrap_or(0);
2370 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2371
2372 let c = &mut self.state.candidates[i];
2373 if !exhausted_the_fallback_chain {
2374 c.agent = agent;
2375 }
2376 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2377 c.stat = stat;
2378 c.files = files;
2379 c.commits = commits;
2380 c.duration_ms = duration;
2381 c.empty = commits == 0 || patch.trim().is_empty();
2382 c.failed = match failed {
2385 Some(_) if c.empty => failed,
2386 _ => None,
2387 };
2388 c.verified_noop = if c.empty { verified_claim } else { None };
2393 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
2394 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
2395 (None, true, Some(_), _) => {
2396 format!("candidate {label}: no change produced (agent-verified no-op)")
2397 }
2398 (None, true, None, _) => format!("candidate {label}: no change produced"),
2399 (None, false, _, true) => {
2400 format!(
2401 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
2402 )
2403 }
2404 (None, false, _, false) => {
2405 format!("candidate {label}: {files} files, {commits} commits")
2406 }
2407 };
2408 self.state.event("implement", note);
2409 self.state.save()?;
2410 }
2411
2412 if resource_blocked {
2413 return self.stop_for_resource();
2414 }
2415 if selected_failed {
2416 self.state.status = RunStatus::Blocked;
2417 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2418 self.state.save()?;
2419 bail!(
2420 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2421 );
2422 }
2423 self.after_implement()
2424 }
2425
2426 async fn resume_undelivered(
2454 &mut self,
2455 results: &mut [(usize, SeatState, AgentOutcome)],
2456 sent: &[SeatJob],
2457 prompts: &Prompts,
2458 run_id: &str,
2459 ) {
2460 for (wi, seat, out) in results.iter_mut() {
2461 let Some(dropped) = (match &*out {
2462 AgentOutcome::Dropped(o) => o.dropped.clone(),
2463 _ => None,
2464 }) else {
2465 continue;
2466 };
2467 let Some(job) = sent.get(*wi) else { continue };
2468 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2470 self.state.event(
2471 "implement",
2472 format!(
2473 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2474 work is in the tree",
2475 seat.key, dropped.output_tokens, dropped.why
2476 ),
2477 );
2478 continue;
2479 }
2480 if !has_context(&job.spec, seat, job.sessions) {
2488 self.state.event(
2489 "implement",
2490 format!(
2491 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2492 is no session left to resume",
2493 seat.key, dropped.output_tokens, dropped.why
2494 ),
2495 );
2496 continue;
2497 }
2498 self.state.event(
2499 "implement",
2500 format!(
2501 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2502 conversation",
2503 seat.key, dropped.output_tokens, dropped.why
2504 ),
2505 );
2506 let mut retry = job.clone();
2507 retry.seat = seat.clone();
2508 retry.prompt = prompt::resume_after_drop(&dropped.why);
2509 retry.timeout = retry_budget(job.timeout, true);
2510 retry.stem = format!("{}-resume", job.stem);
2511 let cache = self.state.config.cache_dir();
2512 let ctx = WaveCtx {
2513 carry_seats: false,
2514 pauses: Some((&self.pause, &self.interrupt)),
2515 run: run_id,
2516 node: "implement",
2517 prompts,
2518 cache: cache.as_deref(),
2519 round: None,
2520 };
2521 let (resumed_seat, resumed) =
2522 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2523 *seat = resumed_seat;
2524 *out = resumed;
2525 }
2526 }
2527
2528 async fn resume_seat_handovers(
2593 &mut self,
2594 results: &mut [(usize, SeatState, AgentOutcome)],
2595 sent: &mut [SeatJob],
2596 prompts: &Prompts,
2597 run_id: &str,
2598 ) -> Result<()> {
2599 let instruction = seeded_instruction(&self.state);
2600 let language = self.state.config.graph.language.clone();
2601 let github_language = self.state.config.graph.github_language.clone();
2602 let brief = self
2603 .state
2604 .advice
2605 .as_ref()
2606 .and_then(|a| a.synthesis.as_deref())
2607 .map(str::to_owned);
2608 let attachments = self.state.attachments.clone();
2609 for (wi, seat, out) in results.iter_mut() {
2610 let others: BTreeSet<String> = sent
2613 .iter()
2614 .enumerate()
2615 .filter(|(j, _)| j != wi)
2616 .map(|(_, j)| j.spec.id.clone())
2617 .collect();
2618 let Some(job) = sent.get_mut(*wi) else {
2619 continue;
2620 };
2621 let start = self
2626 .roles
2627 .implementer_roster
2628 .iter()
2629 .position(|s| s.id == job.spec.id)
2630 .unwrap_or(0);
2631 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2632 let mut fallback_attempt = 0usize;
2633 let mut prev: Option<FailClass> = None;
2634 while let Some(cur) = FailClass::of(&*out) {
2635 if !should_hand_over(prev.as_ref(), &cur) {
2636 break;
2637 }
2638 let Some(next) =
2639 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2640 .cloned()
2641 else {
2642 break;
2643 };
2644 tried.insert(next.id.clone());
2645 fallback_attempt += 1;
2646
2647 let coordination = self.coordinate(&job.cwd).await?;
2648 if let Ok(r) = git::rescue_commit(
2649 &job.cwd,
2650 &format!(
2651 "magi: candidate {} (uncommitted work before {} fallback)",
2652 seat.key,
2653 if cur == FailClass::Quota {
2654 "quota"
2655 } else {
2656 "handover"
2657 }
2658 ),
2659 )
2660 .await
2661 {
2662 self.state.note_withheld("implement", &r.withheld);
2663 }
2664 drop(coordination);
2665
2666 record_handover(
2667 &mut self.state,
2668 "implement",
2669 &seat.key,
2670 &seat.agent,
2671 &next.id,
2672 &cur,
2673 &fail_reason(&*out),
2674 );
2675 prev = Some(cur.clone());
2676
2677 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2678 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2679 job.spec = next.clone();
2687 let mut retry = job.clone();
2688 retry.seat = new_seat;
2689 retry.prompt = prompt::implement_in(
2690 &instruction,
2691 &job.cwd.to_string_lossy(),
2692 &language,
2693 &github_language,
2694 brief.as_deref(),
2695 &attachments,
2696 );
2697 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2698 let cache = self.state.config.cache_dir();
2699 let ctx = WaveCtx {
2700 carry_seats: false,
2701 pauses: Some((&self.pause, &self.interrupt)),
2702 run: run_id,
2703 node: "implement",
2704 prompts,
2705 cache: cache.as_deref(),
2706 round: None,
2707 };
2708 let (fallback_seat, fallback_out) = run_one(
2709 retry,
2710 Arc::clone(&self.sem),
2711 &ctx,
2712 &mut self.state,
2713 fallback_attempt,
2714 )
2715 .await;
2716 *seat = fallback_seat;
2717 *out = fallback_out;
2718 }
2719 }
2720 Ok(())
2721 }
2722
2723 async fn resume_unconfirmed_commands(
2747 &mut self,
2748 results: &mut [(usize, SeatState, AgentOutcome)],
2749 sent: &[SeatJob],
2750 prompts: &Prompts,
2751 run_id: &str,
2752 ) {
2753 for (wi, seat, out) in results.iter_mut() {
2754 let AgentOutcome::Ok(o) = &*out else {
2755 continue;
2756 };
2757 if !has_unconfirmed_command(&o.commands) {
2758 continue;
2759 }
2760 let Some(job) = sent.get(*wi) else { continue };
2761 if !has_context(&job.spec, seat, job.sessions) {
2762 self.state.event(
2763 "implement",
2764 format!(
2765 "{}: the reply named a command whose own CLI never confirmed the exit \
2766 status of, but there is no session left to resume",
2767 seat.key
2768 ),
2769 );
2770 continue;
2771 }
2772 self.state.event(
2773 "implement",
2774 format!(
2775 "{}: the reply named a command whose own CLI never confirmed the exit \
2776 status of; resuming the conversation",
2777 seat.key
2778 ),
2779 );
2780 let mut retry = job.clone();
2781 retry.seat = seat.clone();
2782 retry.prompt = prompt::resume_incomplete(
2783 "a command in your last reply had no confirmed exit status",
2784 );
2785 retry.timeout = retry_budget(job.timeout, true);
2786 retry.stem = format!("{}-confirm", job.stem);
2787 let cache = self.state.config.cache_dir();
2788 let ctx = WaveCtx {
2789 carry_seats: false,
2790 pauses: Some((&self.pause, &self.interrupt)),
2791 run: run_id,
2792 node: "implement",
2793 prompts,
2794 cache: cache.as_deref(),
2795 round: None,
2796 };
2797 let (resumed_seat, resumed) =
2798 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2799 *seat = resumed_seat;
2800 *out = resumed;
2801 }
2802 }
2803
2804 async fn continue_fix_report(
2825 &mut self,
2826 mut seat: SeatState,
2827 parse_err: String,
2828 job: &SeatJob,
2829 prompts: &Prompts,
2830 run_id: &str,
2831 round: usize,
2832 ) -> (
2833 SeatState,
2834 Option<FixReport>,
2835 Option<String>,
2836 ContinuationRecord,
2837 ) {
2838 let mut last_err = parse_err;
2839 let mut cumulative_wait_ms = 0u64;
2840 let mut attempts = 0usize;
2841 loop {
2842 if !has_context(&job.spec, &seat, job.sessions) {
2843 self.state.event(
2844 "fix",
2845 format!(
2846 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2847 session left to resume into"
2848 ),
2849 );
2850 let outcome = if attempts == 0 {
2851 ContinuationOutcome::NoSession
2852 } else {
2853 ContinuationOutcome::Exhausted
2854 };
2855 return (
2856 seat,
2857 None,
2858 Some(format!("unparsable fix report: {last_err}")),
2859 ContinuationRecord {
2860 attempts,
2861 cumulative_wait_ms,
2862 outcome,
2863 },
2864 );
2865 }
2866 if attempts >= MAX_FIX_CONTINUATIONS {
2867 self.state.event(
2868 "fix",
2869 format!(
2870 "round {round}: fixer's reply still had no adoption report after \
2871 {attempts} continuation(s) ({last_err}); giving up"
2872 ),
2873 );
2874 return (
2875 seat,
2876 None,
2877 Some(format!(
2878 "unparsable fix report after {attempts} continuation(s): {last_err}"
2879 )),
2880 ContinuationRecord {
2881 attempts,
2882 cumulative_wait_ms,
2883 outcome: ContinuationOutcome::Exhausted,
2884 },
2885 );
2886 }
2887 attempts += 1;
2888 self.state.event(
2889 "fix",
2890 format!(
2891 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2892 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2893 ),
2894 );
2895 let mut retry = job.clone();
2896 retry.seat = seat.clone();
2897 retry.prompt = prompt::resume_incomplete(&last_err);
2898 retry.timeout = retry_budget(job.timeout, true);
2899 retry.stem = format!("{}-continue{attempts}", job.stem);
2900 let cache = self.state.config.cache_dir();
2901 let ctx = WaveCtx {
2902 carry_seats: false,
2903 pauses: Some((&self.pause, &self.interrupt)),
2904 run: run_id,
2905 node: "fix",
2906 prompts,
2907 cache: cache.as_deref(),
2908 round: Some(round),
2909 };
2910 let (resumed_seat, resumed_out) = run_one(
2911 retry,
2912 Arc::clone(&self.sem),
2913 &ctx,
2914 &mut self.state,
2915 attempts,
2916 )
2917 .await;
2918 seat = resumed_seat;
2919 match resumed_out {
2920 AgentOutcome::Ok(o) => {
2921 cumulative_wait_ms += o.duration_ms;
2922 match verdict::extract_json::<FixReport>(&o.text) {
2923 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2924 self.state.event(
2925 "fix",
2926 format!(
2927 "round {round}: fixer's adoption report recovered after \
2928 {attempts} continuation(s)"
2929 ),
2930 );
2931 return (
2932 seat,
2933 Some(report),
2934 None,
2935 ContinuationRecord {
2936 attempts,
2937 cumulative_wait_ms,
2938 outcome: ContinuationOutcome::Resumed,
2939 },
2940 );
2941 }
2942 Ok(_) => {
2950 last_err = "the reply parsed, but it reported a command whose own CLI \
2951 never confirmed an exit status"
2952 .to_owned();
2953 }
2954 Err(e) => last_err = e.to_string(),
2955 }
2956 }
2957 AgentOutcome::Quota(o) => {
2958 cumulative_wait_ms += o.duration_ms;
2959 self.state.quota.push(QuotaLoss {
2960 seat: seat.key.clone(),
2961 node: "fix".to_owned(),
2962 at: Timestamp::now(),
2963 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2964 });
2965 self.state.event(
2966 "fix",
2967 format!(
2968 "round {round}: continuation rate limited (quota); not retrying now"
2969 ),
2970 );
2971 return (
2972 seat,
2973 None,
2974 Some("rate limited (quota) while recovering the fix report".to_owned()),
2975 ContinuationRecord {
2976 attempts,
2977 cumulative_wait_ms,
2978 outcome: ContinuationOutcome::QuotaLost,
2979 },
2980 );
2981 }
2982 AgentOutcome::Dropped(o) => {
2983 cumulative_wait_ms += o.duration_ms;
2984 let why = o
2985 .dropped
2986 .as_ref()
2987 .map(|d| d.why.as_str())
2988 .unwrap_or("the CLI ended the stream without delivering its answer");
2989 last_err = format!("the CLI dropped the stream ({why})");
2990 }
2991 AgentOutcome::Failed(e) => last_err = e,
2992 AgentOutcome::ResourceBlocked(e) => {
2995 return (
2996 seat,
2997 None,
2998 Some(format!("the continuation was not started: {e}")),
2999 ContinuationRecord {
3000 attempts: attempts.saturating_sub(1),
3001 cumulative_wait_ms,
3002 outcome: ContinuationOutcome::NoSession,
3003 },
3004 );
3005 }
3006 }
3007 }
3008 }
3009
3010 fn after_implement(&mut self) -> Result<()> {
3011 if self.state.leaks.is_empty() {
3013 let cfg = self.state.config.blind.clone();
3014 let mut leaks = Vec::new();
3015 for c in &self.state.candidates {
3016 let Some(patch) =
3017 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3018 else {
3019 continue;
3020 };
3021 leaks.extend(blind::scan(
3022 &format!("candidate {} patch", c.label),
3023 &patch,
3024 &cfg.vendor_tokens,
3025 ));
3026 }
3027 if !leaks.is_empty() {
3028 let summary = leaks
3029 .iter()
3030 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3031 .collect::<Vec<_>>()
3032 .join(", ");
3033 match cfg.on_leak {
3034 LeakPolicy::Fail => {
3035 self.state.status = RunStatus::Failed;
3036 self.state
3037 .event("blind", format!("vendor text in a patch: {summary}"));
3038 self.state.leaks = leaks;
3039 self.state.save()?;
3040 self.settle_questions();
3041 bail!(
3042 "blind.on_leak = \"fail\" and vendor text reached a \
3043 judged patch: {summary}"
3044 );
3045 }
3046 LeakPolicy::Redact => self.state.event(
3047 "blind",
3048 format!("redacting vendor text for judging: {summary}"),
3049 ),
3050 LeakPolicy::Warn => self.state.event(
3051 "blind",
3052 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3053 ),
3054 }
3055 self.state.leaks = leaks;
3056 }
3057 }
3058
3059 if self.state.viable().is_empty() {
3060 if self.state.all_candidates_verified_noop() {
3061 self.state.status = RunStatus::VerifiedNoop;
3072 self.state.save()?;
3073 self.settle_questions();
3074 return Ok(());
3075 }
3076 self.state.status = RunStatus::Failed;
3077 self.state.save()?;
3078 self.settle_questions();
3079 bail!("no candidate produced a change; nothing to judge");
3080 }
3081 self.state.status = RunStatus::Judging;
3082 self.state.save()?;
3083 Ok(())
3084 }
3085
3086 async fn judge(&mut self) -> Result<()> {
3089 let run_id = self.state.id.clone();
3094 let prompts = self.state.config.prompts.clone();
3095 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3096 return Ok(());
3097 }
3098 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3099 if viable.len() == 1 {
3100 self.state.judge_skipped = true;
3107 self.state.event(
3108 "judge",
3109 format!(
3110 "only candidate {} produced a change; judging skipped",
3111 viable[0].label
3112 ),
3113 );
3114 self.state.save()?;
3115 return Ok(());
3116 }
3117 self.state.status = RunStatus::Judging;
3118
3119 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3120 let language = self.state.config.graph.language.clone();
3121 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3122 let sessions = self.state.config.graph.sessions;
3123 let artifacts = agent::artifacts_dir(&self.state.dir());
3124 let root = self.state.worktree_root();
3125 let base_short = short(&self.state.base_commit);
3126
3127 let mut jobs = Vec::new();
3128 let mut orders = Vec::new();
3129 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3130 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3131 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3132 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3133 let seat_key = format!("judge-{}", j + 1);
3134 let seat = self.seat(&seat_key, &spec.id);
3135 jobs.push(SeatJob {
3136 prompt: prompt::judge(
3137 &self.state.instruction,
3138 &views,
3139 self.roles.judges.len(),
3140 &base_short,
3141 &language,
3142 ),
3143 spec,
3144 seat,
3145 cwd: root.join(format!("judge-{}", j + 1)),
3146 timeout,
3147 allow_write: false,
3148 sessions,
3149 artifacts: artifacts.clone(),
3150 stem: format!("judge-{}", j + 1),
3151 handover: None,
3152 });
3153 }
3154
3155 self.state.event(
3156 "judge",
3157 format!(
3158 "{} judges ranking {} candidates blind",
3159 jobs.len(),
3160 viable.len()
3161 ),
3162 );
3163 let labels_for_check = labels.clone();
3164 let mut quota_losses = Vec::new();
3165 let cache = self.state.config.cache_dir();
3166 let ctx = WaveCtx {
3167 carry_seats: false,
3168 pauses: Some((&self.pause, &self.interrupt)),
3169 run: &run_id,
3170 node: "judge",
3171 prompts: &prompts,
3172 cache: cache.as_deref(),
3173 round: None,
3174 };
3175 let results = ask_json_wave::<Ranking>(
3176 jobs,
3177 Arc::clone(&self.sem),
3178 self.state.config.graph.retries,
3179 &self.roles.judge_roster,
3180 &ctx,
3181 &mut quota_losses,
3182 &mut self.state,
3183 &move |r: &Ranking| r.validate(&labels_for_check),
3184 )
3185 .await;
3186 if self.state.resource_block.is_some() {
3187 return self.stop_for_resource();
3188 }
3189 self.state.quota.extend(quota_losses);
3190
3191 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3192 let agent_id = seat.agent.clone();
3193 self.state.seats.insert(seat.key.clone(), seat);
3194 let mut record = Judgement {
3195 judge: j + 1,
3196 seat: format!("judge-{}", j + 1),
3197 agent: agent_id,
3198 ranking: Vec::new(),
3199 reasons: BTreeMap::new(),
3200 confidence: None,
3201 strengths: BTreeMap::new(),
3202 order: orders[j].clone(),
3203 failed: None,
3204 duration_ms: 0,
3205 };
3206 match res {
3207 Ok((ranking, out)) => {
3208 record.ranking = ranking.normalized();
3209 record.reasons = ranking.reasons;
3210 record.strengths = ranking.strengths;
3211 record.confidence = ranking.confidence;
3212 record.duration_ms = out.duration_ms;
3213 self.state.event(
3214 "judge",
3215 format!(
3216 "judge {} ranked {}",
3217 j + 1,
3218 record.ranking.iter().collect::<String>()
3219 ),
3220 );
3221 }
3222 Err(e) => {
3223 record.failed = Some(e.to_string());
3224 self.state
3225 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3226 }
3227 }
3228 self.state.judgements.push(record);
3229 self.state.save()?;
3230 }
3231 Ok(())
3232 }
3233
3234 async fn deliberate(&mut self) -> Result<()> {
3237 let run_id = self.state.id.clone();
3242 let prompts = self.state.config.prompts.clone();
3243 if !self.state.deliberation.is_empty() {
3244 return Ok(());
3245 }
3246 let tops: Vec<char> = self
3247 .state
3248 .judgements
3249 .iter()
3250 .filter_map(|j| j.ranking.first().copied())
3251 .collect();
3252 let rounds = self.state.config.graph.deliberate_rounds;
3253 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3254 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3255 self.state.event(
3256 "deliberate",
3257 format!("judges agreed on {} outright; no deliberation", tops[0]),
3258 );
3259 }
3260 self.state.status = RunStatus::Voting;
3261 self.state.save()?;
3262 return Ok(());
3263 }
3264
3265 self.state.status = RunStatus::Deliberating;
3266 self.state.event(
3267 "deliberate",
3268 format!(
3269 "split: first choices were {} — opening {rounds} round(s)",
3270 tops.iter().collect::<String>()
3271 ),
3272 );
3273
3274 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3275 let language = self.state.config.graph.language.clone();
3276 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3277 let sessions = self.state.config.graph.sessions;
3278 let artifacts = agent::artifacts_dir(&self.state.dir());
3279 let root = self.state.worktree_root();
3280 let base_short = short(&self.state.base_commit);
3281
3282 for round in 1..=rounds {
3286 let mut turns: Vec<DeliberationTurn> = Vec::new();
3287 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3288 if self.state.judgements[j].failed.is_some() {
3289 continue;
3290 }
3291 let seat_key = format!("judge-{}", j + 1);
3292 let spec = self.occupant(&seat_key, spec);
3293 let mut seat = self.seat(&seat_key, &spec.id);
3294 let transcript = self.transcript(&turns, j);
3295 let build = |context: Option<&str>| {
3296 prompt::deliberate(
3297 &self.state.instruction,
3298 context,
3299 &transcript,
3300 round,
3301 rounds,
3302 &language,
3303 )
3304 };
3305 let block = self.candidate_block(&viable, &base_short);
3306 let full = build(Some(&block));
3307 let text = if has_context(&spec, &seat, sessions) {
3308 build(None)
3309 } else {
3310 full.clone()
3311 };
3312 let job = SeatJob {
3313 spec,
3314 seat: seat.clone(),
3315 prompt: text,
3316 cwd: root.join(format!("judge-{}", j + 1)),
3317 timeout,
3318 allow_write: false,
3319 sessions,
3320 artifacts: artifacts.clone(),
3321 stem: format!("delib-{round}-judge-{}", j + 1),
3322 handover: Some(full),
3323 };
3324 let cache = self.state.config.cache_dir();
3325 let ctx = WaveCtx {
3326 carry_seats: false,
3327 pauses: Some((&self.pause, &self.interrupt)),
3328 run: &run_id,
3329 node: "deliberate",
3330 prompts: &prompts,
3331 cache: cache.as_deref(),
3332 round: None,
3333 };
3334 let mut losses = Vec::new();
3338 let mut results = ask_wave_with::<String>(
3339 vec![job],
3340 Arc::clone(&self.sem),
3341 0,
3342 &self.roles.judge_roster,
3343 &ctx,
3344 &mut losses,
3345 &mut self.state,
3346 &|text: &str| {
3347 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3348 },
3349 )
3350 .await;
3351 self.state.quota.extend(losses);
3352 let (updated, res, _) = results.pop().expect("one job in, one result out");
3353 seat = updated;
3354 let agent_id = seat.agent.clone();
3355 self.state.seats.insert(seat.key.clone(), seat);
3356 let body = match res {
3357 Ok((body, _)) => body,
3358 Err(e) => {
3361 self.state
3362 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3363 continue;
3364 }
3365 };
3366 let tentative = verdict::extract_json::<Position>(&body)
3367 .ok()
3368 .and_then(|p| p.tentative)
3369 .and_then(|s| s.trim().chars().next())
3370 .map(|c| c.to_ascii_uppercase());
3371 self.state.event(
3372 "deliberate",
3373 format!(
3374 "round {round}: judge {} now favours {}",
3375 j + 1,
3376 tentative.map_or("—".to_owned(), |c| c.to_string())
3377 ),
3378 );
3379 turns.push(DeliberationTurn {
3380 judge: j + 1,
3381 agent: agent_id,
3382 body: blind::sanitize_prose(&body, &self.state.config.blind),
3383 tentative,
3384 });
3385 }
3386 self.state
3387 .deliberation
3388 .push(DeliberationRound { round, turns });
3389 self.state.save()?;
3390 }
3391
3392 self.state.status = RunStatus::Voting;
3393 self.state.save()?;
3394 Ok(())
3395 }
3396
3397 async fn vote(&mut self) -> Result<()> {
3400 let run_id = self.state.id.clone();
3405 let prompts = self.state.config.prompts.clone();
3406 if !self.state.votes.is_empty() {
3407 return Ok(());
3408 }
3409 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3410 if viable.len() == 1 {
3411 return Ok(());
3412 }
3413 self.state.status = RunStatus::Voting;
3414
3415 let language = self.state.config.graph.language.clone();
3416 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3417 let sessions = self.state.config.graph.sessions;
3418 let artifacts = agent::artifacts_dir(&self.state.dir());
3419 let root = self.state.worktree_root();
3420 let base_short = short(&self.state.base_commit);
3421 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3422
3423 let mut jobs = Vec::new();
3424 let mut seats_at = Vec::new();
3425 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3426 if self
3427 .state
3428 .judgements
3429 .get(j)
3430 .is_some_and(|r| r.failed.is_some())
3431 {
3432 continue;
3433 }
3434 let seat_key = format!("judge-{}", j + 1);
3435 let spec = self.occupant(&seat_key, spec);
3436 let seat = self.seat(&seat_key, &spec.id);
3437 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3438 let text = if has_context(&spec, &seat, sessions) {
3439 prompt::final_vote(&viable, &language)
3440 } else {
3441 full.clone()
3442 };
3443 jobs.push(SeatJob {
3444 spec,
3445 seat,
3446 prompt: text,
3447 cwd: root.join(format!("judge-{}", j + 1)),
3448 timeout,
3449 allow_write: false,
3450 sessions,
3451 artifacts: artifacts.clone(),
3452 stem: format!("vote-judge-{}", j + 1),
3453 handover: Some(full),
3454 });
3455 seats_at.push(j);
3456 }
3457
3458 self.state.event(
3459 "vote",
3460 format!(
3461 "collecting {} final votes one by one, privately",
3462 jobs.len()
3463 ),
3464 );
3465 let allowed = viable.clone();
3466 let mut quota_losses = Vec::new();
3467 let cache = self.state.config.cache_dir();
3468 let ctx = WaveCtx {
3469 carry_seats: false,
3470 pauses: Some((&self.pause, &self.interrupt)),
3471 run: &run_id,
3472 node: "vote",
3473 prompts: &prompts,
3474 cache: cache.as_deref(),
3475 round: None,
3476 };
3477 let results = ask_json_wave::<FinalVote>(
3478 jobs,
3479 Arc::clone(&self.sem),
3480 self.state.config.graph.retries,
3481 &self.roles.judge_roster,
3482 &ctx,
3483 &mut quota_losses,
3484 &mut self.state,
3485 &move |v: &FinalVote| match v.label() {
3486 Some(c) if allowed.contains(&c) => Ok(()),
3487 other => bail!("vote {other:?} is not one of {allowed:?}"),
3488 },
3489 )
3490 .await;
3491 if self.state.resource_block.is_some() {
3492 return self.stop_for_resource();
3493 }
3494 self.state.quota.extend(quota_losses);
3495
3496 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3497 let agent_id = seat.agent.clone();
3498 self.state.seats.insert(seat.key.clone(), seat);
3499 let initial = self
3500 .state
3501 .judgements
3502 .get(j)
3503 .and_then(|r| r.ranking.first().copied());
3504 let mut record = VoteRecord {
3505 judge: j + 1,
3506 agent: agent_id,
3507 vote: None,
3508 reason: String::new(),
3509 changed: false,
3510 };
3511 match res {
3512 Ok((v, _)) => {
3513 record.vote = v.label();
3514 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3515 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3516 self.state.event(
3517 "vote",
3518 format!(
3519 "judge {} voted {}{}",
3520 j + 1,
3521 record.vote.unwrap_or('?'),
3522 if record.changed { " (changed)" } else { "" }
3523 ),
3524 );
3525 }
3526 Err(e) => {
3527 self.state
3528 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3529 }
3530 }
3531 self.state.votes.push(record);
3532 self.state.save()?;
3533 }
3534 Ok(())
3535 }
3536
3537 fn tally(&mut self) -> Result<()> {
3540 if self.state.tally.is_some() {
3541 return Ok(());
3542 }
3543 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3544 let tops: Vec<char> = self
3545 .state
3546 .judgements
3547 .iter()
3548 .filter_map(|j| j.ranking.first().copied())
3549 .collect();
3550 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3551
3552 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3555 let mut cast: Vec<char> = Vec::new();
3556 for (i, j) in self.state.judgements.iter().enumerate() {
3557 let vote = self
3558 .state
3559 .votes
3560 .iter()
3561 .find(|v| v.judge == i + 1)
3562 .and_then(|v| v.vote)
3563 .or_else(|| j.ranking.first().copied());
3564 if let Some(v) = vote {
3565 *first_choice.entry(v).or_insert(0) += 1;
3566 cast.push(v);
3567 }
3568 }
3569
3570 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3571 for j in &self.state.judgements {
3572 let n = j.ranking.len();
3573 for (pos, label) in j.ranking.iter().enumerate() {
3574 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3575 }
3576 }
3577
3578 let best = first_choice.values().copied().max().unwrap_or(0);
3579 let mut leaders: Vec<char> = first_choice
3580 .iter()
3581 .filter(|(_, v)| **v == best)
3582 .map(|(k, _)| *k)
3583 .collect();
3584 let mut tie_break = None;
3585 if leaders.len() > 1 {
3586 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3587 let borda_leaders: Vec<char> = leaders
3588 .iter()
3589 .copied()
3590 .filter(|l| borda[l] == top_borda)
3591 .collect();
3592 tie_break = Some(if borda_leaders.len() == 1 {
3593 format!(
3594 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3595 leaders.len()
3596 )
3597 } else {
3598 format!(
3599 "{} way tie on both first-choice votes and Borda points, broken by label order",
3600 leaders.len()
3601 )
3602 });
3603 leaders = borda_leaders;
3604 leaders.sort_unstable();
3605 }
3606 let winner = *leaders
3607 .first()
3608 .or(viable.first())
3609 .context("no candidate to declare a winner from")?;
3610
3611 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3612 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3613 let deliberated = !self.state.deliberation.is_empty();
3614
3615 let quota_seats: std::collections::BTreeSet<&str> =
3619 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3620 let mut present = 0usize;
3621 for (i, j) in self.state.judgements.iter().enumerate() {
3622 if quota_seats.contains(j.seat.as_str()) {
3623 continue;
3624 }
3625 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3626 let voted = self
3627 .state
3628 .votes
3629 .iter()
3630 .any(|v| v.judge == i + 1 && v.vote.is_some());
3631 if ranked || voted {
3632 present += 1;
3633 }
3634 }
3635 let needs_quorum = viable.len() > 1;
3641 let judges_total = if needs_quorum {
3642 self.roles.judges.len()
3643 } else {
3644 0
3645 };
3646 let quorum = if needs_quorum {
3647 judges_total / 2 + 1
3648 } else {
3649 0
3650 };
3651 let met_quorum = !needs_quorum || present >= quorum;
3652 let uncontested = (!needs_quorum).then(|| {
3653 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3654 });
3655
3656 self.state.event(
3657 "tally",
3658 match &uncontested {
3659 Some(reason) => format!("winner {winner} — {reason}"),
3660 None => format!(
3661 "winner {winner} — votes {} | initial {} | {} changed | \
3662 {present}/{judges_total} judges{}",
3663 first_choice
3664 .iter()
3665 .map(|(k, v)| format!("{k}:{v}"))
3666 .collect::<Vec<_>>()
3667 .join(" "),
3668 if unanimous_initial {
3669 "unanimous"
3670 } else {
3671 "split"
3672 },
3673 changed_votes,
3674 if met_quorum {
3675 String::new()
3676 } else {
3677 format!(" — below quorum ({quorum} required)")
3678 },
3679 ),
3680 },
3681 );
3682 if !met_quorum {
3683 self.state.event(
3684 "stall",
3685 format!(
3686 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3687 the run stops here, resumable"
3688 ),
3689 );
3690 }
3691 self.state.tally = Some(Tally {
3692 first_choice,
3693 borda,
3694 winner,
3695 rankings: tops.len(),
3696 unanimous_initial,
3697 deliberated,
3698 changed_votes,
3699 unanimous_final,
3700 tie_break,
3701 judges: judges_total,
3702 present,
3703 quorum,
3704 met_quorum,
3705 uncontested,
3706 });
3707 self.state.status = if met_quorum {
3708 RunStatus::Reviewing
3709 } else {
3710 RunStatus::Stalled
3711 };
3712 self.state.save()?;
3713 Ok(())
3714 }
3715
3716 #[allow(clippy::too_many_lines)]
3737 async fn recover_stall(&mut self) -> Result<bool> {
3738 let run_id = self.state.id.clone();
3743 let prompts = self.state.config.prompts.clone();
3744 let quota_seats: BTreeSet<&str> =
3749 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3750 let absent: Vec<String> = self
3751 .state
3752 .judgements
3753 .iter()
3754 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3755 .map(|j| j.seat.clone())
3756 .collect();
3757 if absent.is_empty() {
3758 return Ok(false);
3759 }
3760 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3761 if viable.len() <= 1 {
3762 return Ok(false);
3763 }
3764 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3765 let language = self.state.config.graph.language.clone();
3766 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3767 let sessions = self.state.config.graph.sessions;
3768 let artifacts = agent::artifacts_dir(&self.state.dir());
3769 let root = self.state.worktree_root();
3770 let base_short = short(&self.state.base_commit);
3771 let candidates: Vec<Candidate> = viable.clone();
3772
3773 let mut positions: Vec<usize> = absent
3775 .iter()
3776 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3777 .collect();
3778 if positions.is_empty() {
3779 return Ok(false);
3780 }
3781 positions.sort_unstable();
3782 positions.dedup();
3783
3784 let mut judge_jobs = Vec::new();
3786 for &j in &positions {
3787 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3788 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3789 let seat_key = format!("judge-{}", j + 1);
3790 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3791 let seat = self.seat(&seat_key, &spec.id);
3792 judge_jobs.push(SeatJob {
3793 spec,
3794 seat,
3795 prompt: prompt::judge(
3796 &self.state.instruction,
3797 &views,
3798 self.roles.judges.len(),
3799 &base_short,
3800 &language,
3801 ),
3802 cwd: root.join(seat_key),
3803 timeout,
3804 allow_write: false,
3805 sessions,
3806 artifacts: artifacts.clone(),
3807 stem: format!("judge-{}-recover", j + 1),
3808 handover: None,
3809 });
3810 }
3811
3812 let labels_for_check = labels.clone();
3813 let mut judge_losses = Vec::new();
3814 let retries = self.state.config.graph.retries;
3815 let cache = self.state.config.cache_dir();
3816 let ctx = WaveCtx {
3817 carry_seats: false,
3818 pauses: Some((&self.pause, &self.interrupt)),
3819 run: &run_id,
3820 node: "judge",
3821 prompts: &prompts,
3822 cache: cache.as_deref(),
3823 round: None,
3824 };
3825 let results = ask_json_wave::<Ranking>(
3826 judge_jobs,
3827 Arc::clone(&self.sem),
3828 retries,
3829 &self.roles.judge_roster,
3830 &ctx,
3831 &mut judge_losses,
3832 &mut self.state,
3833 &move |r: &Ranking| r.validate(&labels_for_check),
3834 )
3835 .await;
3836 if self.state.resource_block.is_some() {
3837 self.stop_for_resource()?;
3838 return Ok(false);
3839 }
3840
3841 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3843 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3844 let agent_id = seat.agent.clone();
3845 self.state.seats.insert(seat.key.clone(), seat);
3846 let record = &mut self.state.judgements[j];
3847 match res {
3848 Ok((ranking, out)) => {
3849 record.agent = agent_id;
3850 record.ranking = ranking.normalized();
3851 record.reasons = ranking.reasons;
3852 record.strengths = ranking.strengths;
3853 record.confidence = ranking.confidence;
3854 record.failed = None;
3855 record.duration_ms = out.duration_ms;
3856 recovered.insert(j);
3857 self.state.event(
3858 "recover",
3859 format!("judge {} ranked again after the limit", j + 1),
3860 );
3861 }
3862 Err(e) => {
3863 self.state
3864 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3865 }
3866 }
3867 }
3868
3869 let mut vote_jobs = Vec::new();
3871 let mut vote_pos: Vec<usize> = Vec::new();
3872 for &j in &recovered {
3873 let seat_key = format!("judge-{}", j + 1);
3874 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3875 let seat = self.seat(&seat_key, &spec.id);
3876 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3877 let text = if has_context(&spec, &seat, sessions) {
3878 prompt::final_vote(&labels, &language)
3879 } else {
3880 full.clone()
3881 };
3882 vote_jobs.push(SeatJob {
3883 spec,
3884 seat,
3885 prompt: text,
3886 cwd: root.join(seat_key),
3887 timeout,
3888 allow_write: false,
3889 sessions,
3890 artifacts: artifacts.clone(),
3891 stem: format!("vote-judge-{}-recover", j + 1),
3892 handover: Some(full),
3893 });
3894 vote_pos.push(j);
3895 }
3896 let allowed = labels.clone();
3897 let mut vote_losses = Vec::new();
3898 let vote_retries = self.state.config.graph.retries;
3899 let vote_cache = self.state.config.cache_dir();
3900 let ctx = WaveCtx {
3901 carry_seats: false,
3902 pauses: Some((&self.pause, &self.interrupt)),
3903 run: &run_id,
3904 node: "vote",
3905 prompts: &prompts,
3906 cache: vote_cache.as_deref(),
3907 round: None,
3908 };
3909 let votes = ask_json_wave::<FinalVote>(
3910 vote_jobs,
3911 Arc::clone(&self.sem),
3912 vote_retries,
3913 &self.roles.judge_roster,
3914 &ctx,
3915 &mut vote_losses,
3916 &mut self.state,
3917 &move |v: &FinalVote| match v.label() {
3918 Some(c) if allowed.contains(&c) => Ok(()),
3919 other => bail!("vote {other:?} is not one of {allowed:?}"),
3920 },
3921 )
3922 .await;
3923 if self.state.resource_block.is_some() {
3924 self.stop_for_resource()?;
3925 return Ok(false);
3926 }
3927 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3928 let agent_id = seat.agent.clone();
3929 self.state.seats.insert(seat.key.clone(), seat);
3930 match res {
3931 Ok((v, _)) => {
3932 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3933 rec.vote = v.label();
3934 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3935 } else {
3936 self.state.votes.push(VoteRecord {
3937 judge: j + 1,
3938 agent: agent_id,
3939 vote: v.label(),
3940 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3941 changed: false,
3942 });
3943 }
3944 self.state.event(
3945 "recover",
3946 format!("judge {} voted again after the limit", j + 1),
3947 );
3948 }
3949 Err(e) => {
3950 self.state
3951 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3952 }
3953 }
3954 }
3955
3956 let recovered_keys: BTreeSet<String> = recovered
3960 .iter()
3961 .map(|&j| format!("judge-{}", j + 1))
3962 .collect();
3963 self.state
3964 .quota
3965 .retain(|q| !recovered_keys.contains(&q.seat));
3966 for loss in judge_losses.into_iter().chain(vote_losses) {
3970 if recovered_keys.contains(&loss.seat) {
3971 continue;
3972 }
3973 self.state.quota.retain(|q| q.seat != loss.seat);
3974 self.state.quota.push(loss);
3975 }
3976
3977 self.state.tally = None;
3979 self.tally()?;
3980 Ok(self
3981 .state
3982 .tally
3983 .as_ref()
3984 .map(|t| t.met_quorum)
3985 .unwrap_or(false))
3986 }
3987
3988 async fn graft(&mut self) -> Result<()> {
4005 let Err(e) = self.graft_inner().await else {
4006 return Ok(());
4007 };
4008 if e.is::<crate::repo_lock::ResourceBlocked>() {
4009 return Err(e);
4010 }
4011 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4012 self.state.status = RunStatus::Blocked;
4013 self.state.event("graft", format!("blocked: {e:#}"));
4014 self.state.save()?;
4015 return Err(e);
4016 }
4017 if let Some(rec) = self.state.graft.clone()
4022 && !rec.outcome.settled()
4023 {
4024 if let Some(winner) = self.state.winner().cloned() {
4025 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4026 if re.is::<crate::repo_lock::ResourceBlocked>() {
4027 return Err(re);
4028 }
4029 self.state.status = RunStatus::Blocked;
4030 self.state.event("graft", format!("blocked: {re:#}"));
4031 self.state.save()?;
4032 return Err(re);
4033 }
4034 }
4035 if let Some(g) = self.state.graft.as_mut() {
4036 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4037 }
4038 }
4039 self.state.event("graft", format!("skipped: {e:#}"));
4040 if let Err(e) = self.state.save() {
4041 tracing::warn!("save after graft: {e:#}");
4042 }
4043 Ok(())
4044 }
4045
4046 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4049 let _coordination = self.coordinate(wt).await?;
4050 crate::recovery::restore_repair(wt, pre)
4051 .await
4052 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4053 }
4054
4055 fn graft_skipped(&mut self, why: &str) {
4057 let message = format!("skipped: {why}");
4058 if !self
4059 .state
4060 .events
4061 .iter()
4062 .any(|e| e.node == "graft" && e.message == message)
4063 {
4064 self.state.event("graft", message);
4065 }
4066 }
4067
4068 async fn graft_inner(&mut self) -> Result<()> {
4069 if let Some(rec) = self.state.graft.clone()
4072 && !rec.outcome.settled()
4073 {
4074 if self.state.reviews.is_empty()
4078 && let Some(winner) = self.state.winner().cloned()
4079 {
4080 self.graft_restore(&winner.worktree, &rec.pre).await?;
4081 }
4082 if rec.outcome == GraftOutcome::WaitingForRestore {
4083 self.state.graft = None;
4084 self.state.save()?;
4085 } else {
4086 if let Some(g) = self.state.graft.as_mut() {
4087 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4088 }
4089 self.state.event(
4090 "graft",
4091 "rolled back: the previous round was interrupted; the winner is untouched",
4092 );
4093 self.state.save()?;
4094 return Ok(());
4095 }
4096 }
4097 let plan = match graft::plan(&self.state) {
4098 Ok(plan) => plan,
4099 Err(skip) => {
4100 if skip != graft::Skip::AlreadyDone {
4101 self.graft_skipped(skip.reason());
4102 self.state.save()?;
4103 }
4104 return Ok(());
4105 }
4106 };
4107 let Some(winner) = self.state.winner().cloned() else {
4108 return Ok(());
4109 };
4110 let wt = winner.worktree.clone();
4111 if !git::is_clean(&wt).await? {
4112 self.graft_skipped("the winner's worktree has uncommitted changes");
4113 self.state.save()?;
4114 return Ok(());
4115 }
4116 let pre = git::rev_parse(&wt, "HEAD").await?;
4117 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4118 else {
4119 self.graft_skipped("the winner's diff could not be measured");
4120 self.state.save()?;
4121 return Ok(());
4122 };
4123 let limit = graft::limit(winner_lines);
4124 self.state.graft = Some(GraftRecord {
4127 pre: pre.clone(),
4128 post: None,
4129 lines: None,
4130 limit: Some(limit),
4131 agent: None,
4132 outcome: GraftOutcome::Running,
4133 requested: graft::requested(&plan),
4134 adoption: None,
4135 adoption_error: None,
4136 });
4137 self.state.event(
4138 "graft",
4139 format!(
4140 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4141 plan.strengths
4142 .iter()
4143 .map(|(l, _, _)| l.to_string())
4144 .collect::<Vec<_>>()
4145 .join(", "),
4146 plan.winner
4147 ),
4148 );
4149 self.state.save()?;
4150
4151 let fix_prompt = prompt::graft(
4152 &self.state.instruction,
4153 plan.winner,
4154 &plan.strengths,
4155 limit,
4156 &self.state.config.graph.language,
4157 );
4158 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4159 let sessions = self.state.config.graph.sessions;
4160 let artifacts = agent::artifacts_dir(&self.state.dir());
4161 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4162 let (_, seat, out) = self
4163 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4164 SeatJob {
4165 prompt: fix_prompt.clone(),
4166 spec,
4167 seat,
4168 cwd: wt.clone(),
4169 timeout,
4170 allow_write: true,
4171 sessions,
4172 artifacts: artifacts.clone(),
4173 stem: "graft".to_owned(),
4174 handover: None,
4175 }
4176 })
4177 .await;
4178 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4179 self.state.graft.as_mut().expect("graft record").outcome =
4182 GraftOutcome::WaitingForRestore;
4183 return self.stop_for_resource();
4184 }
4185 let mut why: Option<String> = None;
4186 match &out {
4187 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4188 why = Some("the fixer produced nothing usable".to_owned());
4189 }
4190 AgentOutcome::Ok(_) => {}
4191 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4192 AgentOutcome::Quota(o) => {
4193 self.state.quota.push(QuotaLoss {
4194 seat: seat.key.clone(),
4195 node: "graft".to_owned(),
4196 at: Timestamp::now(),
4197 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4198 });
4199 why = Some("rate limited (quota)".to_owned());
4200 }
4201 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4202 why = Some(format!(
4203 "the fixer failed: {}",
4204 e.lines()
4205 .next()
4206 .unwrap_or("")
4207 .chars()
4208 .take(160)
4209 .collect::<String>()
4210 ));
4211 }
4212 }
4213 let agent_id = seat.agent.clone();
4214 self.state.seats.insert(seat.key.clone(), seat);
4215
4216 let mut post = None;
4217 let mut lines = None;
4218 let _coordination = self.coordinate(&wt).await?;
4219 if why.is_none() {
4220 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4221 if let Ok(r) = &rescued {
4222 self.state.note_withheld("graft", &r.withheld);
4223 }
4224 let head = git::rev_parse(&wt, "HEAD").await?;
4225 let dirty = match &rescued {
4229 Ok(r) => git::status_porcelain(&wt)
4230 .await
4231 .map(|st| {
4232 st.lines().any(|l| {
4233 !r.withheld
4234 .iter()
4235 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4236 })
4237 })
4238 .unwrap_or(true),
4239 Err(_) => true,
4240 };
4241 if dirty {
4242 why = Some("the fixer left changes that could not be committed".to_owned());
4243 } else if head == pre {
4244 why = Some("the fixer changed nothing".to_owned());
4245 } else {
4246 lines = git::changed_lines(&wt, &pre, &head).await;
4247 if graft::within_budget(lines, limit) {
4248 post = Some(head);
4249 } else {
4250 why = Some(format!(
4251 "{} changed lines against a limit of {limit}",
4252 lines.map_or_else(
4253 || "an unmeasurable number of".to_owned(),
4254 |l| l.to_string()
4255 )
4256 ));
4257 }
4258 }
4259 }
4260 let (adoption, adoption_error) = match &out {
4261 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4262 _ => (None, None),
4263 };
4264 let requested = graft::requested(&plan);
4265 let outcome = match why {
4266 None => {
4267 self.state.event(
4268 "graft",
4269 format!(
4270 "applied: {} changed lines (limit {limit})",
4271 lines.unwrap_or(0)
4272 ),
4273 );
4274 GraftOutcome::Applied
4275 }
4276 Some(why) => {
4277 crate::recovery::restore_repair(&wt, &pre)
4280 .await
4281 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4282 self.state.event(
4283 "graft",
4284 format!("rolled back: {why}; the winner is untouched"),
4285 );
4286 GraftOutcome::RolledBack(why)
4287 }
4288 };
4289 self.state.graft = Some(GraftRecord {
4290 pre,
4291 post,
4292 lines,
4293 limit: Some(limit),
4294 agent: Some(agent_id),
4295 outcome,
4296 requested,
4297 adoption,
4298 adoption_error,
4299 });
4300 self.state.save()?;
4301 Ok(())
4302 }
4303
4304 async fn fold_losers(&mut self) -> Result<()> {
4307 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4308 return Ok(());
4309 };
4310 let repo = self.state.repo.clone();
4311 let _coordination = self.coordinate(&repo).await?;
4312 let mut folded = Vec::new();
4313 for i in 0..self.state.candidates.len() {
4314 let c = &self.state.candidates[i];
4315 if c.label == winner || c.folded {
4316 continue;
4317 }
4318 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4319 git::worktree_remove(&repo, &wt).await.ok();
4320 git::branch_delete(&repo, &branch).await.ok();
4321 self.state.candidates[i].folded = true;
4322 folded.push(label.to_string());
4323 }
4324 let root = self.state.worktree_root();
4326 for j in 1..=self.roles.judges.len() {
4327 let wt = root.join(format!("judge-{j}"));
4328 if wt.exists() {
4329 git::worktree_remove(&repo, &wt).await.ok();
4330 }
4331 }
4332 if self.state.config.graph.advise {
4335 for k in 1..=self.state.config.graph.advisors {
4336 let wt = root.join(format!("advisor-{k}"));
4337 if wt.exists() {
4338 git::worktree_remove(&repo, &wt).await.ok();
4339 }
4340 }
4341 }
4342 if !folded.is_empty() {
4343 self.state
4344 .event("fold", format!("folded candidates {}", folded.join(", ")));
4345 self.state.save()?;
4346 }
4347 Ok(())
4348 }
4349
4350 async fn sync_to_base(&mut self) -> Result<()> {
4390 if self.state.status == RunStatus::AlreadyInBase {
4391 return Ok(());
4392 }
4393 let conflicted = self
4394 .state
4395 .base_sync
4396 .as_ref()
4397 .is_some_and(|s| s.conflict.is_some());
4398 let Some(winner) = self.state.winner().cloned() else {
4399 return Ok(());
4400 };
4401
4402 let repo = self.state.repo.clone();
4403 let remote = self.state.config.merge.remote.clone();
4404 let base_branch = self.state.base_branch.clone();
4405 let tracking = format!("{remote}/{base_branch}");
4406
4407 git::fetch(&repo, &remote, &base_branch).await.ok();
4408 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4413 return Ok(());
4414 };
4415
4416 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4417 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4418 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4419
4420 if (behind > 0 || conflicted || head == tip)
4429 && self
4430 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4431 .await?
4432 {
4433 return Ok(());
4434 }
4435 if conflicted {
4436 return Ok(());
4437 }
4438
4439 if behind == 0 {
4440 if let Some(from) = self
4449 .state
4450 .rebase_fixes
4451 .iter()
4452 .rev()
4453 .find_map(|r| r.from.clone())
4454 && from != head
4455 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4456 .await
4457 .is_ok_and(|o| o.ok())
4458 {
4459 git::sync_to_head(&winner.worktree).await?;
4460 }
4461 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4467 if let Some(why) = &conflict {
4468 self.state.status = RunStatus::Blocked;
4469 self.state.event("land", why.clone());
4470 }
4471 self.state.base_sync = Some(BaseSync {
4472 tip,
4473 behind: 0,
4474 attempts,
4475 conflict,
4476 already_in: None,
4477 });
4478 self.state.save()?;
4479 return Ok(());
4480 }
4481
4482 if attempts >= BASE_SYNC_ROUNDS {
4483 let why = format!(
4484 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4485 rebase(s); rebasing again would only race it",
4486 winner.branch
4487 );
4488 self.state.status = RunStatus::Blocked;
4489 self.state.base_sync = Some(BaseSync {
4490 tip,
4491 behind,
4492 attempts,
4493 conflict: Some(why.clone()),
4494 already_in: None,
4495 });
4496 self.state.event("land", why);
4497 self.state.save()?;
4498 return Ok(());
4499 }
4500
4501 self.state.event(
4502 "land",
4503 format!(
4504 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4505 winner.branch
4506 ),
4507 );
4508 self.state.save()?;
4509
4510 let branch_tracking = format!("{remote}/{}", winner.branch);
4516 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4517 let remote_tip = if self.state.task_pr_branch().is_none()
4518 && matches!(&fetched_branch, Ok(o) if o.ok())
4519 {
4520 git::rev_parse(&repo, &branch_tracking).await.ok()
4521 } else {
4522 None
4523 };
4524 if let Some(theirs) = &remote_tip
4528 && !git::is_ancestor(&repo, theirs, &head).await
4529 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4530 .await
4531 .is_ok_and(|missing| missing.is_empty())
4532 {
4533 let why = format!(
4534 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4535 them",
4536 short(theirs),
4537 winner.branch
4538 );
4539 self.state.status = RunStatus::Blocked;
4540 self.state.base_sync = Some(BaseSync {
4541 tip,
4542 behind,
4543 attempts,
4544 conflict: Some(why.clone()),
4545 already_in: None,
4546 });
4547 self.state.event("land", why);
4548 self.state.save()?;
4549 return Ok(());
4550 }
4551
4552 let scratch = self.state.dir().join("base-sync");
4553 let rebased = match crate::rebase::rebase_with_fixer(
4554 &mut self.state,
4555 &scratch,
4556 &winner.branch,
4557 &tracking,
4558 )
4559 .await
4560 {
4561 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4562 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4563 Err(e) => Err(e),
4564 };
4565 let attempts = attempts + 1;
4566 match rebased {
4567 Ok(None) => {
4568 git::sync_to_head(&winner.worktree).await?;
4572 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4573 let mut conflict = None;
4574 if let Some(pinned) = &remote_tip {
4575 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4576 match pushed {
4577 Ok(o) if o.ok() => self.state.event(
4578 "land",
4579 format!("pushed rebased {} to {remote}", winner.branch),
4580 ),
4581 Ok(o) => {
4582 conflict = Some(format!(
4583 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4584 winner.branch,
4585 short(pinned),
4586 o.stderr.chars().take(600).collect::<String>()
4587 ));
4588 }
4589 Err(e) => {
4590 conflict = Some(format!(
4591 "rebased {} locally but could not push it: {e:#}",
4592 winner.branch
4593 ));
4594 }
4595 }
4596 }
4597 if let Some(why) = &conflict {
4598 self.state.status = RunStatus::Blocked;
4599 self.state.event("land", why.clone());
4600 }
4601 self.state.base_sync = Some(BaseSync {
4602 tip: tip.clone(),
4603 behind: 0,
4604 attempts,
4605 conflict,
4606 already_in: None,
4607 });
4608 self.state
4609 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4610 }
4611 Ok(Some(conflict)) => {
4612 let why = format!(
4613 "{} conflicts with {tracking} and did not rebase: {}",
4614 winner.branch,
4615 conflict.chars().take(600).collect::<String>()
4616 );
4617 self.state.status = RunStatus::Blocked;
4618 self.state.base_sync = Some(BaseSync {
4619 tip,
4620 behind,
4621 attempts,
4622 conflict: Some(why.clone()),
4623 already_in: None,
4624 });
4625 self.state.event("land", why);
4626 }
4627 Err(e) => {
4628 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4629 self.state.status = RunStatus::Blocked;
4630 self.state.base_sync = Some(BaseSync {
4631 tip,
4632 behind,
4633 attempts,
4634 conflict: Some(why.clone()),
4635 already_in: None,
4636 });
4637 self.state.event("land", why);
4638 }
4639 }
4640 self.state.save()?;
4641 Ok(())
4642 }
4643
4644 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4650 if self.state.task_pr_branch().is_some() {
4653 return None;
4654 }
4655 let repo = self.state.repo.clone();
4656 let remote = self.state.config.merge.remote.clone();
4657 let fetched = git::fetch(&repo, &remote, branch).await;
4658 if !matches!(&fetched, Ok(o) if o.ok()) {
4659 return None;
4660 }
4661 let branch_tracking = format!("{remote}/{branch}");
4662 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4663 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4664 return None;
4665 }
4666 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4667 .await
4668 .is_ok_and(|missing| missing.is_empty())
4669 {
4670 return Some(format!(
4671 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4672 them",
4673 short(&theirs)
4674 ));
4675 }
4676 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4677 Ok(o) if o.ok() => {
4678 self.state
4679 .event("land", format!("pushed rebased {branch} to {remote}"));
4680 None
4681 }
4682 Ok(o) => Some(format!(
4683 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4684 someone may have pushed): {}",
4685 short(&theirs),
4686 o.stderr.chars().take(600).collect::<String>()
4687 )),
4688 Err(e) => Some(format!(
4689 "{branch} is rebased locally but could not be pushed: {e:#}"
4690 )),
4691 }
4692 }
4693
4694 async fn settle_already_in(
4711 &mut self,
4712 branch: &str,
4713 tip: &str,
4714 head: &str,
4715 attempts: usize,
4716 behind: usize,
4717 ) -> Result<bool> {
4718 let repo = self.state.repo.clone();
4719 let remote = self.state.config.merge.remote.clone();
4720 let start = self.state.base_commit.clone();
4721 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4722 Ok(Some(e)) => e,
4723 Ok(None) => return Ok(false),
4724 Err(e) => {
4725 tracing::warn!("already-in-base check for {branch}: {e:#}");
4726 return Ok(false);
4727 }
4728 };
4729 let mut verified = vec![head.to_owned()];
4730 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4731 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4732 if matches!(&fetched, Ok(o) if o.ok())
4733 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4734 && theirs != head
4735 {
4736 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4737 Ok(Some(_)) => verified.push(theirs),
4738 _ => return Ok(false),
4739 }
4740 }
4741 let base_branch = self.state.base_branch.clone();
4742 let message = format!(
4743 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4744 land",
4745 evidence.names(),
4746 evidence.proof.as_str()
4747 );
4748 let closed =
4749 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4750 .await;
4751 match closed {
4752 Ok(Ok(url)) => self
4753 .state
4754 .event("land", format!("closed {url}: superseded on {base_branch}")),
4755 Ok(Err(why)) => self
4756 .state
4757 .event("land", format!("did not close a pull request: {why}")),
4758 Err(e) => {
4759 let why = format!(
4760 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4761 not be closed ({e:#}); resume to retry"
4762 );
4763 self.state.status = RunStatus::Blocked;
4764 self.state.base_sync = Some(BaseSync {
4765 tip: tip.to_owned(),
4766 behind,
4767 attempts,
4768 conflict: Some(why.clone()),
4769 already_in: None,
4770 });
4771 self.state.event("land", why);
4772 self.state.save()?;
4773 return Ok(true);
4774 }
4775 }
4776 self.state.status = RunStatus::AlreadyInBase;
4777 self.state.base_sync = Some(BaseSync {
4778 tip: tip.to_owned(),
4779 behind,
4780 attempts,
4781 conflict: None,
4782 already_in: Some(evidence),
4783 });
4784 self.state.event("land", message);
4785 self.state.save()?;
4786 self.settle_questions();
4787 Ok(true)
4788 }
4789
4790 fn landing_base(&self) -> String {
4800 self.state
4801 .base_sync
4802 .as_ref()
4803 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4804 }
4805
4806 pub async fn fix_selected(
4839 &mut self,
4840 ids: &[String],
4841 reason: &str,
4842 allow_stale: bool,
4843 ) -> Result<()> {
4844 let reason = reason.trim();
4845 if reason.is_empty() {
4846 bail!("a fix request needs a reason — that is the operator's own record of why");
4847 }
4848 if ids.is_empty() {
4849 bail!("no finding id given");
4850 }
4851 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4852 bail!(
4853 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4854 has already concluded — can be given a targeted fix. A run still \
4855 in progress should simply be resumed; a `merged` run's branch has \
4856 already landed, so its answer is a fresh `magi review <branch>`, \
4857 not reopening this run's own record",
4858 self.state.id,
4859 self.state.status.as_str()
4860 );
4861 }
4862 let Some(winner) = self.state.winner().cloned() else {
4863 bail!("run {} has no winning candidate to fix", self.state.id);
4864 };
4865 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4866 bail!(
4867 "branch `{}` no longer exists; this run cannot be extended",
4868 winner.branch
4869 );
4870 }
4871 let home = crate::run::home();
4872 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4873 bail!(
4874 "run {} is currently being worked on by another magi process",
4875 self.state.id
4876 );
4877 }
4878 let _claim = FixClaim::acquire(&self.state.dir())?;
4884
4885 let mut seen = BTreeSet::new();
4889 let mut findings = Vec::new();
4890 let mut missing = Vec::new();
4891 for id in ids {
4892 if !seen.insert(id.clone()) {
4893 continue;
4894 }
4895 match self.state.finding(id) {
4896 Some((round, rec, f)) => findings.push(OperatorFixFinding {
4897 id: f.id.clone(),
4898 severity: f.severity,
4899 reviewer_vote: rec.vote,
4900 round: round.round,
4901 round_head: round.head.clone(),
4902 reviewer: rec.reviewer,
4903 agent: rec.agent.clone(),
4904 file: f.file.clone(),
4905 line: f.line,
4906 title: f.title.clone(),
4907 detail: f.detail.clone(),
4908 outcome: OperatorFixOutcome::Pending,
4909 }),
4910 None => missing.push(id.clone()),
4911 }
4912 }
4913 if !missing.is_empty() {
4914 bail!(
4915 "unknown finding id(s): {}; nothing was changed",
4916 missing.join(", ")
4917 );
4918 }
4919
4920 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
4921 let stale_details: Vec<(String, String)> = findings
4922 .iter()
4923 .filter(|f| f.round_head != head_at_request)
4924 .map(|f| (f.id.clone(), f.round_head.clone()))
4925 .collect();
4926 let stale = !stale_details.is_empty();
4927 if stale && !allow_stale {
4928 bail!(
4929 "the branch has moved since some finding(s) were raised — {} — now \
4930 at {}; pass --allow-stale to fix anyway, or re-run review first",
4931 stale_details
4932 .iter()
4933 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4934 .collect::<Vec<_>>()
4935 .join(", "),
4936 short(&head_at_request)
4937 );
4938 }
4939
4940 let request = OperatorFixRequest {
4941 requested_at: Timestamp::now(),
4942 reason: reason.to_owned(),
4943 findings,
4944 head_at_request: head_at_request.clone(),
4945 allow_stale,
4946 stale,
4947 fix: None,
4948 result_head: None,
4949 follow_up_review_run: None,
4950 };
4951 self.state.event(
4952 "fix",
4953 format!(
4954 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4955 request.findings.len(),
4956 request
4957 .findings
4958 .iter()
4959 .map(|f| f.id.as_str())
4960 .collect::<Vec<_>>()
4961 .join(", "),
4962 ),
4963 );
4964 self.state.operator_fixes.push(request);
4971 self.state.save()?;
4972 let request_index = self.state.operator_fixes.len() - 1;
4973
4974 let repo = self.state.repo.clone();
4983 let coordination = self.coordinate(&repo).await?;
4984 if winner.worktree.exists() {
4985 let dirty = git::git(
4988 &winner.worktree,
4989 &["status", "--porcelain", "--untracked-files=all"],
4990 )
4991 .await?;
4992 let only_withheld = dirty.lines().all(|l| {
4993 l.strip_prefix("?? ")
4994 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4995 });
4996 if !only_withheld {
4997 bail!(
4998 "`{}` has uncommitted changes; refusing to touch it — commit or \
4999 discard them first",
5000 winner.worktree.display()
5001 );
5002 }
5003 git::worktree_remove(&self.state.repo, &winner.worktree)
5004 .await
5005 .ok();
5006 }
5007 let fix_worktree = self.state.worktree_root().join("operator-fix");
5008 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5009 git::git(
5010 &self.state.repo,
5011 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5012 )
5013 .await
5014 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5015 if !git::is_clean(&fix_worktree).await? {
5016 git::worktree_remove(&self.state.repo, &fix_worktree)
5017 .await
5018 .ok();
5019 bail!(
5020 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5021 winner.branch
5022 );
5023 }
5024 let repo_for_setup = self.state.repo.clone();
5025 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5026 Ok(()) => {
5027 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5028 }
5029 Err(e) => Err(e),
5030 };
5031 if let Err(e) = setup {
5032 git::worktree_remove(&self.state.repo, &fix_worktree)
5033 .await
5034 .ok();
5035 return Err(e);
5036 }
5037
5038 let run_id = self.state.id.clone();
5039 let prompts = self.state.config.prompts.clone();
5040 let language = self.state.config.graph.language.clone();
5041 let sessions = self.state.config.graph.sessions;
5042 let artifacts = agent::artifacts_dir(&self.state.dir());
5043 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5044 .findings
5045 .iter()
5046 .map(|f| Finding {
5047 id: f.id.clone(),
5048 severity: f.severity,
5049 file: f.file.clone(),
5050 line: f.line,
5051 title: f.title.clone(),
5052 detail: f.detail.clone(),
5053 })
5054 .collect();
5055 let fix_prompt = prompt::operator_fix_in(
5056 &self.state.instruction,
5057 &finding_list,
5058 reason,
5059 &stale_details,
5060 &head_at_request,
5061 &language,
5062 &self.state.config.graph.github_language,
5063 );
5064 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5065 drop(coordination);
5066 let (job, seat, out) = self
5067 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5068 prompt: fix_prompt.clone(),
5069 spec,
5070 seat,
5071 cwd: fix_worktree.clone(),
5072 timeout,
5073 allow_write: true,
5074 sessions,
5075 artifacts: artifacts.clone(),
5076 stem: "operator-fix".to_owned(),
5077 handover: None,
5078 })
5079 .await;
5080 let agent_id = seat.agent.clone();
5081
5082 let mut fix = FixRecord {
5083 agent: agent_id,
5084 addressed: Vec::new(),
5085 rejected: Vec::new(),
5086 notes: String::new(),
5087 committed: false,
5088 failed: None,
5089 duration_ms: 0,
5090 continuation: None,
5091 };
5092 let mut final_seat = seat.clone();
5093 match out {
5094 AgentOutcome::Ok(o) => {
5095 fix.duration_ms = o.duration_ms;
5096 let parsed = verdict::extract_json::<FixReport>(&o.text);
5097 let incomplete_reason = match &parsed {
5098 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5099 "the reply parsed, but it reported a command whose own CLI \
5100 never confirmed an exit status"
5101 .to_owned(),
5102 ),
5103 Ok(_) => None,
5104 Err(e) => Some(e.to_string()),
5105 };
5106 match incomplete_reason {
5107 None => {
5108 let report = parsed.expect("checked Ok above");
5109 fix.addressed = report.addressed;
5110 fix.rejected = report.rejected;
5111 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5112 }
5113 Some(reason) => {
5114 let (resumed_seat, resolved, failure, cont) = self
5115 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5116 .await;
5117 fix.duration_ms += cont.cumulative_wait_ms;
5118 fix.continuation = Some(cont);
5119 final_seat = resumed_seat;
5120 match resolved {
5121 Some(report) => {
5122 fix.addressed = report.addressed;
5123 fix.rejected = report.rejected;
5124 fix.notes =
5125 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5126 }
5127 None => fix.failed = failure,
5128 }
5129 }
5130 }
5131 }
5132 AgentOutcome::Dropped(o) => {
5133 fix.duration_ms = o.duration_ms;
5134 let why = o
5135 .dropped
5136 .as_ref()
5137 .map(|d| d.why.as_str())
5138 .unwrap_or("the CLI ended the stream without delivering its answer");
5139 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5140 }
5141 AgentOutcome::Quota(o) => {
5142 self.state.quota.push(QuotaLoss {
5143 seat: final_seat.key.clone(),
5144 node: "fix".to_owned(),
5145 at: Timestamp::now(),
5146 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5147 });
5148 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5149 }
5150 AgentOutcome::Failed(e) => fix.failed = Some(e),
5151 AgentOutcome::ResourceBlocked(e) => {
5152 fix.failed = Some(format!("the fixer was not started: {e}"));
5153 }
5154 }
5155 if fix.continuation.is_none() {
5156 fix.continuation = Some(ContinuationRecord::not_needed());
5157 }
5158 self.state.seats.insert(final_seat.key.clone(), final_seat);
5159
5160 let rescue_message = format!(
5161 "magi: operator-selected fix ({}) (uncommitted work)",
5162 self.state.operator_fixes[request_index]
5163 .findings
5164 .iter()
5165 .map(|f| f.id.as_str())
5166 .collect::<Vec<_>>()
5167 .join(", ")
5168 );
5169 let repo = self.state.repo.clone();
5170 let coordination = self.coordinate(&repo).await?;
5171 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5172 self.state.note_withheld("fix", &r.withheld);
5173 }
5174 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5175 fix.committed = after != head_at_request;
5176 git::worktree_remove(&self.state.repo, &fix_worktree)
5177 .await
5178 .ok();
5179 drop(coordination);
5180
5181 self.state.event(
5182 "fix",
5183 match &fix.failed {
5184 Some(reason) => format!(
5185 "operator fix: adoption report was lost ({reason}); {}",
5186 if fix.committed {
5187 "committed"
5188 } else {
5189 "NO new commit"
5190 }
5191 ),
5192 None => format!(
5193 "operator fix: {} addressed, {} rejected, {}",
5194 fix.addressed.len(),
5195 fix.rejected.len(),
5196 if fix.committed {
5197 "committed"
5198 } else {
5199 "NO new commit"
5200 }
5201 ),
5202 },
5203 );
5204
5205 for f in &mut self.state.operator_fixes[request_index].findings {
5212 f.outcome = if fix.failed.is_some() {
5213 OperatorFixOutcome::Unreported
5214 } else if fix.addressed.contains(&f.id) {
5215 OperatorFixOutcome::Addressed
5216 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5217 OperatorFixOutcome::Rejected { why: r.why.clone() }
5218 } else {
5219 OperatorFixOutcome::Unreported
5220 };
5221 }
5222
5223 let committed = fix.committed;
5224 if committed {
5225 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5226 }
5227 self.state.operator_fixes[request_index].fix = Some(fix);
5228 self.state.save()?;
5231
5232 if committed {
5233 self.state.event(
5234 "fix",
5235 format!(
5236 "operator fix committed {}; opening a follow-up review-only run",
5237 short(&after)
5238 ),
5239 );
5240 let origin =
5243 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5244 match Self::review(
5245 &self.state.repo,
5246 &winner.branch,
5247 self.state.config.clone(),
5248 origin,
5249 )
5250 .await
5251 {
5252 Ok(mut follow_up) => {
5253 follow_up.state.event(
5254 "start",
5255 format!(
5256 "requested by an operator fix on run {} for finding(s) {}",
5257 self.state.id,
5258 self.state.operator_fixes[request_index]
5259 .findings
5260 .iter()
5261 .map(|f| f.id.as_str())
5262 .collect::<Vec<_>>()
5263 .join(", "),
5264 ),
5265 );
5266 follow_up.state.save()?;
5267 let follow_up_id = follow_up.state.id.clone();
5268 let adopted = match crate::direct::adopt(
5271 &follow_up.state,
5272 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5273 ) {
5274 Ok(a) => a,
5275 Err(e) => {
5276 self.state.event(
5279 "fix",
5280 format!(
5281 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5282 ),
5283 );
5284 self.state.save()?;
5285 return Ok(());
5286 }
5287 };
5288 let executed = follow_up.execute().await;
5289 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5290 if let Some(a) = adopted {
5291 a.finish(&follow_up.state, executed);
5292 }
5293 if let Some(e) = failure {
5294 self.state.event(
5295 "fix",
5296 format!(
5297 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5298 ),
5299 );
5300 }
5301 self.state.operator_fixes[request_index].follow_up_review_run =
5302 Some(follow_up_id);
5303 }
5304 Err(e) => {
5305 self.state.event(
5306 "fix",
5307 format!("committed the fix but could not open a follow-up review: {e:#}"),
5308 );
5309 }
5310 }
5311 self.state.save()?;
5312 }
5313
5314 Ok(())
5315 }
5316
5317 async fn ask_fixer(
5337 &mut self,
5338 winner: &Candidate,
5339 node: &'static str,
5340 round: Option<usize>,
5341 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5342 ) -> (SeatJob, SeatState, AgentOutcome) {
5343 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5344 self.ask_fixer_among(attempts, node, round, None, build)
5345 .await
5346 }
5347
5348 async fn ask_fixer_among(
5353 &mut self,
5354 attempts: Vec<(AgentSpec, String)>,
5355 node: &'static str,
5356 round: Option<usize>,
5357 restore: Option<(&Path, &str)>,
5358 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5359 ) -> (SeatJob, SeatState, AgentOutcome) {
5360 let run_id = self.state.id.clone();
5361 let prompts = self.state.config.prompts.clone();
5362 let cache = self.state.config.cache_dir();
5363 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5364 let mut last = None;
5365 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5366 let seat = self.seat(&key, &spec.id);
5367 let mut job = build(spec.clone(), seat);
5368 if i > 0 {
5369 job.stem = format!("{}-{}", job.stem, spec.id);
5370 }
5371 let ctx = WaveCtx {
5372 carry_seats: false,
5373 pauses: Some((&self.pause, &self.interrupt)),
5374 run: &run_id,
5375 node,
5376 prompts: &prompts,
5377 cache: cache.as_deref(),
5378 round,
5379 };
5380 let (seat, out) =
5381 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5382 let advances = match &out {
5385 AgentOutcome::Ok(o) => agent::output_advances(o),
5386 AgentOutcome::ResourceBlocked(_) => false,
5387 _ => true,
5388 };
5389 if let Some(next) = ids.get(i + 1)
5390 && advances
5391 {
5392 self.state.seats.insert(seat.key.clone(), seat.clone());
5393 let class =
5394 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5395 let reason = match &out {
5396 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5397 other => fail_reason(other),
5398 };
5399 if let Some((wt, rev)) = restore
5403 && let Err(e) = self.graft_restore(wt, rev).await
5404 {
5405 let why =
5406 format!("could not restore the worktree before the next fixer: {e:#}");
5407 tracing::warn!("{why}");
5408 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5409 AgentOutcome::ResourceBlocked(why)
5410 } else {
5411 AgentOutcome::Failed(why)
5412 };
5413 last = Some((job, seat, out));
5414 break;
5415 }
5416 record_handover(
5417 &mut self.state,
5418 node,
5419 &seat.key,
5420 &spec.id,
5421 next,
5422 &class,
5423 &reason,
5424 );
5425 continue;
5426 }
5427 last = Some((job, seat, out));
5428 break;
5429 }
5430 last.expect("the fixer chain always has an entry")
5431 }
5432
5433 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5444 use crate::recovery::{Mode, Repair};
5445 if !self.state.reviews.is_empty() {
5446 return Ok(());
5447 }
5448 let Some(mut rec) = self.state.recovery.clone() else {
5449 return Ok(());
5450 };
5451 if rec.mode != Mode::VerifyRepair {
5452 return Ok(());
5453 }
5454 let wt = winner.worktree.clone();
5455 let mut head = git::rev_parse(&wt, "HEAD").await?;
5456 match rec.repair.clone() {
5457 Repair::Done { .. } => return Ok(()),
5458 Repair::WaitingForRestore { before } => {
5459 self.graft_restore(&wt, &before).await?;
5460 head = before;
5461 rec.repair = Repair::NotStarted;
5462 self.state.recovery = Some(rec.clone());
5463 self.state.save()?;
5464 }
5465 Repair::Started { before } => {
5466 if let Err(e) = self.graft_restore(&wt, &before).await {
5470 if e.is::<crate::repo_lock::ResourceBlocked>() {
5471 return Err(e);
5472 }
5473 return self
5474 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5475 .await;
5476 }
5477 rec.repair = Repair::Done {
5478 before: before.clone(),
5479 after: before,
5480 committed: false,
5481 failed: Some("interrupted; rolled back".to_owned()),
5482 };
5483 self.state
5484 .event("recovery", "the repair was interrupted and rolled back");
5485 self.state.recovery = Some(rec);
5486 self.state.save()?;
5487 return Ok(());
5488 }
5489 Repair::NotStarted => {}
5490 }
5491 if crate::recovery::recheck(&mut rec, &head) {
5492 self.state.event("recovery", rec.summary());
5493 self.state.recovery = Some(rec);
5494 self.state.save()?;
5495 return Ok(());
5496 }
5497 rec.repair = Repair::Started {
5498 before: head.clone(),
5499 };
5500 self.state.recovery = Some(rec.clone());
5501 self.state.save()?;
5502
5503 let language = self.state.config.graph.language.clone();
5504 let sessions = self.state.config.graph.sessions;
5505 let artifacts = agent::artifacts_dir(&self.state.dir());
5506 let fix_prompt = prompt::recovery_fix(
5507 &self.state.instruction,
5508 &rec,
5509 self.state.config.graph.review_rounds,
5510 &language,
5511 );
5512 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5513 self.state.event(
5514 "recovery",
5515 format!(
5516 "repair: the fixer starts from the failed verification on {}",
5517 short(&head)
5518 ),
5519 );
5520 let (_job, seat, out) = self
5521 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5522 prompt: fix_prompt.clone(),
5523 spec,
5524 seat,
5525 cwd: wt.clone(),
5526 timeout,
5527 allow_write: true,
5528 sessions,
5529 artifacts: artifacts.clone(),
5530 stem: "fix-recovery".to_owned(),
5531 handover: None,
5532 })
5533 .await;
5534 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5535 rec.repair = Repair::WaitingForRestore { before: head };
5536 self.state.recovery = Some(rec);
5537 return self.stop_for_resource();
5538 }
5539 let failed = match &out {
5540 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5541 AgentOutcome::Quota(o) => {
5542 self.state.quota.push(QuotaLoss {
5543 seat: seat.key.clone(),
5544 node: "fix".to_owned(),
5545 at: Timestamp::now(),
5546 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5547 });
5548 Some("rate limited (quota); the fixer could not run".to_owned())
5549 }
5550 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5551 AgentOutcome::Failed(e) => Some(e.clone()),
5552 };
5553 self.state.seats.insert(seat.key.clone(), seat);
5554 let coordination = self.coordinate(&wt).await?;
5555 let failed = match failed {
5556 Some(why) => {
5558 if let Err(e) = git::restore_to(&wt, &head).await {
5559 return self
5560 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5561 .await;
5562 }
5563 Some(why)
5564 }
5565 None => {
5566 if let Ok(r) = git::rescue_commit(
5567 &wt,
5568 "magi: recovery repair of the failed verification (uncommitted work)",
5569 )
5570 .await
5571 {
5572 self.state.note_withheld("fix", &r.withheld);
5573 }
5574 None
5575 }
5576 };
5577 let after = git::rev_parse(&wt, "HEAD").await?;
5578 drop(coordination);
5579 self.state.event(
5580 "recovery",
5581 match &failed {
5582 Some(why) => format!("repair: nothing kept ({why})"),
5583 None if after != head => {
5584 format!("repair: committed {}", short(&after))
5585 }
5586 None => "repair: the fixer made no commit".to_owned(),
5587 },
5588 );
5589 rec.repair = Repair::Done {
5590 before: head.clone(),
5591 committed: after != head,
5592 after,
5593 failed,
5594 };
5595 self.state.recovery = Some(rec);
5596 self.state.save()?;
5597 Ok(())
5598 }
5599
5600 async fn stop_recovery(
5603 &mut self,
5604 mut rec: crate::recovery::RecoveryRecord,
5605 head: &str,
5606 why: String,
5607 ) -> Result<()> {
5608 rec.repair = crate::recovery::Repair::Started {
5610 before: head.to_owned(),
5611 };
5612 self.state.recovery = Some(rec);
5613 self.state
5614 .event("recovery", format!("repair could not be undone: {why}"));
5615 self.state.status = RunStatus::Blocked;
5616 self.state.save()?;
5617 bail!("recovery repair could not be rolled back: {why}")
5618 }
5619
5620 async fn review_loop(&mut self) -> Result<()> {
5621 if self
5626 .state
5627 .base_sync
5628 .as_ref()
5629 .is_some_and(|s| s.conflict.is_some())
5630 {
5631 return Ok(());
5632 }
5633 let run_id = self.state.id.clone();
5638 let prompts = self.state.config.prompts.clone();
5639 let Some(winner) = self.state.winner().cloned() else {
5640 return Ok(());
5641 };
5642 let max_rounds = self.state.config.graph.review_rounds;
5643 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5653 if status == RunStatus::Gating {
5656 self.record_contested_handoff();
5657 }
5658 self.state.status = status;
5659 self.state.save()?;
5660 return Ok(());
5661 }
5662 self.state.status = RunStatus::Reviewing;
5663 if self
5673 .state
5674 .reviews
5675 .last()
5676 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5677 {
5678 let shell = self.state.config.shell();
5679 return self
5680 .stop_reviewing(
5681 "the last round's own verification never resolved",
5682 &shell,
5683 &winner.worktree,
5684 )
5685 .await;
5686 }
5687
5688 self.recovery_repair(&winner).await?;
5689 if self.resource_halt()? {
5690 return Ok(());
5691 }
5692
5693 let repo = self.state.repo.clone();
5694 let root = self.state.worktree_root();
5695 let language = self.state.config.graph.language.clone();
5696 let sessions = self.state.config.graph.sessions;
5697 let artifacts = self.merge_repair_artifacts();
5698 let base = self.landing_base();
5699 let base_short = short(&base);
5700 let reviewers = self.roles.reviewers.clone();
5701 let shell = self.state.config.shell();
5702
5703 for round in (self.state.reviews.len() + 1)..=max_rounds {
5704 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5705 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5706 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5707 let prev_verification = self
5716 .state
5717 .reviews
5718 .last()
5719 .and_then(|r| r.verification_summary(&head));
5720 let prior_section = match (&self.state.recovery, round) {
5723 (Some(rec), 1) if self.state.reviews.is_empty() => {
5724 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5725 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5726 Some(crate::recovery::head_of(
5727 &git::diff(&winner.worktree, reviewed, "HEAD")
5728 .await
5729 .unwrap_or_default(),
5730 crate::recovery::SINCE_DIFF_MAX,
5731 ))
5732 }
5733 _ => None,
5734 };
5735 Some(prompt::recovery_review(rec, since.as_deref()))
5736 }
5737 _ => None,
5738 };
5739
5740 let mut jobs = Vec::new();
5744 let coordination = self.coordinate(&repo).await?;
5745 for (r, spec) in reviewers.iter().cloned().enumerate() {
5746 let wt = root.join(format!("review-{}", r + 1));
5747 if wt.exists() {
5750 git::reset_detached(&wt, &head).await?;
5751 } else {
5752 git::worktree_add_detached(&repo, &wt, &head).await?;
5753 }
5754 self.setup_seat_worktree(&repo, &wt, false).await?;
5755 let seat_key = format!("review-{}", r + 1);
5756 let spec = pick_start_spec(
5759 &self.roles.reviewer_roster,
5760 spec,
5761 self.state.seat_history.get(&seat_key),
5762 );
5763 let seat = self.seat(&seat_key, &spec.id);
5764 jobs.push(SeatJob {
5765 prompt: prompt::review(&prompt::ReviewCtx {
5766 instruction: &self.state.instruction,
5767 branch: &winner.branch,
5768 base_short: &base_short,
5769 stat: &stat,
5770 patch: &patch,
5771 verification: prev_verification.as_ref(),
5772 prior: prior_section.as_deref(),
5773 reviewers: reviewers.len(),
5774 round,
5775 rounds: max_rounds,
5776 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5779 lens: Lens::for_seat(r),
5780 language: &language,
5781 github_language: &self.state.config.graph.github_language,
5782 }),
5783 spec,
5784 seat,
5785 cwd: wt,
5786 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5787 allow_write: false,
5788 sessions,
5789 artifacts: artifacts.clone(),
5790 stem: format!("review-{round}-{}", r + 1),
5791 handover: None,
5792 });
5793 }
5794
5795 drop(coordination);
5796 self.state.event(
5797 "review",
5798 format!(
5799 "round {round}: {} reviewers on {}",
5800 jobs.len(),
5801 short(&head)
5802 ),
5803 );
5804 let mut quota_losses = Vec::new();
5805 let review_retries = self.state.config.graph.retries;
5806 let review_cache = self.state.config.cache_dir();
5807 let ctx = WaveCtx {
5808 carry_seats: true,
5809 pauses: Some((&self.pause, &self.interrupt)),
5810 run: &run_id,
5811 node: "review",
5812 prompts: &prompts,
5813 cache: review_cache.as_deref(),
5814 round: Some(round),
5815 };
5816 let results = ask_json_wave::<Review>(
5817 jobs,
5818 Arc::clone(&self.sem),
5819 review_retries,
5820 &self.roles.reviewer_roster,
5821 &ctx,
5822 &mut quota_losses,
5823 &mut self.state,
5824 &|_: &Review| Ok(()),
5825 )
5826 .await;
5827 if self.state.resource_block.is_some() {
5828 return self.stop_for_resource();
5829 }
5830 let round_quota_missing = quota_losses.len();
5834 self.state.quota.extend(quota_losses);
5835
5836 let mut records = Vec::new();
5837 let mut all_findings = Vec::new();
5838 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5839 let agent_id = seat.agent.clone();
5840 self.state.seats.insert(seat.key.clone(), seat);
5841 let mut record = ReviewRecord {
5842 reviewer: r + 1,
5843 agent: agent_id,
5844 summary: String::new(),
5845 findings: Vec::new(),
5846 vote: None,
5847 failed: None,
5848 duration_ms: 0,
5849 attempts,
5855 };
5856 match res {
5857 Ok((review, out)) => {
5858 record.summary =
5866 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5867 record.vote = Some(review.vote);
5868 record.duration_ms = out.duration_ms;
5869 for (n, mut f) in review.findings.into_iter().enumerate() {
5870 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5873 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5874 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5875 f.file = f
5881 .file
5882 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
5883 all_findings.push(f.clone());
5884 record.findings.push(f);
5885 }
5886 self.state.event(
5887 "review",
5888 format!(
5889 "round {round}: reviewer {} voted {} with {} finding(s)",
5890 r + 1,
5891 review.vote.label(),
5892 record.findings.len()
5893 ),
5894 );
5895 }
5896 Err(e) => {
5897 record.failed = Some(e.to_string());
5898 self.state.event(
5899 "review",
5900 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
5901 );
5902 }
5903 }
5904 records.push(record);
5905 }
5906
5907 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
5914 let vote_split =
5915 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
5916 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
5917 if vote_split {
5918 self.state.event(
5919 "review",
5920 format!(
5921 "round {round}: votes split ({}) — one round of reconsideration",
5922 initial_votes
5923 .iter()
5924 .map(|v| v.label())
5925 .collect::<Vec<_>>()
5926 .join(", ")
5927 ),
5928 );
5929 let panel: Vec<ReviewSeatReport<'_>> = records
5932 .iter()
5933 .filter_map(|r| {
5934 r.vote.map(|vote| ReviewSeatReport {
5935 reviewer: r.reviewer,
5936 vote,
5937 summary: &r.summary,
5938 findings: &r.findings,
5939 })
5940 })
5941 .collect();
5942
5943 let mut jobs = Vec::new();
5944 let mut seats_at = Vec::new();
5945 for (r, spec) in reviewers.iter().cloned().enumerate() {
5946 if records[r].vote.is_none() {
5950 continue;
5951 }
5952 let wt = root.join(format!("review-{}", r + 1));
5953 let seat_key = format!("review-{}", r + 1);
5954 let spec = self.occupant(&seat_key, spec);
5955 let seat = self.seat(&seat_key, &spec.id);
5956 let build = |with_patch: bool| {
5964 prompt::review_reconsider(&ReviewReconsiderCtx {
5965 instruction: &self.state.instruction,
5966 reviewer: r + 1,
5967 lens: Lens::for_seat(r),
5968 panel: &panel,
5969 patch: with_patch.then_some(ReviewPatch {
5970 branch: &winner.branch,
5971 base_short: &base_short,
5972 stat: &stat,
5973 patch: &patch,
5974 }),
5975 round,
5976 rounds: max_rounds,
5977 language: &language,
5978 })
5979 };
5980 let full = build(true);
5981 let prompt = if has_context(&spec, &seat, sessions) {
5982 build(false)
5983 } else {
5984 full.clone()
5985 };
5986 jobs.push(SeatJob {
5987 prompt,
5988 spec,
5989 seat,
5990 cwd: wt,
5991 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5992 allow_write: false,
5993 sessions,
5994 artifacts: artifacts.clone(),
5995 stem: format!("review-{round}-reconsider-{}", r + 1),
5996 handover: Some(full),
5997 });
5998 seats_at.push(r);
5999 }
6000
6001 let mut recon_quota_losses = Vec::new();
6002 let recon_cache = self.state.config.cache_dir();
6003 let recon_ctx = WaveCtx {
6004 carry_seats: true,
6005 pauses: Some((&self.pause, &self.interrupt)),
6006 run: &run_id,
6007 node: "review",
6008 prompts: &prompts,
6009 cache: recon_cache.as_deref(),
6010 round: Some(round),
6011 };
6012 let recon_results = ask_json_wave::<ReviewRevote>(
6013 jobs,
6014 Arc::clone(&self.sem),
6015 review_retries,
6016 &self.roles.reviewer_roster,
6017 &recon_ctx,
6018 &mut recon_quota_losses,
6019 &mut self.state,
6020 &|_: &ReviewRevote| Ok(()),
6021 )
6022 .await;
6023 if self.state.resource_block.is_some() {
6024 return self.stop_for_resource();
6025 }
6026 self.state.quota.extend(recon_quota_losses);
6027
6028 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6029 let agent_id = seat.agent.clone();
6030 self.state.seats.insert(seat.key.clone(), seat);
6031 let mut rec = ReviewRevoteRecord {
6032 reviewer: r + 1,
6033 agent: agent_id,
6034 vote: None,
6035 reason: String::new(),
6036 failed: None,
6037 };
6038 match res {
6039 Ok((rv, _)) => {
6040 rec.vote = Some(rv.vote);
6041 rec.reason =
6042 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6043 self.state.event(
6044 "review",
6045 format!(
6046 "round {round}: reviewer {} revoted {}",
6047 r + 1,
6048 rv.vote.label()
6049 ),
6050 );
6051 }
6052 Err(e) => {
6053 rec.failed = Some(e.to_string());
6054 self.state.event(
6055 "review",
6056 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6057 );
6058 }
6059 }
6060 reconsideration.push(rec);
6061 }
6062 } else if initial_votes.len() > 1 {
6063 self.state.event(
6064 "review",
6065 format!(
6066 "round {round}: votes agreed ({}) — no reconsideration",
6067 initial_votes[0].label()
6068 ),
6069 );
6070 }
6071
6072 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6073 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6074 let defer_e2e =
6085 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6086 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6087 let reason =
6088 format!("{blocking} blocking finding(s) already required a fix this round");
6089 self.state.event(
6090 "verify",
6091 format!(
6092 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6093 {}); it will run once a round has none left",
6094 short(&head)
6095 ),
6096 );
6097 (Vec::new(), false, true, Some(reason))
6098 } else {
6099 let e2e_commands = self.state.config.verify.e2e.clone();
6100 let cache_dir = self.state.config.cache_dir();
6101 let context = format!("round {round}");
6102 let wait = self.repository_wait();
6103 let (e2e, verify_retried) = with_cache_lease(
6104 &mut self.state,
6105 cache_dir.as_deref(),
6106 "e2e",
6107 "e2e",
6108 &winner.worktree,
6109 &head,
6110 verify_timeout,
6111 &context,
6112 &wait,
6113 |state, budget| {
6114 let shell = shell.clone();
6115 let e2e_commands = e2e_commands.clone();
6116 let worktree = winner.worktree.clone();
6117 let context = context.clone();
6118 async move {
6119 run_e2e_with_retry(
6120 state,
6121 &shell,
6122 &e2e_commands,
6123 &worktree,
6124 budget,
6125 &context,
6126 )
6127 .await
6128 }
6129 },
6130 )
6131 .await?;
6132 (e2e, verify_retried, false, None)
6133 };
6134
6135 let expected = records.len();
6136 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6137 let incomplete = answered < expected;
6138 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6139 let policy = self.state.config.graph.incomplete_review;
6140 let clean = round_is_clean(
6141 blocking,
6142 e2e_ok,
6143 answered,
6144 expected,
6145 round_quota_missing,
6146 policy,
6147 );
6148
6149 let mut round_record = ReviewRound {
6150 round,
6151 head: head.clone(),
6152 verified_head: None,
6153 verified_at: None,
6154 reviews: records,
6155 e2e,
6156 verify_retried,
6157 e2e_deferred,
6158 e2e_defer_reason,
6159 fix: None,
6160 blocking,
6161 answered,
6162 expected,
6163 clean,
6164 progressed: false,
6165 vote_split,
6166 reconsideration,
6167 verdict: None,
6168 };
6169 round_record.verdict = ReviewVote::worst(
6173 round_record
6174 .final_votes()
6175 .into_iter()
6176 .map(|(_, _, vote)| vote),
6177 );
6178 if !matches!(
6189 round_record.e2e_status(),
6190 E2eStatus::Deferred | E2eStatus::NotConfigured
6191 ) {
6192 round_record.verified_head = Some(head.clone());
6193 round_record.verified_at = Some(Timestamp::now());
6194 }
6195 let this_round_verification = round_record.verification_summary(&head);
6196
6197 if incomplete {
6198 let missing: Vec<String> = round_record
6199 .reviews
6200 .iter()
6201 .filter(|r| r.failed.is_some())
6202 .map(|r| format!("review-{}", r.reviewer))
6203 .collect();
6204 self.state.event(
6205 "review",
6206 format!(
6207 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6208 missing.join(", ")
6209 ),
6210 );
6211 }
6212
6213 if clean {
6214 self.state.event(
6215 "review",
6216 if incomplete && policy == IncompleteReviewPolicy::Warn {
6217 format!(
6218 "round {round}: clean (warn policy, incomplete panel) — no \
6219 blocking findings from the seats that answered, verification green"
6220 )
6221 } else if incomplete {
6222 format!(
6223 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6224 quorum) — no blocking findings from the seats that answered, \
6225 verification green",
6226 expected - answered
6227 )
6228 } else {
6229 format!("round {round}: clean — no blocking findings, verification green")
6230 },
6231 );
6232 self.state.reviews.push(round_record);
6233 self.state.status = RunStatus::Gating;
6234 self.state.save()?;
6235 return Ok(());
6236 }
6237
6238 if incomplete && blocking == 0 && e2e_ok {
6246 self.state.reviews.push(round_record);
6247 self.state.save()?;
6248 if round == max_rounds {
6249 self.state.status = RunStatus::Blocked;
6250 self.state.event(
6251 "review",
6252 format!(
6253 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6254 refusing to call it clean",
6255 expected - answered
6256 ),
6257 );
6258 return Ok(());
6259 }
6260 continue;
6261 }
6262
6263 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6275 self.state.reviews.push(round_record);
6276 return self
6277 .stop_reviewing(
6278 "the round's own verification could not run",
6279 &shell,
6280 &winner.worktree,
6281 )
6282 .await;
6283 }
6284
6285 if round == max_rounds {
6286 self.state.reviews.push(round_record);
6287 return self
6288 .stop_reviewing(
6289 &format!(
6290 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6291 ),
6292 &shell,
6293 &winner.worktree,
6294 )
6295 .await;
6296 }
6297
6298 let blocking_findings: Vec<_> = all_findings
6301 .iter()
6302 .filter(|f| f.severity.blocks())
6303 .cloned()
6304 .collect();
6305 let fix_prompt = prompt::fix_in(
6306 &self.state.instruction,
6307 &blocking_findings,
6308 this_round_verification.as_ref(),
6309 round,
6310 max_rounds,
6311 &language,
6312 &self.state.config.graph.github_language,
6313 );
6314 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6315 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6316 let (job, seat, out) = self
6317 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6318 prompt: fix_prompt.clone(),
6319 spec,
6320 seat,
6321 cwd: winner.worktree.clone(),
6322 timeout,
6323 allow_write: true,
6324 sessions,
6325 artifacts: artifacts.clone(),
6326 stem: format!("fix-{round}"),
6327 handover: None,
6328 })
6329 .await;
6330 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6333 return self.stop_for_resource();
6334 }
6335 let agent_id = seat.agent.clone();
6336
6337 let mut fix = FixRecord {
6338 agent: agent_id,
6339 addressed: Vec::new(),
6340 rejected: Vec::new(),
6341 notes: String::new(),
6342 committed: false,
6343 failed: None,
6344 duration_ms: 0,
6345 continuation: None,
6346 };
6347 let mut continuation = ContinuationRecord::not_needed();
6348 let mut final_seat = seat.clone();
6349 match out {
6350 AgentOutcome::Ok(o) => {
6351 fix.duration_ms = o.duration_ms;
6352 let parsed = verdict::extract_json::<FixReport>(&o.text);
6353 let incomplete_reason = match &parsed {
6360 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6361 "the reply parsed, but it reported a command whose own CLI never \
6362 confirmed an exit status"
6363 .to_owned(),
6364 ),
6365 Ok(_) => None,
6366 Err(e) => Some(e.to_string()),
6367 };
6368 match incomplete_reason {
6369 None => {
6370 let report = parsed.expect("checked Ok above");
6371 fix.addressed = report.addressed;
6372 fix.rejected = report.rejected;
6373 fix.notes =
6374 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6375 }
6376 Some(reason) => {
6377 let (resumed_seat, resolved, failure, cont) = self
6378 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6379 .await;
6380 fix.duration_ms += cont.cumulative_wait_ms;
6381 continuation = cont;
6382 final_seat = resumed_seat;
6383 match resolved {
6384 Some(report) => {
6385 fix.addressed = report.addressed;
6386 fix.rejected = report.rejected;
6387 fix.notes = blind::sanitize_prose(
6388 &report.notes,
6389 &self.state.config.blind,
6390 );
6391 }
6392 None => fix.failed = failure,
6393 }
6394 }
6395 }
6396 }
6397 AgentOutcome::Dropped(o) => {
6399 fix.duration_ms = o.duration_ms;
6400 let why = o
6401 .dropped
6402 .as_ref()
6403 .map(|d| d.why.as_str())
6404 .unwrap_or("the CLI ended the stream without delivering its answer");
6405 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6406 }
6407 AgentOutcome::Quota(o) => {
6408 self.state.quota.push(QuotaLoss {
6409 seat: final_seat.key.clone(),
6410 node: "fix".to_owned(),
6411 at: Timestamp::now(),
6412 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6413 });
6414 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6415 }
6416 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6417 }
6418 fix.continuation = Some(continuation);
6419 self.state.seats.insert(final_seat.key.clone(), final_seat);
6420 let coordination = self.coordinate(&winner.worktree).await?;
6421 if let Ok(r) = git::rescue_commit(
6422 &winner.worktree,
6423 &format!("magi: review round {round} fixes (uncommitted work)"),
6424 )
6425 .await
6426 {
6427 self.state.note_withheld("fix", &r.withheld);
6428 }
6429 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6430 fix.committed = after != before;
6431 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6439 drop(coordination);
6440 let progressed = diff_after != patch;
6441 let commit_note = if fix.committed {
6442 "committed"
6443 } else {
6444 "NO new commit"
6445 };
6446 let tree_note = if progressed {
6447 "changed vs base"
6448 } else {
6449 "unchanged vs base"
6450 };
6451 self.state.event(
6452 "fix",
6453 match &fix.failed {
6454 Some(reason) => {
6460 format!(
6461 "round {round}: fixer's adoption report was lost ({reason}); \
6462 {commit_note}, tree {tree_note}"
6463 )
6464 }
6465 None => format!(
6466 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6467 {tree_note}{}",
6468 fix.addressed.len(),
6469 fix.rejected.len(),
6470 if continuation.outcome == ContinuationOutcome::Resumed {
6471 format!(
6472 " (adoption report recovered after {} continuation(s))",
6473 continuation.attempts
6474 )
6475 } else {
6476 String::new()
6477 },
6478 ),
6479 },
6480 );
6481 round_record.fix = Some(fix);
6482 round_record.progressed = progressed;
6483 self.state.reviews.push(round_record);
6484 self.state.save()?;
6485
6486 if matches!(
6499 continuation.outcome,
6500 ContinuationOutcome::Exhausted
6501 | ContinuationOutcome::QuotaLost
6502 | ContinuationOutcome::NoSession
6503 ) {
6504 return self
6505 .stop_reviewing(
6506 "the fixer's adoption report never came back, even after resuming its \
6507 own seat; refusing to start another round against the same worktree \
6508 while that is unresolved",
6509 &shell,
6510 &winner.worktree,
6511 )
6512 .await;
6513 }
6514
6515 let streak = self
6516 .state
6517 .reviews
6518 .iter()
6519 .rev()
6520 .take_while(|r| !r.progressed)
6521 .count();
6522 if streak >= STAGNANT_LIMIT {
6523 return self
6524 .stop_reviewing(
6525 &format!(
6526 "the tree has not moved against base for {streak} round(s) in a row"
6527 ),
6528 &shell,
6529 &winner.worktree,
6530 )
6531 .await;
6532 }
6533 }
6534 Ok(())
6535 }
6536
6537 fn record_contested_handoff(&mut self) {
6571 if self.state.contested_handoff.is_some() {
6572 return;
6573 }
6574 let Some(contested) = self
6575 .state
6576 .reviews
6577 .last()
6578 .and_then(ReviewRound::contested_handoff)
6579 else {
6580 return;
6581 };
6582 self.state.event(
6583 "review",
6584 format!(
6585 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6586 wait for the owner's approval",
6587 contested.findings.len(),
6588 contested.rejecters.len()
6589 ),
6590 );
6591 self.state.contested_handoff = Some(contested);
6592 }
6593
6594 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6595 let round_idx = self.state.reviews.len() - 1;
6596 let needs_catchup_run = matches!(
6604 self.state.reviews[round_idx].e2e_status(),
6605 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6606 );
6607 if needs_catchup_run {
6608 let round = self.state.reviews[round_idx].round;
6609 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6610 let commands = self.state.config.verify.e2e.clone();
6611 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6612 let cache_dir = self.state.config.cache_dir();
6613 let context = format!(
6614 "round {round}: verification unresolved, catching up before the final decision"
6615 );
6616 let wait = self.repository_wait();
6617 let (outcomes, verify_retried) = with_cache_lease(
6618 &mut self.state,
6619 cache_dir.as_deref(),
6620 "e2e",
6621 "e2e",
6622 worktree,
6623 &attempted_head,
6624 timeout,
6625 &context,
6626 &wait,
6627 |state, budget| {
6628 let shell = shell.to_vec();
6629 let commands = commands.clone();
6630 let context = context.clone();
6631 async move {
6632 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6633 .await
6634 }
6635 },
6636 )
6637 .await?;
6638 let last = &mut self.state.reviews[round_idx];
6639 last.e2e = outcomes;
6640 last.verify_retried = verify_retried;
6641 last.verified_head = Some(attempted_head);
6648 last.verified_at = Some(Timestamp::now());
6649 if verify_inconclusive(&last.e2e) {
6650 self.state.save()?;
6657 return Ok(());
6658 }
6659 last.e2e_deferred = false;
6660 }
6661 let last = &self.state.reviews[round_idx];
6662 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6663
6664 match last.e2e_status() {
6665 E2eStatus::Failed => {
6666 let red: Vec<String> = last
6667 .e2e
6668 .iter()
6669 .filter(|o| !o.ok())
6670 .map(|o| {
6671 format!(
6672 "`{}` -> {:?}\n{}",
6673 o.command,
6674 o.code,
6675 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6676 )
6677 })
6678 .collect();
6679 self.state
6680 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6681 self.state.status = RunStatus::Blocked;
6682 }
6683 E2eStatus::ResourceBlocked => {
6688 self.state.event(
6689 "review",
6690 format!(
6691 "{why}; e2e could not run (shared build cache unavailable); not \
6692 deciding yet"
6693 ),
6694 );
6695 }
6696 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6697 self.state.event(
6698 "review",
6699 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6700 );
6701 self.record_contested_handoff();
6702 self.state.status = RunStatus::Gating;
6703 }
6704 }
6705 self.state.save()?;
6706 Ok(())
6707 }
6708
6709 async fn gate(&mut self) -> Result<()> {
6712 if self.state.status == RunStatus::Failed
6724 || self
6725 .state
6726 .base_sync
6727 .as_ref()
6728 .is_some_and(|s| s.conflict.is_some())
6729 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6730 != Some(RunStatus::Gating)
6731 {
6732 return Ok(());
6733 }
6734 if self.state.gate_ran {
6735 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6746 self.state.status = RunStatus::Blocked;
6747 self.state.save()?;
6748 }
6749 return Ok(());
6750 }
6751 let Some(winner) = self.state.winner().cloned() else {
6752 return Ok(());
6753 };
6754 self.state.status = RunStatus::Gating;
6755 let mut outcomes = self.run_gate(&winner).await?;
6756 loop {
6757 if verify_inconclusive(&outcomes) {
6768 self.state.save()?;
6769 return Ok(());
6770 }
6771 if outcomes.iter().all(CommandOutcome::ok) {
6772 break;
6773 }
6774 match self.gate_fix_round(&winner, &outcomes).await? {
6775 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6776 GateFix::Stop => break,
6777 GateFix::Defer => {
6778 self.state.save()?;
6779 return Ok(());
6780 }
6781 }
6782 }
6783 let passed = outcomes.iter().all(CommandOutcome::ok);
6784 self.state.gate = outcomes;
6785 self.state.gate_ran = true;
6786 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6787 if !passed {
6788 self.state.status = RunStatus::Blocked;
6789 let spent = self.state.gate_fixes.len();
6790 self.state.event(
6791 "gate",
6792 if spent == 0 {
6793 "gate failed; not merging".to_owned()
6794 } else {
6795 format!("gate failed after {spent} gate-fix round(s); not merging")
6796 },
6797 );
6798 }
6799 self.state.save()?;
6800 Ok(())
6801 }
6802
6803 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6813 let commands = state.config.verify.pre_gate.clone();
6814 if commands.is_empty() {
6815 return;
6816 }
6817 let shell = state.config.shell();
6818 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6819 let (outcomes, _) = run_commands(
6820 state,
6821 "pre_gate",
6822 "pre_gate",
6823 0,
6824 &shell,
6825 &commands,
6826 &winner.worktree,
6827 timeout,
6828 )
6829 .await;
6830 for o in &outcomes {
6831 if !o.ok() {
6832 tracing::warn!(
6833 "pre_gate `{}` failed ({:?}); the gate decides",
6834 o.command,
6835 o.code
6836 );
6837 }
6838 state.event(
6839 "pre_gate",
6840 format!(
6841 "`{}` -> {}",
6842 o.command,
6843 if o.ok() {
6844 "pass".to_owned()
6845 } else {
6846 format!(
6847 "FAIL ({:?})\n{}",
6848 o.code,
6849 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6850 )
6851 }
6852 ),
6853 );
6854 }
6855 state.pre_gate = outcomes;
6856 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6857 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6858 Ok(head) => {
6859 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6860 state.pre_gate_commit = Some(head);
6861 }
6862 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6863 },
6864 Ok(false) => {}
6865 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6866 }
6867 if let Err(e) = state.save() {
6868 tracing::warn!("could not persist the pre_gate record: {e:#}");
6869 }
6870 }
6871
6872 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6875 let shell = self.state.config.shell();
6876 let gate_commands = self.state.config.verify.gate.clone();
6877 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
6878 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
6879 Vec::new()
6880 } else {
6881 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6882 let cache_dir = self.state.config.cache_dir();
6883 let wait = self.repository_wait();
6884 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6885 let (outcomes, _) = with_cache_lease(
6886 &mut self.state,
6887 cache_dir.as_deref(),
6888 "gate",
6889 "gate",
6890 &winner.worktree,
6891 &head,
6892 timeout,
6893 "final gate",
6894 &wait,
6895 |state, budget| {
6896 let shell = shell.clone();
6897 let gate_commands = gate_commands.clone();
6898 let winner = winner.clone();
6899 async move {
6900 Self::run_pre_gate(state, &winner).await;
6901 let (outcomes, timed_out_pids) = run_commands(
6902 state,
6903 "gate",
6904 "gate",
6905 0,
6906 &shell,
6907 &gate_commands,
6908 &winner.worktree,
6909 budget,
6910 )
6911 .await;
6912 (outcomes, false, timed_out_pids)
6913 }
6914 },
6915 )
6916 .await?;
6917 outcomes
6918 };
6919 if outcomes.is_empty() {
6920 self.state.event(
6925 "gate",
6926 "no gate commands configured; nothing to check, passing",
6927 );
6928 }
6929 for o in &outcomes {
6930 self.state.event(
6931 "gate",
6932 format!(
6933 "`{}` -> {}",
6934 o.command,
6935 if o.ok() {
6936 "pass".to_owned()
6937 } else {
6938 format!(
6939 "FAIL ({:?})\n{}",
6940 o.code,
6941 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6942 )
6943 }
6944 ),
6945 );
6946 }
6947 Ok(outcomes)
6948 }
6949
6950 async fn gate_fix_round(
6963 &mut self,
6964 winner: &Candidate,
6965 outcomes: &[CommandOutcome],
6966 ) -> Result<GateFix> {
6967 let cap = self.state.config.graph.gate_fix_rounds;
6968 let spent = self.state.gate_fixes.len();
6969 if spent >= cap {
6970 if cap > 0 {
6971 self.state.event(
6972 "gate",
6973 format!("{spent} gate-fix round(s) spent and the gate still fails"),
6974 );
6975 }
6976 return Ok(GateFix::Stop);
6977 }
6978 if !gate_fixable(outcomes) {
6979 self.state.event(
6980 "gate",
6981 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
6982 command or similar); not spending a fix round on it",
6983 );
6984 return Ok(GateFix::Stop);
6985 }
6986 let min_free = self.state.config.disk.min_free_bytes;
6987 if min_free > 0 {
6988 match crate::disk::free_bytes(&winner.worktree) {
6989 Ok(free) if crate::disk::enough_space(free, min_free) => {}
6990 Ok(free) => {
6991 self.state.event(
6992 "gate",
6993 format!(
6994 "only {free} bytes free ({min_free} required by `[disk] \
6995 min_free_bytes`); not spending a fix round on a failure the disk \
6996 may explain"
6997 ),
6998 );
6999 return Ok(GateFix::Stop);
7000 }
7001 Err(e) => {
7002 self.state.event(
7003 "gate",
7004 format!("free disk space could not be measured ({e:#}); no fix round"),
7005 );
7006 return Ok(GateFix::Stop);
7007 }
7008 }
7009 }
7010
7011 let attempt = spent + 1;
7012 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7013 let base = self.landing_base();
7014 let fix_prompt = prompt::gate_fix_in(
7015 &self.state.instruction,
7016 &failed,
7017 attempt,
7018 cap,
7019 &self.state.config.graph.language,
7020 &self.state.config.graph.github_language,
7021 );
7022 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7023 let sessions = self.state.config.graph.sessions;
7024 let artifacts = self.merge_repair_artifacts();
7025 self.state.event(
7026 "gate",
7027 format!("gate failed; gate-fix round {attempt} of {cap}"),
7028 );
7029 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7030 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7031 let (_, seat, out) = self
7032 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7033 prompt: fix_prompt.clone(),
7034 spec,
7035 seat,
7036 cwd: winner.worktree.clone(),
7037 timeout,
7038 allow_write: true,
7039 sessions,
7040 artifacts: artifacts.clone(),
7041 stem: format!("gate-fix-{attempt}"),
7042 handover: None,
7043 })
7044 .await;
7045 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7048 return Ok(GateFix::Defer);
7049 }
7050 let mut record = GateFixRecord {
7051 agent: seat.agent.clone(),
7052 failed,
7053 notes: String::new(),
7054 committed: false,
7055 error: None,
7056 };
7057 match out {
7058 AgentOutcome::Ok(o) => {
7059 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7062 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7063 }
7064 }
7065 AgentOutcome::Dropped(_) => {
7066 record.error = Some("the CLI dropped the stream".to_owned());
7067 }
7068 AgentOutcome::Quota(o) => {
7069 self.state.quota.push(QuotaLoss {
7070 seat: seat.key.clone(),
7071 node: "gate-fix".to_owned(),
7072 at: Timestamp::now(),
7073 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7074 });
7075 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7076 }
7077 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7078 }
7079 self.state.seats.insert(seat.key.clone(), seat);
7080 let coordination = self.coordinate(&winner.worktree).await?;
7081 if let Ok(r) = git::rescue_commit(
7082 &winner.worktree,
7083 &format!("magi: gate fix {attempt} (uncommitted work)"),
7084 )
7085 .await
7086 {
7087 self.state.note_withheld("gate-fix", &r.withheld);
7088 }
7089 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7090 record.committed = after != before;
7091 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7092 let note = record.error.clone();
7093 self.state.gate_fixes.push(record);
7094 self.state.save()?;
7095 if !changed {
7096 self.state.event(
7097 "gate",
7098 match note {
7099 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7100 None => format!("gate-fix round {attempt}: the tree did not change"),
7101 },
7102 );
7103 return Ok(GateFix::Stop);
7104 }
7105 self.state.event(
7106 "gate",
7107 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7108 );
7109
7110 drop(coordination);
7111 let commands = self.state.config.verify.e2e.clone();
7112 if !commands.is_empty() {
7113 let shell = self.state.config.shell();
7114 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7115 let cache_dir = self.state.config.cache_dir();
7116 let context = format!("gate-fix round {attempt}");
7117 let wait = self.repository_wait();
7118 let (e2e, _) = with_cache_lease(
7119 &mut self.state,
7120 cache_dir.as_deref(),
7121 "e2e",
7122 "e2e",
7123 &winner.worktree,
7124 &after,
7125 timeout,
7126 &context,
7127 &wait,
7128 |state, budget| {
7129 let shell = shell.clone();
7130 let commands = commands.clone();
7131 let context = context.clone();
7132 let worktree = winner.worktree.clone();
7133 async move {
7134 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7135 .await
7136 }
7137 },
7138 )
7139 .await?;
7140 if verify_inconclusive(&e2e) {
7141 return Ok(GateFix::Defer);
7142 }
7143 if e2e.iter().any(|o| !o.ok()) {
7144 self.state.event(
7145 "gate",
7146 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7147 );
7148 return Ok(GateFix::Stop);
7149 }
7150 }
7151 Ok(GateFix::Retry)
7152 }
7153
7154 async fn guarded_pr_message(
7161 &mut self,
7162 winner: &Candidate,
7163 facts: Option<&BranchFacts>,
7164 posting: bool,
7165 ) -> Result<Option<PrMessage>> {
7166 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7167 let head = match final_head.as_deref() {
7168 Some(h) => crate::pr_outcome::FinalHead::At(h),
7169 None => crate::pr_outcome::FinalHead::Unknown,
7170 };
7171 let summarized = if posting {
7173 Some(self.summarize_change(winner, final_head.as_deref()).await)
7174 } else {
7175 None
7176 };
7177 if self.state.resource_block.is_some() {
7182 self.state.event(
7183 "pr-summary",
7184 "a shared resource was busy; nothing was published, resume to summarize again",
7185 );
7186 return Ok(None);
7187 }
7188 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7189 anyhow::bail!(
7190 "the final commit moved during public description generation; retry before publishing"
7191 );
7192 }
7193 let note = match (&summarized, &self.state.change_summary) {
7194 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7195 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7196 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7197 (None, _) => ChangeNote::NotAsked,
7198 };
7199 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7200 if !posting {
7201 let identity = crate::scrub::Identity::current();
7202 return Ok(Some(PrMessage {
7203 title: crate::scrub::scrub(&pr.title, &identity),
7204 body: crate::scrub::scrub(&pr.body, &identity),
7205 }));
7206 }
7207 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7210 if let Some(known) = self
7211 .state
7212 .github_text
7213 .clone()
7214 .filter(|g| g.fingerprint == raw_fp)
7215 {
7216 return self.settle_withheld(winner, &pr, known).await;
7217 }
7218 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7219 let mut violations =
7220 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7221 let identity = crate::scrub::Identity::current();
7222 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7223 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7224 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7225 {
7226 violations.push(crate::github_text::Violation::SensitiveData);
7227 }
7228 let has_language = violations
7232 .iter()
7233 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7234 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7235 self.state.event(
7236 "github-text",
7237 if has_language {
7238 format!("description rejected: {violations:?}; requesting one rewrite")
7239 } else {
7240 format!("description rejected: {violations:?}; asking the owner")
7241 },
7242 );
7243 let mut rewritten = false;
7244 let mut rejected_rewrite = None;
7245 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7246 {
7247 let output = text;
7248 if let Some(title) = summary_title(&output) {
7249 let raw_body = summary_without_title(&output);
7252 let clean_title = crate::scrub::scrub(&title, &identity);
7253 let mut body = crate::scrub::scrub(&raw_body, &identity);
7254 let redacted = clean_title != title || body != raw_body;
7255 let title = clean_title;
7256 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7257 if !complete {
7258 self.state.event(
7259 "github-text",
7260 "description rewrite omitted required findings",
7261 );
7262 }
7263 if !body.trim().is_empty() {
7264 body.push_str("\n\n");
7265 body.push_str(&public_pr_evidence(&self.state, head));
7266 }
7267 let marker = format!("magi:run/{}", self.state.id);
7268 if !body.trim().is_empty() && !body.contains(&marker) {
7269 body.push_str(&format!("\n\n{marker}"));
7270 }
7271 let rewrite = PrMessage {
7272 title: title.clone(),
7273 body: body.clone(),
7274 };
7275 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7276 let mut failed = crate::github_text::check_for(
7277 &self.state.config.graph,
7278 &title,
7279 &body,
7280 redecision,
7281 );
7282 if (!crate::github_text::shareable(&title)
7283 || !crate::github_text::shareable(&body))
7284 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7285 {
7286 failed.push(crate::github_text::Violation::SensitiveData);
7287 }
7288 if complete
7289 && !title.trim().is_empty()
7290 && !body.trim().is_empty()
7291 && !failed.is_empty()
7292 {
7293 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7294 title: title.clone(),
7295 body: body.clone(),
7296 failed: failed
7297 .iter()
7298 .map(|v| crate::github_text::category(*v).to_owned())
7299 .collect(),
7300 });
7301 }
7302 if complete
7303 && !title.trim().is_empty()
7304 && !raw_body.trim().is_empty()
7305 && failed.is_empty()
7306 {
7307 decision = redecision;
7308 pr = PrMessage { title, body };
7309 rewritten = true;
7310 self.state
7311 .event("github-text", "description rewrite passed");
7312 if redacted {
7313 self.state.event(
7314 "github-text",
7315 "sensitive data removed from description rewrite",
7316 );
7317 }
7318 }
7319 }
7320 }
7321 if !rewritten {
7322 self.state.event(
7323 "github-text",
7324 "description rewrite unavailable or rejected; asking the owner",
7325 );
7326 if let Some(w) =
7327 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7328 {
7329 self.retire_github_text_question("the withheld text changed");
7332 let record = crate::run::GithubTextAsk {
7333 rewrite: rejected_rewrite,
7334 fingerprint: raw_fp,
7335 title: w.title,
7336 body: w.body,
7337 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7338 question: None,
7339 asks: 0,
7340 resolved: false,
7341 chosen_title: None,
7342 choice: None,
7343 };
7344 return self.settle_withheld(winner, &pr, record).await;
7345 }
7346 }
7347 }
7348 let (title, body) =
7349 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7350 Ok(Some(PrMessage { title, body }))
7351 }
7352
7353 async fn summarize_change(
7364 &mut self,
7365 winner: &Candidate,
7366 head: Option<&str>,
7367 ) -> std::result::Result<(), String> {
7368 let Some(head) = head else {
7369 return Err("the final commit could not be read".to_owned());
7370 };
7371 let remote = self.state.config.merge.remote.clone();
7372 let tracking = format!("{remote}/{}", self.state.base_branch);
7373 if !git::rev_exists(&winner.worktree, &tracking).await {
7374 return Err(format!("the remote base `{tracking}` could not be read"));
7375 }
7376 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7377 Ok(b) if !b.is_empty() => b,
7378 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7379 };
7380 let evidence = crate::pr_outcome::summary_evidence(
7381 &self.state,
7382 crate::pr_outcome::FinalHead::At(head),
7383 );
7384 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7385 if self
7386 .state
7387 .change_summary
7388 .as_ref()
7389 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7390 {
7391 return Ok(());
7392 }
7393 if self.state.change_summary.take().is_some() {
7394 self.state.event(
7395 "pr-summary",
7396 "the final diff or evidence changed; the earlier change summary was discarded",
7397 );
7398 }
7399 if let Err(e) = write_artifact(
7401 &self.state,
7402 &format!("public-description-{fingerprint}.md"),
7403 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7404 ) {
7405 tracing::warn!("could not save public description evidence: {e:#}");
7406 }
7407 let chain = match agent::pick_chain(
7408 &self.state.config.agents,
7409 self.state.config.roles.synthesizer.as_ref(),
7410 &agent::installed,
7411 "synthesizer",
7412 ) {
7413 Ok(chain) => chain,
7414 Err(e) => {
7415 tracing::warn!("no agent for the change summary: {e:#}");
7416 self.state.event("pr-summary", "no summary agent available");
7417 return Err("no agent was available to summarize it".to_owned());
7418 }
7419 };
7420 let subjects = git::subjects(&winner.worktree, &base, head)
7421 .await
7422 .unwrap_or_default();
7423 let stat = git::diff_stat(&winner.worktree, &base, head)
7424 .await
7425 .unwrap_or_default();
7426 let stat: String = stat.chars().take(6_000).collect();
7427 let mut prompt = crate::prompt::change_summary_in(
7428 &base,
7429 head,
7430 &subjects,
7431 stat.trim(),
7432 &self.state.config.graph.github_language,
7433 );
7434 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7435 let sessions = self.state.config.graph.sessions;
7436 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7437 let prompts = self.state.config.prompts.clone();
7438 let cache = self.state.config.cache_dir();
7439 let run = self.state.id.clone();
7440 for (i, spec) in chain.into_iter().enumerate() {
7441 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7442 let job = SeatJob {
7443 spec: spec.clone(),
7444 seat,
7445 cwd: winner.worktree.clone(),
7446 prompt: prompt.clone(),
7447 timeout,
7448 allow_write: false,
7449 sessions,
7450 artifacts: agent::artifacts_dir(&self.state.dir()),
7451 stem: if i == 0 {
7452 "change-summary".to_owned()
7453 } else {
7454 format!("change-summary-{}", spec.id)
7455 },
7456 handover: None,
7457 };
7458 let ctx = WaveCtx {
7459 run: &run,
7460 node: "pr-summary",
7461 prompts: &prompts,
7462 cache: cache.as_deref(),
7463 round: None,
7464 carry_seats: false,
7465 pauses: Some((&self.pause, &self.interrupt)),
7466 };
7467 let (seat, outcome) =
7468 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7469 self.state.seats.insert(seat.key.clone(), seat);
7470 let text = match outcome {
7471 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7472 AgentOutcome::ResourceBlocked(why) => {
7473 if self.state.resource_block.is_none() {
7476 self.state.resource_block = Some(crate::run::ResourceBlock {
7477 node: "pr-summary".to_owned(),
7478 kind: "shared resource".to_owned(),
7479 detail: why,
7480 at: Timestamp::now(),
7481 });
7482 }
7483 return Err("a shared resource was busy".to_owned());
7484 }
7485 _ => continue,
7486 };
7487 let blind = &self.state.config.blind;
7488 let Some(title) = summary_title(&text) else {
7489 continue;
7490 };
7491 let body: String = summary_without_title(&text)
7492 .lines()
7493 .filter(|l| !l.contains("magi:run/"))
7494 .collect::<Vec<_>>()
7495 .join("\n");
7496 let body = blind::sanitize_prose(body.trim(), blind);
7497 let title = blind::sanitize_prose(&title, blind);
7498 if body.trim().is_empty() || title.trim().is_empty() {
7499 continue;
7500 }
7501 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7502 continue;
7503 }
7504 if git::rev_parse(&winner.worktree, "HEAD")
7506 .await
7507 .ok()
7508 .as_deref()
7509 != Some(head)
7510 {
7511 return Err("the branch moved while the summary was written".to_owned());
7512 }
7513 if git::merge_base(&winner.worktree, &tracking, head)
7514 .await
7515 .ok()
7516 .as_deref()
7517 != Some(base.as_str())
7518 || crate::pr_outcome::summary_evidence(
7519 &self.state,
7520 crate::pr_outcome::FinalHead::At(head),
7521 ) != evidence
7522 {
7523 return Err("the summary evidence changed while it was written".to_owned());
7524 }
7525 self.state.change_summary = Some(crate::run::ChangeSummary {
7526 head: head.to_owned(),
7527 base,
7528 evidence: fingerprint,
7529 title,
7530 body,
7531 });
7532 self.state.event(
7533 "pr-summary",
7534 format!("change summary written by {} for the final diff", spec.id),
7535 );
7536 if let Err(e) = self.state.save() {
7537 tracing::warn!("could not save the change summary: {e:#}");
7538 }
7539 return Ok(());
7540 }
7541 self.state
7542 .event("pr-summary", "no agent produced a change summary");
7543 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7544 }
7545
7546 async fn rewrite_pr_text(
7554 &mut self,
7555 winner: &Candidate,
7556 pr: &PrMessage,
7557 violations: &[crate::github_text::Violation],
7558 ) -> Option<String> {
7559 let prompt = format!(
7560 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7561 pr.title, pr.body
7562 );
7563 let sessions = self.state.config.graph.sessions;
7564 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7565 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7567 let key = format!("impl-{}", winner.label);
7568 if let Some(spec) = self
7569 .state
7570 .config
7571 .agents
7572 .iter()
7573 .find(|a| a.id == winner.agent)
7574 .cloned()
7575 && let Some(existing) = self.state.seats.get(&key)
7576 && existing.agent == spec.id
7577 && agent::has_session(spec.kind, existing, sessions)
7578 {
7579 attempts.push((spec, existing.clone(), true));
7580 } else {
7581 let chain = agent::pick_chain(
7582 &self.state.config.agents,
7583 self.state.config.roles.language_judge.as_ref(),
7584 &agent::installed,
7585 "language_judge",
7586 );
7587 match chain {
7588 Ok(chain) => {
7589 for spec in chain {
7590 let seat = crate::fixer::seat_for(
7591 &mut self.state,
7592 "github-text-rewrite",
7593 &spec.id,
7594 );
7595 attempts.push((spec, seat, false));
7596 }
7597 }
7598 Err(e) => {
7599 tracing::warn!("no agent for the description rewrite: {e:#}");
7600 self.state
7601 .event("github-text", "no rewrite agent available");
7602 return None;
7603 }
7604 }
7605 }
7606 let prompts = self.state.config.prompts.clone();
7607 let cache = self.state.config.cache_dir();
7608 let run = self.state.id.clone();
7609 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7610 let job = SeatJob {
7611 spec: spec.clone(),
7612 seat,
7613 cwd: winner.worktree.clone(),
7614 prompt: prompt.clone(),
7615 timeout: retry_budget(timeout_full, resumed),
7616 allow_write: false,
7617 sessions,
7618 artifacts: agent::artifacts_dir(&self.state.dir()),
7619 stem: if i == 0 {
7620 "github-text-rewrite".to_owned()
7621 } else {
7622 format!("github-text-rewrite-{}", spec.id)
7623 },
7624 handover: None,
7625 };
7626 let ctx = WaveCtx {
7627 run: &run,
7628 node: "github-text",
7629 prompts: &prompts,
7630 cache: cache.as_deref(),
7631 round: None,
7632 carry_seats: false,
7633 pauses: Some((&self.pause, &self.interrupt)),
7634 };
7635 let (seat, outcome) =
7636 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7637 self.state.seats.insert(seat.key.clone(), seat);
7638 match outcome {
7639 AgentOutcome::Ok(o)
7640 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7641 {
7642 return Some(o.text);
7643 }
7644 AgentOutcome::ResourceBlocked(_) => return None,
7645 _ => {}
7646 }
7647 }
7648 None
7649 }
7650
7651 fn retire_github_text_question(&mut self, why: &str) {
7653 let Some(id) = self
7654 .state
7655 .github_text
7656 .as_ref()
7657 .and_then(|g| g.question.clone())
7658 else {
7659 return;
7660 };
7661 let _ = ask::Questions::open().update(&id, |q| {
7662 q.abandon(why);
7663 Ok(())
7664 });
7665 }
7666
7667 async fn settle_withheld(
7671 &mut self,
7672 winner: &Candidate,
7673 pr: &PrMessage,
7674 mut g: crate::run::GithubTextAsk,
7675 ) -> Result<Option<PrMessage>> {
7676 use crate::github_text as gt;
7677 let store = ask::Questions::open();
7678 let timeout = self.state.config.graph.answer_timeout;
7679 if !g.resolved {
7680 let mut standing = g
7681 .question
7682 .as_ref()
7683 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7684 if let Some(q) = standing.as_ref()
7685 && q.status.open()
7686 && gt::expired(q, timeout)
7687 {
7688 let why = format!("no answer within {}s of asking", timeout.max(1));
7689 standing = store
7690 .update(&q.id, |q| {
7691 q.abandon(&why);
7692 Ok(())
7693 })
7694 .ok()
7695 .map(|(q, ())| q);
7696 }
7697 let mut ask_again: Option<Option<Vec<String>>> = None;
7698 match standing {
7699 None if g.asks <= 1 => {
7702 g.asks = 0;
7703 ask_again = Some(None);
7704 }
7705 None => {
7706 g.resolved = true;
7707 g.chosen_title = None;
7708 g.choice = Some(GithubTextChoice::Fallback);
7709 }
7710 Some(q) if q.status.open() => {
7711 self.park_on_github_text(&g)?;
7712 return Ok(None);
7713 }
7714 Some(q) => match gt::read_reply(&q) {
7715 gt::Reply::Fallback => {
7716 g.resolved = true;
7717 g.chosen_title = None;
7718 g.choice = Some(GithubTextChoice::Fallback);
7719 }
7720 gt::Reply::Original => {
7721 g.resolved = true;
7722 g.chosen_title = None;
7723 g.choice = Some(GithubTextChoice::Original);
7724 }
7725 gt::Reply::Rewrite => {
7726 g.resolved = true;
7727 g.chosen_title = None;
7728 g.choice = Some(if g.rewrite.is_some() {
7729 GithubTextChoice::Rewrite
7730 } else {
7731 GithubTextChoice::Fallback
7732 });
7733 }
7734 gt::Reply::Title(text) => {
7735 let first = text
7736 .lines()
7737 .map(str::trim)
7738 .find(|l| !l.is_empty())
7739 .unwrap_or("");
7740 let decision = if first.is_empty() {
7741 None
7742 } else {
7743 let candidate = PrMessage {
7744 title: first.to_owned(),
7745 body: String::new(),
7746 };
7747 self.judge_pr_language(&winner.worktree, &candidate).await
7748 };
7749 match gt::vet_title(
7750 &text,
7751 decision,
7752 self.state.config.graph.github_english(),
7753 ) {
7754 Ok(title) => {
7755 g.resolved = true;
7756 g.chosen_title = Some(title);
7757 g.choice = Some(GithubTextChoice::MyTitle);
7758 }
7759 Err(cats) if g.asks < 2 => {
7760 ask_again = Some(Some(
7761 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7762 ))
7763 }
7764 Err(_) => {
7765 g.resolved = true;
7766 g.chosen_title = None;
7767 g.choice = Some(GithubTextChoice::Fallback);
7768 }
7769 }
7770 }
7771 },
7772 }
7773 if let Some(rejected) = ask_again {
7774 let w = gt::Withheld {
7777 title: g.title,
7778 body: g.body,
7779 categories: g.categories.clone(),
7780 };
7781 let artifact = match crate::run::write_artifact(
7784 &self.state,
7785 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7786 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7787 ) {
7788 Ok(p) => Some(p),
7789 Err(e) => {
7790 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7791 None
7792 }
7793 };
7794 let (detail, comparison) = gt::question_content(
7795 &w,
7796 &pr.title,
7797 &pr.body,
7798 &self.state.id,
7799 rejected.as_deref(),
7800 artifact.as_deref(),
7801 g.rewrite.as_ref(),
7802 );
7803 let mut q = ask::Question::new(
7804 self.state.id.clone(),
7805 gt::ASK_NODE.to_owned(),
7806 gt::ASK_SEAT.to_owned(),
7807 gt::question_summary(&self.state.config.graph.language, &w),
7808 detail,
7809 gt::question_choices(&w, g.rewrite.as_ref()),
7810 );
7811 q.posting_comparison = Some(comparison);
7812 q.answer_timeout = timeout;
7813 g.question = Some(q.id.clone());
7816 g.asks += 1;
7817 self.state.github_text = Some(g.clone());
7818 self.state.event(
7819 "github-text",
7820 format!("asking the owner about the withheld text ({})", q.short()),
7821 );
7822 self.state.save()?;
7823 store
7824 .put(&mut q)
7825 .context("file the posting-gate question")?;
7826 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7827 tracing::warn!(
7828 "could not notify about posting-gate question {}: {e:#}",
7829 q.short()
7830 );
7831 }
7832 self.park_on_github_text(&g)?;
7833 return Ok(None);
7834 }
7835 }
7836 let resolved = g.chosen_title.clone();
7837 let original = g.choice == Some(GithubTextChoice::Original);
7838 self.state.event(
7840 "github-text",
7841 if original {
7842 format!(
7843 "posting the original text for: {}",
7844 match (g.title, g.body) {
7845 (true, true) => "title, description",
7846 (true, false) => "title",
7847 _ => "description",
7848 }
7849 )
7850 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7851 "posting the saved rewrite title and description".to_owned()
7852 } else if resolved.is_some() {
7853 "posting the owner's replacement title".to_owned()
7854 } else {
7855 "posting the neutral text for the withheld fields".to_owned()
7856 },
7857 );
7858 self.state.github_text = Some(g.clone());
7859 self.state.save()?;
7860 if g.choice == Some(GithubTextChoice::Rewrite)
7861 && let Some(r) = g.rewrite.as_ref()
7862 {
7863 return Ok(Some(PrMessage {
7864 title: r.title.clone(),
7865 body: r.body.clone(),
7866 }));
7867 }
7868 let w = gt::Withheld {
7869 title: g.title,
7870 body: g.body,
7871 categories: g.categories.clone(),
7872 };
7873 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7874 if let Some(t) = resolved {
7875 title = t;
7876 }
7877 let english = gt::LanguageDecision {
7879 title_english: true,
7880 body_english: true,
7881 body_complete: true,
7882 };
7883 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7884 if original {
7885 if g.title {
7888 title = pr.title.clone();
7889 }
7890 if g.body {
7891 body = pr.body.clone();
7892 }
7893 }
7894 Ok(Some(PrMessage { title, body }))
7895 }
7896
7897 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7898 self.state.github_text = Some(g.clone());
7899 self.state.end_pass_parked(ParkCause::Question);
7900 self.state.parked = true;
7901 self.state.event(
7902 "github-text",
7903 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7904 );
7905 self.state.save()
7906 }
7907
7908 async fn judge_pr_language(
7911 &mut self,
7912 cwd: &Path,
7913 pr: &PrMessage,
7914 ) -> Option<crate::github_text::LanguageDecision> {
7915 self.state.config.roles.language_judge.as_ref()?;
7916 if !self.state.config.graph.language_gate() {
7917 return None;
7918 }
7919 let decision =
7920 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
7921 self.state.event(
7922 "github-text",
7923 if decision.is_some() {
7924 "language decided by the language judge"
7925 } else {
7926 "language judge unavailable; using built-in heuristics"
7927 },
7928 );
7929 decision
7930 }
7931
7932 async fn merge(&mut self) -> Result<()> {
7933 if self
7948 .state
7949 .base_sync
7950 .as_ref()
7951 .is_some_and(|s| s.conflict.is_some())
7952 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7953 != Some(RunStatus::Gating)
7954 || !self.state.gate_status().ok()
7963 {
7964 return Ok(());
7965 }
7966 if self.state.merge.is_some() && !self.state.pr_publication_pending {
7975 return Ok(());
7976 }
7977 let Some(winner) = self.state.winner().cloned() else {
7978 return Ok(());
7979 };
7980 let repo = self.state.repo.clone();
7981 let base = self.state.base_branch.clone();
7982 let mode = self.state.config.merge.mode;
7983 let style = self.state.config.merge.style;
7984 let facts = if is_review_run(&self.state) {
7985 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
7986 let start = review_base(
7987 &repo,
7988 &self.state.config.merge.remote,
7989 &base,
7990 &self.state.base_commit,
7991 &winner.branch,
7992 )
7993 .await;
7994 branch_facts(&repo, &start, &winner.branch).await
7995 } else {
7996 None
7997 };
7998 let leaked = if is_review_run(&self.state) {
8001 leaked_subjects(&self.state, &winner.branch).await
8002 } else {
8003 Some(Vec::new())
8004 };
8005 let Some(pr) = self
8009 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8010 .await?
8011 else {
8012 return Ok(());
8013 };
8014 let message = pr.commit_message();
8015
8016 let outcome = match mode {
8017 MergeMode::None => MergeOutcome {
8018 mode,
8019 ok: true,
8020 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8021 empty: false,
8022 },
8023 MergeMode::Pr | MergeMode::Local
8024 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8025 {
8026 MergeOutcome {
8027 mode,
8028 ok: false,
8029 detail: empty_candidate_detail(&self.state, &base),
8030 empty: true,
8031 }
8032 }
8033 MergeMode::Local => {
8034 let on = git::current_branch(&repo).await?;
8035 if on.as_deref() != Some(base.as_str()) {
8036 MergeOutcome {
8037 mode,
8038 ok: false,
8039 detail: format!(
8040 "{} has {} checked out, not the base branch {base}",
8041 repo.display(),
8042 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8043 ),
8044 empty: false,
8045 }
8046 } else if !git::is_clean(&repo).await? {
8047 MergeOutcome {
8048 mode,
8049 ok: false,
8050 detail: format!("{} is dirty; refusing to merge", repo.display()),
8051 empty: false,
8052 }
8053 } else {
8054 let out = match style {
8055 MergeStyle::Merge => {
8056 git::merge_no_ff(&repo, &winner.branch, &message).await?
8057 }
8058 MergeStyle::Squash => {
8059 git::merge_squash(&repo, &winner.branch, &message).await?
8060 }
8061 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8062 };
8063 MergeOutcome {
8064 mode,
8065 ok: out.ok(),
8066 detail: if out.ok() { out.stdout } else { out.stderr },
8067 empty: false,
8068 }
8069 }
8070 }
8071 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8072 self.state.pr_publication_pending = true;
8073 self.state.save()?;
8074 let out =
8075 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8076 self.state.pr_publication_pending = out.is_err();
8077 MergeOutcome {
8078 mode,
8079 ok: out.is_ok(),
8080 detail: match out {
8081 Ok(url) => url,
8082 Err(e) => format!("{e:#}"),
8083 },
8084 empty: false,
8085 }
8086 }
8087 MergeMode::Pr => {
8088 let remote = self.state.config.merge.remote.clone();
8089 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8090 if !pushed.ok() {
8091 MergeOutcome {
8092 mode,
8093 ok: false,
8094 detail: pushed.stderr,
8095 empty: false,
8096 }
8097 } else {
8098 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8104 let out = match pr_merge_plan(found) {
8105 PrPlan::Create => {
8106 gh_pr_create(
8107 &winner.worktree,
8108 &base,
8109 &winner.branch,
8110 &pr.title,
8111 &pr.body,
8112 )
8113 .await
8114 }
8115 PrPlan::Adopt { url, title } => {
8116 self.state
8117 .event("merge", format!("Pr: adopted open pull request {url}"));
8118 if self.state.github_text.as_ref().is_some_and(|g| {
8119 g.resolved
8120 && g.choice == Some(GithubTextChoice::Rewrite)
8121 && g.rewrite.is_some()
8122 }) {
8123 land::set_pr_text_verbatim(
8126 &winner.worktree,
8127 &url,
8128 &pr.title,
8129 &pr.body,
8130 )
8131 .await
8132 .map(|()| url)
8133 } else {
8134 if title != pr.title
8135 && (!is_review_run(&self.state)
8136 || leaked
8137 .as_deref()
8138 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8139 && let Err(e) =
8140 if self.state.github_text.as_ref().is_some_and(|g| {
8141 g.resolved
8142 && g.title
8143 && g.choice == Some(GithubTextChoice::Original)
8144 }) {
8145 land::set_pr_title_verbatim(
8147 &winner.worktree,
8148 &url,
8149 &pr.title,
8150 )
8151 .await
8152 } else {
8153 land::set_pr_title(
8154 &mut self.state,
8155 &winner.worktree,
8156 &url,
8157 &pr.title,
8158 )
8159 .await
8160 }
8161 {
8162 tracing::warn!("could not refresh title of {url}: {e:#}");
8163 self.state
8164 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8165 }
8166 Ok(url)
8167 }
8168 }
8169 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8170 };
8171 match out {
8172 Ok(url) => MergeOutcome {
8173 mode,
8174 ok: true,
8175 detail: url,
8176 empty: false,
8177 },
8178 Err(e) => MergeOutcome {
8179 mode,
8180 ok: false,
8181 detail: e.to_string(),
8182 empty: false,
8183 },
8184 }
8185 }
8186 }
8187 };
8188
8189 self.state.status = match (mode, outcome.ok) {
8190 (MergeMode::None, _) => RunStatus::Ready,
8191 (_, true) => RunStatus::Merged,
8192 (_, false) => RunStatus::Blocked,
8193 };
8194 self.state.event(
8195 "merge",
8196 format!(
8197 "{:?}: {}",
8198 mode,
8199 outcome.detail.lines().next().unwrap_or("")
8200 ),
8201 );
8202 self.state.merge = Some(outcome);
8203 self.state.save()?;
8204
8205 if self.state.config.graph.land
8211 && mode == MergeMode::Pr
8212 && self.state.status == RunStatus::Merged
8213 {
8214 self.run_land().await?;
8215 }
8216 self.settle_questions();
8221 Ok(())
8222 }
8223
8224 async fn run_land(&mut self) -> Result<()> {
8235 crate::pr_head::ensure_owner(&self.state)?;
8236 let url = self
8237 .state
8238 .merge
8239 .as_ref()
8240 .map(|m| m.detail.clone())
8241 .unwrap_or_default();
8242 let url = url.lines().next().unwrap_or("").trim().to_owned();
8243 if !url.starts_with("http") {
8244 return Ok(());
8245 }
8246 loop {
8249 match land::land(&mut self.state, &url).await {
8250 Ok(_)
8251 if self
8252 .state
8253 .merge_repairs
8254 .last()
8255 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8256 {
8257 if self.repair_merge_review().await? {
8258 continue;
8259 }
8260 }
8261 Ok(pr) if self.state.parked => {
8262 let _ = pr;
8266 }
8267 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8268 self.state.status = RunStatus::Superseded;
8269 self.state.save()?;
8270 }
8271 Ok(pr) => {
8272 self.state.status = match pr.state {
8273 land::PrLifecycle::Merged => RunStatus::Merged,
8274 _ => RunStatus::Blocked,
8275 };
8276 if bump::should_release_bump(self.state.status)
8283 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8284 {
8285 self.state
8291 .event("bump", format!("release bump skipped: {e:#}"));
8292 }
8293 if self.state.status == RunStatus::Merged {
8296 crate::followup::after_merge(&mut self.state, &pr.url).await;
8297 }
8298 self.state.save()?;
8299 }
8300 Err(e) => {
8301 if e.is::<crate::repo_lock::ResourceBlocked>() {
8302 return Err(e);
8303 }
8304 self.state.status = RunStatus::Blocked;
8305 self.state.event("land", format!("gave up: {e}"));
8306 self.state.save()?;
8307 }
8308 }
8309 break;
8310 }
8311 Ok(())
8312 }
8313
8314 async fn repair_merge_review(&mut self) -> Result<bool> {
8317 use crate::run::MergeRepairPhase as Phase;
8318 crate::pr_head::ensure_owner(&self.state)?;
8319 let i = self.state.merge_repairs.len() - 1;
8320 let request = self.state.merge_repairs[i].clone();
8321 if request.phase == Phase::Fixing {
8322 let restored = match self.state.winner().cloned() {
8323 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8324 None => Err(anyhow::anyhow!("repair has no winning branch")),
8325 };
8326 let why = match restored {
8327 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8328 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8329 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8330 };
8331 self.hold_merge_repair(i, why)?;
8332 return Ok(false);
8333 }
8334 let failure = if i >= self.state.config.graph.land_rounds
8335 || self.state.config.graph.review_rounds == 0
8336 {
8337 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8338 } else {
8339 None
8340 };
8341 if let Some(why) = failure {
8342 self.hold_merge_repair(i, why)?;
8343 return Ok(false);
8344 }
8345 if request.phase == Phase::Requested {
8346 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8347 Ok(pr) => pr,
8348 Err(e) => {
8349 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8350 return Ok(false);
8351 }
8352 };
8353 self.state.merge_repairs[i].phase = Phase::Fixing;
8354 self.state.land_approval = None;
8355 self.state.land_armed_head = None;
8356 self.state.save()?;
8357 let evidence = serde_json::to_string_pretty(&request)?;
8360 let reason = format!(
8361 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8362 );
8363 let budget = self.state.config.graph.land_rounds;
8364 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8365 match result {
8366 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8367 Ok(land::Fixed::Failed(why)) => {
8368 self.hold_merge_repair(
8369 i,
8370 format!(
8371 "repair failed: {why}; inspect the branch and run an explicit review"
8372 ),
8373 )?;
8374 return Ok(false);
8375 }
8376 Err(e) => {
8377 self.hold_merge_repair(
8378 i,
8379 format!(
8380 "repair failed: {e:#}; inspect the branch and run an explicit review"
8381 ),
8382 )?;
8383 return Ok(false);
8384 }
8385 }
8386 self.begin_merge_repair_review(i)?;
8390 }
8391 self.review_loop().await?;
8392 self.gate().await?;
8393 if self.resource_halt()? || self.state.parked {
8394 return Ok(false);
8395 }
8396 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8397 != Some(RunStatus::Gating)
8398 || !self.state.gate_status().ok()
8399 {
8400 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8401 return Ok(false);
8402 }
8403 let winner = self
8406 .state
8407 .winner()
8408 .cloned()
8409 .context("repair has no winning branch")?;
8410 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8411 if self.state.gate_head.as_deref() != Some(after.as_str())
8412 || !git::is_clean(&winner.worktree).await?
8413 {
8414 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8415 return Ok(false);
8416 }
8417 let expected = self
8418 .state
8419 .pr_pushed_head
8420 .as_deref()
8421 .unwrap_or(&request.head)
8422 .to_owned();
8423 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8424 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8427 let refspec = format!("{after}:refs/heads/{public}");
8428 let push = git::git_raw(
8429 &winner.worktree,
8430 &["push", &lease, &self.state.config.merge.remote, &refspec],
8431 )
8432 .await;
8433 match push {
8434 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8435 Ok(out) => {
8436 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8437 return Ok(false);
8438 }
8439 Err(e) => {
8440 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8441 return Ok(false);
8442 }
8443 }
8444 self.state.merge_repairs[i].phase = Phase::Complete;
8445 self.state.status = RunStatus::Landing;
8446 self.state.event(
8447 "repair",
8448 "new review and gate completed; fresh owner merge decision required",
8449 );
8450 self.state.save()?;
8451 Ok(true)
8452 }
8453
8454 fn merge_repair_artifacts(&self) -> PathBuf {
8455 let artifacts = agent::artifacts_dir(&self.state.dir());
8456 match self
8457 .state
8458 .merge_repairs
8459 .last()
8460 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8461 {
8462 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8463 None => artifacts,
8464 }
8465 }
8466
8467 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8468 self.state.reviews.clear();
8469 self.state.gate.clear();
8470 self.state.gate_ran = false;
8471 self.state.gate_head = None;
8472 self.state.gate_fixes.clear();
8473 self.state.pre_gate.clear();
8474 self.state.pre_gate_commit = None;
8475 self.state.contested_handoff = None;
8476 self.state.land_reruns = None;
8477 self.state.land_approval = None;
8478 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8479 self.state.status = RunStatus::Reviewing;
8480 self.state.save()
8481 }
8482
8483 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8484 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8485 self.state.merge_repairs[index].failure = Some(why.clone());
8486 self.state.status = RunStatus::Blocked;
8487 self.state.event("repair", why);
8488 self.state.save()
8489 }
8490
8491 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8495 if let Some(existing) = self.state.seats.get(key)
8496 && existing.agent == agent
8497 {
8498 return existing.clone();
8499 }
8500 let fresh = if self.state.seats.contains_key(key) {
8503 handover_seat(key, agent, self.state.next_seat_seed())
8504 } else {
8505 SeatState::new(key, agent, self.state.seed)
8506 };
8507 self.state.seats.insert(key.to_owned(), fresh.clone());
8508 fresh
8509 }
8510
8511 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8517 match self.state.seats.get(key) {
8518 Some(s) if s.agent != spec.id => {
8519 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8520 }
8521 _ => spec,
8522 }
8523 }
8524
8525 fn view(&self, c: &Candidate) -> CandidateView {
8527 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8528 .unwrap_or_default();
8529 let (patch, _) = blind::sanitize_patch(
8530 &format!("candidate {} patch", c.label),
8531 &raw,
8532 &self.state.config.blind,
8533 );
8534 CandidateView {
8535 label: c.label,
8536 branch: c.branch.clone(),
8537 summary: c.summary.clone(),
8538 stat: c.stat.clone(),
8539 patch,
8540 }
8541 }
8542
8543 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8545 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8546 prompt::judge(
8547 "(see above)",
8548 &views,
8549 self.roles.judges.len(),
8550 base_short,
8551 "en",
8552 )
8553 }
8554
8555 fn vote_prompt_full(
8561 &self,
8562 j: usize,
8563 labels: &[char],
8564 language: &str,
8565 candidates: &[Candidate],
8566 base_short: &str,
8567 ) -> String {
8568 let mut text = format!(
8569 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8570 prompt::final_vote(labels, language),
8571 self.state.instruction,
8572 self.candidate_block(candidates, base_short)
8573 );
8574 if let Some(own) = self
8575 .state
8576 .judgements
8577 .get(j)
8578 .filter(|r| !r.ranking.is_empty())
8579 {
8580 let reasons = own
8581 .reasons
8582 .iter()
8583 .map(|(k, v)| format!("- {k}: {v}"))
8584 .collect::<Vec<_>>()
8585 .join("\n");
8586 text.push_str(&format!(
8587 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8588 own.ranking.iter().collect::<String>(),
8589 if reasons.is_empty() {
8590 ""
8591 } else {
8592 ", because:\n"
8593 }
8594 ));
8595 }
8596 if !self.state.deliberation.is_empty() {
8597 text.push_str("\n# What was argued before this vote\n");
8598 for t in self.transcript(&[], j) {
8599 text.push_str(&format!(
8600 "\n## {}{}\n\n{}\n",
8601 t.who,
8602 if t.is_self { " (you)" } else { "" },
8603 t.body.trim()
8604 ));
8605 }
8606 }
8607 text
8608 }
8609
8610 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8617 let mut turns = Vec::new();
8618 for j in &self.state.judgements {
8619 if j.ranking.is_empty() {
8620 continue;
8621 }
8622 let reasons = j
8623 .reasons
8624 .iter()
8625 .map(|(k, v)| format!("- {k}: {v}"))
8626 .collect::<Vec<_>>()
8627 .join("\n");
8628 turns.push(Turn {
8629 who: format!("Judge {} (opening ranking)", j.judge),
8630 is_self: j.judge == self_idx + 1,
8631 body: format!(
8632 "Ranked {}{}{reasons}",
8633 j.ranking.iter().collect::<String>(),
8634 if reasons.is_empty() {
8635 ""
8636 } else {
8637 ", because:\n"
8638 }
8639 ),
8640 });
8641 }
8642 for t in self
8643 .state
8644 .deliberation
8645 .iter()
8646 .flat_map(|r| r.turns.iter())
8647 .chain(current)
8648 {
8649 turns.push(Turn {
8650 who: format!("Judge {}", t.judge),
8651 is_self: t.judge == self_idx + 1,
8652 body: t.body.clone(),
8653 });
8654 }
8655 turns
8656 }
8657}
8658
8659fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8661 agent::has_session(spec.kind, seat, sessions)
8662}
8663
8664fn next_untried_in_roster<'a>(
8685 roster: &'a [AgentSpec],
8686 start: usize,
8687 tried: &BTreeSet<String>,
8688) -> Option<&'a AgentSpec> {
8689 roster
8690 .get(start + 1..)?
8691 .iter()
8692 .find(|s| !tried.contains(&s.id))
8693}
8694
8695fn pick_successor<'a>(
8716 roster: &'a [AgentSpec],
8717 start: usize,
8718 tried: &BTreeSet<String>,
8719 carried: Option<&BTreeSet<String>>,
8720 others: &BTreeSet<String>,
8721) -> Option<&'a AgentSpec> {
8722 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8723 roster
8724 .get(start + 1..)
8725 .and_then(|tail| {
8726 tail.iter()
8727 .filter(free)
8728 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8729 })
8730 .or_else(|| {
8731 carried
8732 .is_some()
8733 .then(|| roster.iter().find(free))
8734 .flatten()
8735 })
8736 .or_else(|| match carried {
8737 Some(c) => next_for_seat(roster, start, tried, c),
8738 None => next_untried_in_roster(roster, start, tried),
8739 })
8740}
8741
8742fn next_for_seat<'a>(
8752 roster: &'a [AgentSpec],
8753 start: usize,
8754 round_tried: &BTreeSet<String>,
8755 carried_failed: &BTreeSet<String>,
8756) -> Option<&'a AgentSpec> {
8757 roster
8758 .get(start + 1..)?
8759 .iter()
8760 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8761 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8762}
8763
8764fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8770 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8771 return spec;
8772 };
8773 let ok = |id: &str| !h.failed.contains(id);
8774 if let Some(last) = h.last_ok.as_deref()
8775 && ok(last)
8776 && let Some(s) = roster.iter().find(|s| s.id == last)
8777 {
8778 return s.clone();
8779 }
8780 if ok(&spec.id) {
8781 return spec;
8782 }
8783 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8784 roster
8785 .get(from + 1..)
8786 .into_iter()
8787 .flatten()
8788 .chain(roster.iter())
8789 .find(|s| ok(&s.id))
8790 .cloned()
8791 .unwrap_or(spec)
8792}
8793
8794pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8798 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8799}
8800
8801const TIMED_OUT: &str = "timed out";
8804
8805impl FailClass {
8806 fn of(out: &AgentOutcome) -> Option<Self> {
8808 match out {
8809 AgentOutcome::Ok(_) => None,
8810 AgentOutcome::Quota(_) => Some(Self::Quota),
8811 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8812 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8813 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8814 AgentOutcome::ResourceBlocked(_) => None,
8817 }
8818 }
8819
8820 fn stem_word(&self) -> &'static str {
8822 match self {
8823 Self::Quota => "quota",
8824 _ => "handover",
8825 }
8826 }
8827}
8828
8829fn failure_signature(msg: &str) -> String {
8833 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8834 let mut out = Vec::new();
8835 for word in line.split_whitespace() {
8836 if word.contains('/') || word.contains('\\') {
8837 out.push("<path>".to_owned());
8838 continue;
8839 }
8840 let mut w = String::new();
8841 let mut in_digits = false;
8842 for c in word.chars() {
8843 if c.is_ascii_digit() {
8844 if !in_digits {
8845 w.push('#');
8846 }
8847 in_digits = true;
8848 } else {
8849 in_digits = false;
8850 w.push(c);
8851 }
8852 }
8853 out.push(w);
8854 }
8855 out.join(" ").chars().take(120).collect()
8856}
8857
8858fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8865 match cur {
8866 FailClass::Quota | FailClass::Timeout => true,
8867 FailClass::Other(_) => prev != Some(cur),
8868 }
8869}
8870
8871fn fail_reason(out: &AgentOutcome) -> String {
8873 match out {
8874 AgentOutcome::Ok(_) => String::new(),
8875 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8876 AgentOutcome::Dropped(o) => format!(
8877 "the CLI dropped the stream ({})",
8878 o.dropped
8879 .as_ref()
8880 .map(|d| d.why.as_str())
8881 .unwrap_or("it ended without delivering its answer")
8882 ),
8883 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8884 e.lines().next().unwrap_or("").chars().take(160).collect()
8885 }
8886 }
8887}
8888
8889pub(crate) fn record_handover(
8892 state: &mut RunState,
8893 node: &str,
8894 seat: &str,
8895 from: &str,
8896 to: &str,
8897 class: &FailClass,
8898 reason: &str,
8899) {
8900 let message = if *class == FailClass::Quota {
8901 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
8902 } else {
8903 format!("{seat}: handed over {from} -> {to} ({reason})")
8904 };
8905 state.event(node, message);
8906 state.handovers.push(Handover {
8907 at: Timestamp::now(),
8908 node: node.to_owned(),
8909 seat: seat.to_owned(),
8910 from: from.to_owned(),
8911 to: to.to_owned(),
8912 reason: reason.to_owned(),
8913 });
8914}
8915
8916fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
8930 commands.iter().any(|c| c.exit_code.is_none())
8931}
8932
8933fn verified_noop_claim(
8946 usable: bool,
8947 commands: &[agent::CommandEvidence],
8948 text: &str,
8949) -> Option<String> {
8950 (usable && !has_unconfirmed_command(commands))
8951 .then(|| verdict::verified_noop(text))
8952 .flatten()
8953}
8954
8955fn short(commit: &str) -> String {
8956 commit.chars().take(7).collect()
8957}
8958
8959fn make_executable(path: &Path) -> Result<()> {
8960 #[cfg(unix)]
8961 {
8962 use std::os::unix::fs::PermissionsExt as _;
8963 let mut perms = std::fs::metadata(path)?.permissions();
8964 perms.set_mode(0o755);
8965 std::fs::set_permissions(path, perms)?;
8966 }
8967 #[cfg(not(unix))]
8968 {
8969 let _ = path;
8970 }
8971 Ok(())
8972}
8973
8974struct WaveCtx<'a> {
8981 run: &'a str,
8984 node: &'a str,
8986 prompts: &'a Prompts,
8987 cache: Option<&'a Path>,
8989 round: Option<usize>,
8992 carry_seats: bool,
8996 pauses: Option<(&'a Pause, &'a Pause)>,
8999}
9000
9001fn repository_wait(
9003 state: &RunState,
9004 pauses: Option<(&Pause, &Pause)>,
9005) -> crate::repo_lock::WaitPolicy {
9006 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9007 let inherited = crate::repo_lock::WaitPolicy::current();
9008 crate::repo_lock::WaitPolicy::new(
9009 Duration::from_secs(state.config.graph.timeout_resource_wait),
9010 move || {
9011 inherited.cancelled()
9012 || pauses
9013 .as_ref()
9014 .is_some_and(|(a, b)| a.parked() || b.parked())
9015 },
9016 )
9017}
9018
9019fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9020 if error.is::<crate::repo_lock::ResourceBlocked>() {
9021 state.resource_block = Some(crate::run::ResourceBlock {
9022 node: "repository".into(),
9023 kind: "repository coordination".into(),
9024 detail: error.to_string(),
9025 at: Timestamp::now(),
9026 });
9027 state.event("resource", error.to_string());
9028 }
9029}
9030
9031async fn coordinate_operation(
9032 state: &mut RunState,
9033 cwd: &Path,
9034 wait: &crate::repo_lock::WaitPolicy,
9035) -> Result<Option<crate::repo_lock::SeatLease>> {
9036 let result = crate::repo_lock::SeatLease::operation_with(cwd, wait).await;
9037 if let Err(e) = &result {
9038 record_repository_block(state, e);
9039 }
9040 result
9041}
9042
9043async fn run_one(
9045 job: SeatJob,
9046 sem: Arc<Semaphore>,
9047 ctx: &WaveCtx<'_>,
9048 state: &mut RunState,
9049 attempt: usize,
9050) -> (SeatState, AgentOutcome) {
9051 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9052 .await
9053 .pop()
9054 .expect("one job in, one result out");
9055 (seat, out)
9056}
9057
9058async fn wave(
9064 jobs: Vec<SeatJob>,
9065 sem: Arc<Semaphore>,
9066 ctx: &WaveCtx<'_>,
9067 state: &mut RunState,
9068 attempt: usize,
9069) -> Vec<(usize, SeatState, AgentOutcome)> {
9070 let WaveCtx {
9071 run,
9072 node,
9073 prompts,
9074 cache,
9075 round,
9076 carry_seats: _,
9077 pauses,
9078 } = *ctx;
9079 for job in &jobs {
9080 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9081 }
9082 if let Err(e) = state.save() {
9083 tracing::warn!("could not persist in-progress seats: {e:#}");
9088 }
9089 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9105 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9109 let cache_guard = if let Some(cache_dir) = cache {
9110 if jobs_had_a_writer {
9111 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9112 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9113 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9114 {
9115 Ok(Some(g)) => Some(g),
9116 Ok(None) => {
9119 state.resource_block = Some(crate::run::ResourceBlock {
9120 node: node.to_owned(),
9121 kind: "shared build cache".to_owned(),
9122 detail: "the run is parking".to_owned(),
9123 at: Timestamp::now(),
9124 });
9125 return block_wave(
9126 jobs,
9127 state,
9128 node,
9129 attempt,
9130 "parked while waiting for the shared build cache",
9131 );
9132 }
9133 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9136 Err(e) => {
9137 let why = format!("{e:#}");
9141 state.resource_block = Some(crate::run::ResourceBlock {
9142 node: node.to_owned(),
9143 kind: "shared build cache".to_owned(),
9144 detail: why.clone(),
9145 at: Timestamp::now(),
9146 });
9147 state.event(
9148 "resource",
9149 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9150 );
9151 return block_wave(jobs, state, node, attempt, &why);
9152 }
9153 }
9154 } else {
9155 None
9156 }
9157 } else {
9158 None
9159 };
9160 state.seats_spawning(node, attempt);
9163 if let Err(e) = state.save() {
9164 tracing::warn!("could not persist the start of a wave: {e:#}");
9165 }
9166 let slot_limit = resource_limit
9170 + jobs
9171 .iter()
9172 .map(|j| j.timeout)
9173 .max()
9174 .unwrap_or_default()
9175 .saturating_mul(jobs.len() as u32);
9176 let mut set = tokio::task::JoinSet::new();
9177 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9180 let overlay = prompts.overlay(node);
9181 let repository_wait = repository_wait(state, pauses);
9182 for (i, mut job) in jobs.into_iter().enumerate() {
9183 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9184 if cache.is_some() {
9185 job.prompt.push('\n');
9186 job.prompt
9187 .push_str(&prompt::build_cache_note(node, job.allow_write));
9188 }
9189 let sem = Arc::clone(&sem);
9190 let run = run.to_owned();
9191 let node = node.to_owned();
9192 let attachments = if node == "implement" {
9195 state.attachments.clone()
9196 } else {
9197 Vec::new()
9198 };
9199 let cache = cache
9210 .filter(|_| job.allow_write && cache_guard.is_some())
9211 .map(Path::to_path_buf);
9212 let spawned_tx = spawned_tx.clone();
9213 let repository_wait = repository_wait.clone();
9214 set.spawn(async move {
9215 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9219 Ok(p) => p,
9220 Err(_) => {
9221 return (
9222 i,
9223 job.seat,
9224 AgentOutcome::ResourceBlocked(format!(
9225 "no parallel slot became free within {}s",
9226 slot_limit.as_secs()
9227 )),
9228 );
9229 }
9230 };
9231 spawned_tx.send(job.seat.key.clone()).ok();
9232 let mut seat = job.seat;
9233 let out = repository_wait
9234 .scope(agent::invoke(
9235 &job.spec,
9236 &mut seat,
9237 &Invocation {
9238 cwd: &job.cwd,
9239 prompt: &job.prompt,
9240 timeout: job.timeout,
9241 allow_write: job.allow_write,
9242 unsandboxed: false,
9243 sessions: job.sessions,
9244 artifacts: &job.artifacts,
9245 stem: &job.stem,
9246 run: &run,
9247 node: &node,
9248 cache_dir: cache.as_deref(),
9249 attachments: &attachments,
9250 writable: &[],
9251 },
9252 ))
9253 .await;
9254 let out = match out {
9255 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9256 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9257 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9265 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9266 Ok(o) => AgentOutcome::Failed(format!(
9267 "exited with {:?} and no usable output",
9268 o.exit_code
9269 )),
9270 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9271 AgentOutcome::ResourceBlocked(e.to_string())
9272 }
9273 Err(e) => AgentOutcome::Failed(e.to_string()),
9274 };
9275 (i, seat, out)
9276 });
9277 }
9278 drop(spawned_tx);
9279 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9280 loop {
9281 let joined = tokio::select! {
9282 Some(key) = spawned_rx.recv() => {
9283 state.seat_spawned(&key);
9285 if let Err(e) = state.save() {
9286 tracing::warn!("could not persist a seat's start: {e:#}");
9287 }
9288 continue;
9289 }
9290 joined = set.join_next() => joined,
9291 };
9292 let Some(joined) = joined else {
9293 break;
9294 };
9295 let (i, seat, out) = match joined {
9296 Ok(v) => v,
9297 Err(e) => {
9301 tracing::error!("agent task panicked: {e}");
9302 continue;
9303 }
9304 };
9305 state.seat_finished(&seat.key);
9306 if let AgentOutcome::ResourceBlocked(why) = &out
9307 && state.resource_block.is_none()
9308 {
9309 state.resource_block = Some(crate::run::ResourceBlock {
9310 node: node.to_owned(),
9311 kind: if why.contains("repository coordination") {
9312 "repository coordination"
9313 } else {
9314 "parallel slot"
9315 }
9316 .to_owned(),
9317 detail: why.clone(),
9318 at: Timestamp::now(),
9319 });
9320 state.event("resource", format!("{node}: no agent was started - {why}"));
9321 }
9322 record_jobs(state, node, round, &seat.key, &out);
9323 if let Err(e) = state.save() {
9324 tracing::warn!("could not persist a seat's completion: {e:#}");
9325 }
9326 if collected.len() <= i {
9327 collected.resize_with(i + 1, || None);
9328 }
9329 collected[i] = Some((i, seat, out));
9330 }
9331 if state
9337 .active
9338 .values()
9339 .any(|a| a.node == node && a.attempt == attempt)
9340 {
9341 state
9342 .active
9343 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9344 if let Err(e) = state.save() {
9345 tracing::warn!("could not persist the end of a wave: {e:#}");
9346 }
9347 }
9348 if let Some(cache_dir) = cache
9355 && jobs_had_a_writer
9356 {
9357 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9358 }
9359 if let Some(guard) = cache_guard {
9360 guard.release();
9361 }
9362 collected.into_iter().flatten().collect()
9363}
9364
9365fn record_jobs(
9376 state: &mut RunState,
9377 node: &str,
9378 round: Option<usize>,
9379 seat: &str,
9380 out: &AgentOutcome,
9381) {
9382 let commands: &[agent::CommandEvidence] = match out {
9383 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9384 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9385 };
9386 let checked_at = Timestamp::now();
9387 for c in commands {
9388 state.jobs.push(JobRecord {
9389 node: node.to_owned(),
9390 round,
9391 seat: seat.to_owned(),
9392 id: c.id.clone(),
9393 description: c.description.clone(),
9394 checked_at,
9395 status: match c.exit_code {
9396 Some(0) => JobStatus::Completed,
9397 Some(_) => JobStatus::Failed,
9398 None => JobStatus::Unknown,
9399 },
9400 exit_code: c.exit_code,
9401 result_summary: c.result_summary.clone(),
9402 source: c.source.clone(),
9403 });
9404 }
9405}
9406
9407fn round_is_clean(
9428 blocking: usize,
9429 e2e_ok: bool,
9430 answered: usize,
9431 expected: usize,
9432 quota_missing: usize,
9433 policy: IncompleteReviewPolicy,
9434) -> bool {
9435 if blocking != 0 || !e2e_ok {
9436 return false;
9437 }
9438 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9439 return true;
9440 }
9441 answered > 0 && expected - answered <= quota_missing
9442}
9443
9444fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9468 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9469 return Some(RunStatus::Gating);
9470 }
9471 let last = reviews.last()?;
9472 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9473 if reviews.len() < max_rounds && !stagnant {
9474 return None;
9475 }
9476 if last.incomplete() && last.blocking == 0 {
9477 return Some(RunStatus::Blocked);
9478 }
9479 if last.e2e_status() == E2eStatus::ResourceBlocked {
9480 return None;
9481 }
9482 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9483 RunStatus::Gating
9484 } else {
9485 RunStatus::Blocked
9486 })
9487}
9488
9489pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9504 if nudged {
9505 (full / 4).max(Duration::from_secs(120)).min(full)
9506 } else {
9507 full
9508 }
9509}
9510
9511#[allow(clippy::too_many_arguments)]
9540async fn ask_json_wave<T>(
9541 jobs: Vec<SeatJob>,
9542 sem: Arc<Semaphore>,
9543 retries: usize,
9544 roster: &[AgentSpec],
9545 ctx: &WaveCtx<'_>,
9546 losses: &mut Vec<QuotaLoss>,
9547 state: &mut RunState,
9548 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9549) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9550where
9551 T: serde::de::DeserializeOwned + Send + 'static,
9552{
9553 ask_wave_with(
9554 jobs,
9555 sem,
9556 retries,
9557 roster,
9558 ctx,
9559 losses,
9560 state,
9561 &|text: &str| {
9562 let v = verdict::extract_json::<T>(text)?;
9563 validate(&v)?;
9564 Ok(v)
9565 },
9566 )
9567 .await
9568}
9569
9570#[allow(clippy::too_many_arguments)]
9579async fn ask_wave_with<T>(
9580 jobs: Vec<SeatJob>,
9581 sem: Arc<Semaphore>,
9582 retries: usize,
9583 roster: &[AgentSpec],
9584 ctx: &WaveCtx<'_>,
9585 losses: &mut Vec<QuotaLoss>,
9586 state: &mut RunState,
9587 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9588) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9589where
9590 T: Send + 'static,
9591{
9592 let n = jobs.len();
9593 let originals: Vec<SeatJob> = jobs;
9594 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9595 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9596 let mut nudges: Vec<usize> = vec![0; n];
9603 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9607 let mut tried: Vec<BTreeSet<String>> = specs
9608 .iter()
9609 .map(|s| BTreeSet::from([s.id.clone()]))
9610 .collect();
9611 let starts: Vec<usize> = specs
9612 .iter()
9613 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9614 .collect();
9615 let carry = ctx.carry_seats && !roster.is_empty();
9616 let carried: Vec<BTreeSet<String>> = originals
9619 .iter()
9620 .map(|j| {
9621 state
9622 .seat_history
9623 .get(&j.seat.key)
9624 .filter(|_| carry)
9625 .map(|h| h.failed.clone())
9626 .unwrap_or_default()
9627 })
9628 .collect();
9629 let mut prev: Vec<Option<FailClass>> = originals
9630 .iter()
9631 .map(|j| {
9632 state
9633 .seat_history
9634 .get(&j.seat.key)
9635 .filter(|_| carry)
9636 .and_then(|h| h.last_fail.clone())
9637 })
9638 .collect();
9639 let next_agent =
9640 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9641 let others: BTreeSet<String> = specs
9642 .iter()
9643 .enumerate()
9644 .filter(|(j, _)| *j != i)
9645 .map(|(_, s)| s.id.clone())
9646 .collect();
9647 pick_successor(
9648 roster,
9649 starts[i],
9650 tried,
9651 carry.then(|| &carried[i]),
9652 &others,
9653 )
9654 .cloned()
9655 };
9656 let mut fresh: Vec<Option<String>> = vec![None; n];
9657 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9658 let mut pending: Vec<usize> = (0..n).collect();
9659
9660 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9665 for round in 0..max_rounds {
9666 if pending.is_empty() {
9667 break;
9668 }
9669 let mut batch = Vec::with_capacity(pending.len());
9670 let mut renudged: Vec<&str> = Vec::new();
9671 for &i in &pending {
9672 let src = &originals[i];
9673 let full: &str = match &src.handover {
9678 Some(h) if specs[i].id != src.spec.id => h,
9679 _ => &src.prompt,
9680 };
9681 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9684 (
9685 full.to_owned(),
9686 src.timeout,
9687 format!("{}-{word}-{}", src.stem, specs[i].id),
9688 )
9689 } else if nudges[i] == 0 {
9690 (full.to_owned(), src.timeout, src.stem.clone())
9691 } else {
9692 renudged.push(src.seat.key.as_str());
9693 let why = done[i]
9694 .as_ref()
9695 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9696 .unwrap_or_else(|| "no parsable answer".to_owned());
9697 let nudge = prompt::nudge(&why);
9698 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9699 let prompt = if nudged {
9700 nudge
9701 } else {
9702 format!("{full}\n\n---\n\n{nudge}")
9703 };
9704 (
9705 prompt,
9706 retry_budget(src.timeout, nudged),
9707 format!("{}-retry{}", src.stem, nudges[i]),
9708 )
9709 };
9710 batch.push(SeatJob {
9711 spec: specs[i].clone(),
9712 seat: seats[i].clone(),
9713 cwd: src.cwd.clone(),
9714 prompt,
9715 timeout,
9716 allow_write: src.allow_write,
9717 sessions: src.sessions,
9718 artifacts: src.artifacts.clone(),
9719 stem,
9720 handover: None,
9721 });
9722 }
9723
9724 if !renudged.is_empty() {
9725 state.event(
9726 ctx.node,
9727 format!("retry {round}: re-asking {}", renudged.join(", ")),
9728 );
9729 }
9730 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9731 let mut still = Vec::new();
9732 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9733 seats[i] = seat;
9734 if let AgentOutcome::ResourceBlocked(why) = &out {
9737 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9738 continue;
9739 }
9740 let class = FailClass::of(&out);
9741 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9744 && nudges[i] < retries
9745 && !roster.is_empty();
9746 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9747 let next = should_hand_over(prev[i].as_ref(), &cur)
9748 .then(|| next_agent(i, &tried[i], &specs))
9749 .flatten();
9750 if carry {
9751 let h = state
9752 .seat_history
9753 .entry(originals[i].seat.key.clone())
9754 .or_default();
9755 h.failed.insert(specs[i].id.clone());
9756 h.last_fail = Some(cur.clone());
9757 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9758 h.last_ok = None;
9759 }
9760 if let Err(e) = state.save() {
9763 tracing::warn!("could not persist a seat's failure history: {e:#}");
9764 }
9765 }
9766 if let Some(next) = next {
9767 record_handover(
9768 state,
9769 ctx.node,
9770 &originals[i].seat.key,
9771 &specs[i].id,
9772 &next.id,
9773 &cur,
9774 &fail_reason(&out),
9775 );
9776 tried[i].insert(next.id.clone());
9777 prev[i] = Some(cur.clone());
9778 seats[i] =
9779 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9780 specs[i] = next;
9781 fresh[i] = Some(cur.stem_word().to_owned());
9782 nudges[i] = 0;
9783 done[i] = Some(Err(anyhow::anyhow!(
9784 "handed over after: {}",
9785 fail_reason(&out)
9786 )));
9787 still.push(i);
9788 continue;
9789 }
9790 }
9791 if carry
9792 && !nudge_first
9793 && let Some(cur) = class.clone()
9794 {
9795 let h = state
9797 .seat_history
9798 .entry(originals[i].seat.key.clone())
9799 .or_default();
9800 h.failed.insert(specs[i].id.clone());
9801 h.last_fail = Some(cur);
9802 }
9803 let parsed = match out {
9804 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9805 AgentOutcome::Quota(o) => {
9806 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9807 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9808 }
9809 AgentOutcome::Dropped(o) => {
9813 let why = o
9814 .dropped
9815 .as_ref()
9816 .map(|d| d.why.as_str())
9817 .unwrap_or("the CLI ended the stream without delivering its answer");
9818 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9819 }
9820 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9821 Err(anyhow::anyhow!(e))
9822 }
9823 };
9824 let quota = class == Some(FailClass::Quota);
9825 let failed = parsed.is_err();
9826 done[i] = Some(parsed);
9827 if carry && !failed {
9828 let h = state
9829 .seat_history
9830 .entry(originals[i].seat.key.clone())
9831 .or_default();
9832 h.failed.remove(&specs[i].id);
9833 h.last_ok = Some(specs[i].id.clone());
9834 h.last_fail = None;
9835 }
9836 let agent_failure = class.is_some()
9844 && !nudge_first
9845 && !roster.is_empty()
9846 && next_agent(i, &tried[i], &specs).is_some();
9847 if failed && !quota && !agent_failure && nudges[i] < retries {
9848 nudges[i] += 1;
9849 still.push(i);
9850 }
9851 }
9852 pending = still;
9853 }
9854
9855 for (i, q) in last_quota.into_iter().enumerate() {
9858 if let Some(reset) = q {
9859 losses.push(QuotaLoss {
9860 seat: originals[i].seat.key.clone(),
9861 node: ctx.node.to_owned(),
9862 at: Timestamp::now(),
9863 reset,
9864 });
9865 }
9866 }
9867
9868 seats
9869 .into_iter()
9870 .zip(done)
9871 .zip(nudges)
9872 .map(|((seat, res), attempts)| {
9873 (
9874 seat,
9875 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
9876 attempts,
9877 )
9878 })
9879 .collect()
9880}
9881
9882fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
9884 results
9885 .iter()
9886 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
9887}
9888
9889fn block_wave(
9893 jobs: Vec<SeatJob>,
9894 state: &mut RunState,
9895 node: &str,
9896 attempt: usize,
9897 why: &str,
9898) -> Vec<(usize, SeatState, AgentOutcome)> {
9899 state.resource_wait = None;
9900 state
9901 .active
9902 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9903 if let Err(e) = state.save() {
9904 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
9905 }
9906 jobs.into_iter()
9907 .enumerate()
9908 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
9909 .collect()
9910}
9911
9912async fn acquire_cache_lease(
9925 state: &mut RunState,
9926 cache_dir: &Path,
9927 owner: &crate::cache::Owner,
9928 budget: Duration,
9929 context: &str,
9930 cancel: &(dyn Fn() -> bool + Sync),
9931) -> Result<Option<crate::cache::Guard>> {
9932 let home = crate::run::home();
9933 let started = Instant::now();
9934 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
9935 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
9936 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
9937 Err(e) => {
9938 state.event(
9939 "verify",
9940 format!("{context}: could not check the shared build cache: {e:#}"),
9941 );
9942 if let Err(e2) = state.save() {
9943 tracing::warn!("could not persist a cache-check failure: {e2:#}");
9944 }
9945 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
9946 }
9947 };
9948 state.event(
9949 "verify",
9950 format!(
9951 "{context}: waiting for the shared build cache at {} ({})",
9952 cache_dir.display(),
9953 busy.describe()
9954 ),
9955 );
9956 state.resource_wait = Some(crate::run::ResourceWait {
9959 node: context.to_owned(),
9960 kind: "shared build cache".to_owned(),
9961 holder: busy.describe(),
9962 since: Timestamp::now(),
9963 limit_secs: budget.as_secs(),
9964 });
9965 if let Err(e) = state.save() {
9966 tracing::warn!("could not persist a cache wait: {e:#}");
9967 }
9968 let remaining = budget.saturating_sub(started.elapsed());
9969 let waited = crate::cache::wait_for_unless(
9970 &home,
9971 cache_dir,
9972 owner,
9973 remaining,
9974 Duration::from_secs(5),
9975 cancel,
9976 )
9977 .await;
9978 state.resource_wait = None;
9979 match waited {
9980 Ok(g) => Ok(g),
9981 Err(e) => {
9982 state.event("verify", format!("{context}: {e:#}"));
9983 if let Err(e2) = state.save() {
9984 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
9985 }
9986 Err(e)
9987 }
9988 }
9989}
9990
9991#[derive(Debug)]
9996struct CacheCheckError(String);
9997
9998impl std::fmt::Display for CacheCheckError {
9999 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10000 f.write_str(&self.0)
10001 }
10002}
10003
10004impl std::error::Error for CacheCheckError {}
10005
10006#[allow(clippy::too_many_arguments)]
10027async fn with_cache_lease<'s, F, Fut>(
10028 state: &'s mut RunState,
10029 cache_dir: Option<&Path>,
10030 node: &str,
10031 seat: &str,
10032 worktree: &Path,
10033 head: &str,
10034 budget: Duration,
10035 context: &str,
10036 wait: &crate::repo_lock::WaitPolicy,
10037 body: F,
10038) -> Result<(Vec<CommandOutcome>, bool)>
10039where
10040 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10041 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10042{
10043 let started = Instant::now();
10046 let home = crate::run::home();
10047 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10048 let guard = if let Some(cache_dir) = cache_dir {
10049 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10050 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10051 wait.cancelled()
10052 })
10053 .await
10054 .and_then(|g| {
10055 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10056 }) {
10057 Ok(g) => Some(g),
10058 Err(e) => {
10059 return Ok((
10060 vec![CommandOutcome {
10061 command: "(waiting for the shared build cache)".into(),
10062 code: None,
10063 output_tail: e.to_string(),
10064 duration_ms: started.elapsed().as_millis() as u64,
10065 resource_blocked: true,
10066 }],
10067 false,
10068 ));
10069 }
10070 }
10071 } else {
10072 None
10073 };
10074 let _coordination = coordinate_operation(state, worktree, wait).await?;
10075 if let Some(cache_dir) = cache_dir {
10076 let identity = crate::cache::Identity::new(worktree, head);
10077 if let Err(e) = crate::cache::ensure_fresh(
10078 &home,
10079 cache_dir,
10080 &identity,
10081 &state.config.verify.cargo_manifests(),
10082 ) {
10083 state.event(
10084 "verify",
10085 format!(
10086 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10087 worktree.display(),
10088 short(head)
10089 ),
10090 );
10091 if let Some(guard) = guard {
10092 guard.release();
10093 }
10094 return Ok((
10095 vec![CommandOutcome {
10096 command: "(confirming the shared build cache is fresh)".into(),
10097 code: None,
10098 output_tail: e.to_string(),
10099 duration_ms: started.elapsed().as_millis() as u64,
10100 resource_blocked: true,
10101 }],
10102 false,
10103 ));
10104 }
10105 }
10106 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10107 if !timed_out_pids.is_empty() {
10108 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10109 }
10110 drop(_coordination);
10113 if let Some(guard) = guard {
10114 guard.release();
10115 }
10116 Ok((outcomes, retried))
10117}
10118
10119async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10131 wait_for_pids_with(
10132 pids,
10133 crate::proc::pid_alive,
10134 LEASE_RELEASE_POLL,
10135 LEASE_RELEASE_MAX_WAIT,
10136 )
10137 .await;
10138}
10139
10140async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10146 pids: &[u32],
10147 alive: F,
10148 poll: Duration,
10149 max_wait: Duration,
10150) {
10151 let deadline = Instant::now() + max_wait;
10152 loop {
10153 if pids.iter().all(|&pid| !alive(pid)) {
10154 return;
10155 }
10156 if Instant::now() >= deadline {
10157 return;
10158 }
10159 tokio::time::sleep(poll).await;
10160 }
10161}
10162
10163fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10170 outcomes.iter().any(|o| o.resource_blocked)
10171}
10172
10173enum GateFix {
10175 Retry,
10177 Stop,
10180 Defer,
10183}
10184
10185fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10193 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10194 red.peek().is_some()
10195 && red.all(|o| {
10196 !o.resource_blocked
10197 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10198 && !o.output_tail.trim().is_empty()
10199 })
10200}
10201
10202fn e2e_outcome_label(o: &CommandOutcome) -> String {
10206 if o.ok() {
10207 return "pass".to_owned();
10208 }
10209 let reason = if o.build_failed() {
10210 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10211 } else {
10212 format!("FAIL ({:?})", o.code)
10213 };
10214 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10215}
10216
10217async fn run_e2e_with_retry(
10225 state: &mut RunState,
10226 shell: &[String],
10227 commands: &[String],
10228 worktree: &Path,
10229 timeout: Duration,
10230 context: &str,
10231) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10232 let (mut e2e, mut timed_out_pids) = run_commands(
10233 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10234 )
10235 .await;
10236 for o in &e2e {
10237 state.event(
10238 "verify",
10239 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10240 );
10241 }
10242 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10246 if verify_retried {
10247 state.event(
10248 "verify",
10249 format!(
10250 "{context}: verify could not build/link, not a test result — retrying once \
10251 before concluding"
10252 ),
10253 );
10254 let retried = run_commands(
10255 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10256 )
10257 .await;
10258 e2e = retried.0;
10259 timed_out_pids.extend(retried.1);
10262 for o in &e2e {
10263 state.event(
10264 "verify",
10265 format!(
10266 "{context}: retry `{}` -> {}",
10267 o.command,
10268 e2e_outcome_label(o)
10269 ),
10270 );
10271 }
10272 }
10273 (e2e, verify_retried, timed_out_pids)
10274}
10275
10276#[allow(clippy::too_many_arguments)]
10292async fn run_commands(
10293 state: &mut RunState,
10294 node: &str,
10295 task: &str,
10296 attempt: usize,
10297 shell: &[String],
10298 commands: &[String],
10299 cwd: &Path,
10300 timeout: Duration,
10301) -> (Vec<CommandOutcome>, Vec<u32>) {
10302 if commands.is_empty() {
10303 return (Vec::new(), Vec::new());
10308 }
10309 let mut out = Vec::new();
10310 let mut timed_out_pids = Vec::new();
10311 let total = commands.len();
10312 for (idx, command) in commands.iter().enumerate() {
10313 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10314 if let Err(e) = state.save() {
10315 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10316 }
10317 let started = Instant::now();
10318 let mut cmd = tokio::process::Command::new(&shell[0]);
10319 cmd.quiet();
10320 cmd.args(&shell[1..])
10321 .arg(command)
10322 .current_dir(cwd)
10323 .stdin(std::process::Stdio::null())
10324 .stdout(std::process::Stdio::piped())
10325 .stderr(std::process::Stdio::piped())
10326 .kill_on_drop(true);
10327 let spawned = cmd.spawn();
10328 let (code, body) = match spawned {
10329 Ok(child) => {
10330 let pid = child.id();
10335 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10336 Ok(Ok(o)) => {
10337 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10338 body.push_str(&String::from_utf8_lossy(&o.stderr));
10339 (o.status.code(), body)
10340 }
10341 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10342 Err(_) => {
10343 if let Some(pid) = pid {
10344 timed_out_pids.push(pid);
10345 }
10346 (None, format!("timed out after {}s", timeout.as_secs()))
10347 }
10348 }
10349 }
10350 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10351 };
10352 out.push(CommandOutcome {
10353 command: command.clone(),
10354 code,
10355 output_tail: tail(&body, OUTPUT_TAIL),
10356 duration_ms: started.elapsed().as_millis() as u64,
10357 resource_blocked: false,
10358 });
10359 }
10360 state.task_finished(task);
10361 if let Err(e) = state.save() {
10362 tracing::warn!("could not persist the end of {task}: {e:#}");
10363 }
10364 (out, timed_out_pids)
10365}
10366
10367fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10379 let repo = repo.display();
10380 match style {
10381 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10382 MergeStyle::Squash => {
10383 let subject = message
10386 .lines()
10387 .next()
10388 .unwrap_or(branch)
10389 .replace(['\\', '"', '$', '`'], "");
10390 format!(
10391 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10392 )
10393 }
10394 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10395 }
10396}
10397
10398const PR_TITLE_MAX: usize = 240;
10409
10410fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10421 let line = queue::first_line(text)?;
10422 let chars: Vec<char> = line.chars().collect();
10423 let (kept, mark) = if chars.len() <= max {
10424 (line.to_owned(), "")
10425 } else if let Some(end) = sentence_end(&chars, max) {
10426 (chars[..end].iter().collect::<String>(), "")
10427 } else {
10428 let room = max.saturating_sub(3);
10429 let cut = if chars[room].is_whitespace() {
10432 room
10433 } else {
10434 chars[..room]
10435 .iter()
10436 .rposition(|c| c.is_whitespace())
10437 .unwrap_or(room)
10438 };
10439 let head: String = chars[..cut].iter().collect();
10440 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10441 (head.to_owned(), "...")
10442 };
10443 if kept.is_empty() {
10444 return None;
10445 }
10446 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10447 return None;
10448 }
10449 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10450 return None;
10451 }
10452 Some(format!("{kept}{mark}"))
10453}
10454
10455fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10459 const MIN: usize = 20;
10460 for i in MIN..max.min(chars.len()) {
10461 if !matches!(chars[i], '.' | '!' | '?') {
10462 continue;
10463 }
10464 let Some(&next) = chars.get(i + 1) else {
10465 continue;
10466 };
10467 if !next.is_whitespace() {
10468 continue;
10469 }
10470 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10471 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10472 continue;
10473 }
10474 let word: String = chars[..i]
10475 .iter()
10476 .rev()
10477 .take_while(|c| !c.is_whitespace())
10478 .collect::<Vec<_>>()
10479 .into_iter()
10480 .rev()
10481 .collect();
10482 let word = word.to_ascii_lowercase();
10483 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10484 continue;
10485 }
10486 let end = chars[..i]
10487 .iter()
10488 .rposition(|c| !c.is_whitespace())
10489 .map_or(i, |p| p + 1);
10490 return Some(end);
10491 }
10492 None
10493}
10494
10495struct PrMessage {
10500 title: String,
10501 body: String,
10502}
10503
10504impl PrMessage {
10505 fn commit_message(&self) -> String {
10509 format!("{}\n\n{}", self.title, self.body)
10510 }
10511}
10512
10513fn title_marker(line: &str) -> Option<&str> {
10515 let line = line.trim();
10516 let head = line.get(..6)?;
10517 head.eq_ignore_ascii_case("title:")
10518 .then(|| line[6..].trim())
10519}
10520
10521fn summary_title(summary: &str) -> Option<String> {
10526 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10527 let raw = title_marker(first)?;
10528 if raw.is_empty() {
10529 return None;
10530 }
10531 let title = queue::title_from(raw, PR_TITLE_MAX);
10532 let lower = title.to_ascii_lowercase();
10533 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10534 return None;
10535 }
10536 Some(title)
10537}
10538
10539const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10541
10542pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10544
10545fn is_review_run(state: &RunState) -> bool {
10549 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10550}
10551
10552fn review_title(state: &RunState) -> Option<String> {
10557 english_subject(
10558 state.reviewed_commits.as_ref()?.first()?,
10559 state.config.graph.github_english(),
10560 )
10561}
10562
10563fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10566 let raw = raw.trim();
10567 if raw.is_empty() {
10568 return None;
10569 }
10570 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10571 return None;
10572 }
10573 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10574 return None;
10575 }
10576 let title = queue::title_from(raw, PR_TITLE_MAX);
10577 let lower = title.to_ascii_lowercase();
10578 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10579 return None;
10580 }
10581 Some(title)
10582}
10583
10584#[derive(Debug, Clone, PartialEq, Eq)]
10587struct BranchFacts {
10588 commits: Vec<(String, String)>,
10590 stat: String,
10592}
10593
10594const STAT_FILE_LINES: usize = 25;
10596const COMMITS_MAX_CHARS: usize = 20_000;
10598
10599async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10602 let commits = git::commit_log(repo, base, branch).await.ok()?;
10603 if commits.is_empty() {
10604 return None;
10605 }
10606 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10607 let lines: Vec<&str> = stat.lines().collect();
10608 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10609 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10610 let more = format!(" ... {omitted} more file(s)");
10611 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10612 kept.push(&more);
10613 kept.push(lines[lines.len() - 1]);
10614 kept.join("\n")
10615 } else {
10616 lines.join("\n")
10617 };
10618 Some(BranchFacts { commits, stat })
10619}
10620
10621fn markdown_safe(text: &str) -> String {
10624 text.replace("</details>", "</details>")
10625 .replace("\x60\x60\x60", "~~~")
10626}
10627
10628fn neutral_title(state: &RunState, winner: char) -> String {
10629 format!(
10630 "chore: land candidate {} of run {}",
10631 winner.to_ascii_uppercase(),
10632 state.id
10633 )
10634}
10635
10636pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10642 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10643 ""
10644 } else {
10645 pr_title
10646 }
10647}
10648
10649pub fn landing_subject_source(state: &RunState) -> String {
10653 if is_review_run(state) {
10654 let winner = state.candidates.first().map_or('A', |c| c.label);
10655 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10656 }
10657 state.instruction.clone()
10658}
10659
10660fn summary_without_title(summary: &str) -> String {
10663 let mut lines = summary.trim().lines().peekable();
10664 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10665 lines.next();
10666 }
10667 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10668}
10669
10670#[cfg(test)]
10682fn pr_message(state: &RunState, winner: char) -> PrMessage {
10683 pr_message_with(state, winner, None)
10684}
10685
10686#[cfg(test)]
10689fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10690 let raw = pr_message_raw(
10691 state,
10692 winner,
10693 facts,
10694 crate::pr_outcome::FinalHead::Unreported,
10695 ChangeNote::NotAsked,
10696 );
10697 let id = crate::scrub::Identity::current();
10698 PrMessage {
10699 title: crate::scrub::scrub(&raw.title, &id),
10700 body: crate::scrub::scrub(&raw.body, &id),
10701 }
10702}
10703
10704#[derive(Debug, Clone, Copy)]
10706enum ChangeNote<'a> {
10707 NotAsked,
10709 Summary(&'a crate::run::ChangeSummary),
10711 Unavailable(&'a str),
10713}
10714
10715fn pr_message_raw(
10716 state: &RunState,
10717 winner: char,
10718 facts: Option<&BranchFacts>,
10719 head: crate::pr_outcome::FinalHead<'_>,
10720 change: ChangeNote<'_>,
10721) -> PrMessage {
10722 if !matches!(change, ChangeNote::NotAsked) {
10723 let (title, mut body) = match change {
10724 ChangeNote::Summary(c) => (
10725 c.title.clone(),
10726 format!("## Summary\n\n{}\n\n", c.body.trim()),
10727 ),
10728 ChangeNote::Unavailable(why) => (
10729 neutral_title(state, winner),
10730 format!(
10731 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10732 ),
10733 ),
10734 ChangeNote::NotAsked => unreachable!(),
10735 };
10736 body.push_str(&public_pr_evidence(state, head));
10737 body.push_str(&format!(
10738 "\n---\nmagi:run/{} magi:candidate-{}\n",
10739 state.id,
10740 winner.to_ascii_lowercase()
10741 ));
10742 return PrMessage { title, body };
10743 }
10744 pr_preview(state, winner, facts, head)
10745}
10746
10747fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10750 let mut body = crate::pr_outcome::review_rounds(state);
10751 body.push_str(&crate::pr_outcome::final_verification(state, head));
10752 let open = state.last_round_findings();
10753 if !open.is_empty() {
10754 body.push_str("## Open review findings\n\n");
10755 for f in open {
10756 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10757 }
10758 body.push('\n');
10759 }
10760 if let Some(c) = state.change_summary.as_ref()
10761 && head == crate::pr_outcome::FinalHead::At(&c.head)
10762 {
10763 body.push_str(&format!(
10764 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10765 c.base, c.head
10766 ));
10767 }
10768 body
10769}
10770
10771fn pr_preview(
10773 state: &RunState,
10774 winner: char,
10775 facts: Option<&BranchFacts>,
10776 head: crate::pr_outcome::FinalHead<'_>,
10777) -> PrMessage {
10778 let summary = state
10779 .candidates
10780 .iter()
10781 .find(|c| c.label == winner)
10782 .map(|c| c.summary.as_str())
10783 .unwrap_or_default();
10784 let review = is_review_run(state);
10787 let title = if review {
10788 facts
10789 .and_then(|f| {
10790 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10791 })
10792 .or_else(|| review_title(state))
10793 .or_else(|| {
10794 state
10795 .candidates
10796 .iter()
10797 .find(|c| c.label == winner)
10798 .filter(|c| !c.branch.starts_with("magi/"))
10799 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10800 })
10801 .unwrap_or_else(|| neutral_title(state, winner))
10802 } else {
10803 summary_title(summary).unwrap_or_else(|| {
10804 english_title(
10805 &state.instruction,
10806 PR_TITLE_MAX,
10807 state.config.graph.github_english(),
10808 )
10809 .unwrap_or_else(|| neutral_title(state, winner))
10810 })
10811 };
10812
10813 let mut body = String::new();
10814 let what = summary_without_title(summary);
10815 if !what.is_empty() {
10816 body.push_str("## Summary\n\n");
10817 body.push_str(&what);
10818 body.push_str("\n\n");
10819 }
10820
10821 body.push_str(&crate::pr_outcome::review_rounds(state));
10823 body.push_str(&crate::pr_outcome::final_verification(state, head));
10824 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10825
10826 let open = state.open_findings();
10827 if !open.is_empty() {
10828 body.push_str("## Open review findings\n\n");
10829 for f in &open {
10830 body.push_str(&format!(
10831 "- `{}` [{:?}] {}\n",
10832 f.id, f.severity, "Context retained in local review records."
10833 ));
10834 }
10835 body.push('\n');
10836 }
10837
10838 if let Some(fix) = fix
10839 && !fix.rejected.is_empty()
10840 {
10841 body.push_str("## Declined by the fixer\n\n");
10842 for r in &fix.rejected {
10843 body.push_str(&format!(
10844 "- `{}`: {}\n",
10845 r.id, "Reason retained in local review records."
10846 ));
10847 }
10848 body.push('\n');
10849 }
10850
10851 if review {
10852 body.push_str("## Commits under review\n\n");
10854 if let Some(facts) = facts {
10855 let mut left = COMMITS_MAX_CHARS;
10856 for (i, (subject, text)) in facts.commits.iter().enumerate() {
10857 let mut entry = format!("- {}\n", markdown_safe(subject));
10858 for l in markdown_safe(text).lines() {
10859 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
10860 }
10861 if left == 0 {
10862 body.push_str(&format!(
10863 "- ... {} more commit(s)\n",
10864 facts.commits.len() - i
10865 ));
10866 break;
10867 }
10868 if entry.len() > left {
10869 let mut end = left;
10872 while !entry.is_char_boundary(end) {
10873 end -= 1;
10874 }
10875 entry.truncate(end);
10876 entry.push_str("\n ... (truncated)\n");
10877 left = 0;
10878 } else {
10879 left -= entry.len();
10880 }
10881 body.push_str(&entry);
10882 }
10883 if !facts.stat.trim().is_empty() {
10884 body.push_str(&format!(
10885 "\n## Diff stat\n\n```\n{}\n```\n",
10886 markdown_safe(facts.stat.trim())
10887 ));
10888 }
10889 } else {
10890 match &state.reviewed_commits {
10891 Some(subjects) => {
10892 for s in subjects {
10893 body.push_str(&format!("- {}\n", s.trim()));
10894 }
10895 }
10896 None => {
10897 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
10900 body.push_str(rest.trim());
10901 body.push('\n');
10902 }
10903 }
10904 }
10905 } else {
10906 let task = state.instruction.trim();
10907 let task = if task.is_empty() {
10908 "(empty task)"
10909 } else {
10910 task
10911 };
10912 body.push_str(&format!(
10913 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
10914 task.replace("</details>", "</details>")
10915 ));
10916 }
10917
10918 body.push_str(&format!(
10919 "\n---\nmagi:run/{} magi:candidate-{}\n",
10920 state.id,
10921 winner.to_ascii_lowercase()
10922 ));
10923
10924 PrMessage { title, body }
10925}
10926
10927fn seeded_instruction(state: &RunState) -> String {
10931 match refs::describe(&state.seeds) {
10932 Some(facts) => format!(
10933 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
10934 Candidates start from the unmerged branch named above, when there \
10935 is one, and carry any unmerged commit named by sha as a \
10936 cherry-pick. Check that this is what the task meant before \
10937 building on it.",
10938 state.instruction
10939 ),
10940 None => state.instruction.clone(),
10941 }
10942}
10943
10944async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
10948 let base = &state.base_branch;
10949 let mut against = base.clone();
10950 if mode == MergeMode::Pr {
10951 let remote = &state.config.merge.remote;
10955 let tracking = format!("{remote}/{base}");
10956 let fetched = git::fetch(repo, remote, base).await;
10957 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
10958 against = tracking;
10959 } else {
10960 return false;
10961 }
10962 }
10963 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
10964}
10965
10966fn empty_candidate_detail(state: &RunState, base: &str) -> String {
10969 let mut detail = format!(
10970 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
10971 nothing to open a pull request for"
10972 );
10973 match refs::describe(&state.seeds) {
10974 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
10975 None => detail.push_str(
10976 "\nThe task names no existing branch or commit; if it means to land work \
10977 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
10978 ),
10979 }
10980 detail
10981}
10982
10983#[derive(Debug, PartialEq, Eq)]
10986enum PrPlan {
10987 Create,
10988 Adopt { url: String, title: String },
10989 Stop(String),
10990}
10991
10992fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
10995 match found {
10996 Ok(land::OpenPr::None) => PrPlan::Create,
10997 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
10998 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
10999 "several open pull requests exist for this branch, not picking one: {}",
11000 urls.join(" ")
11001 )),
11002 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11003 }
11004}
11005
11006pub(crate) async fn gh_pr_create(
11008 cwd: &Path,
11009 base: &str,
11010 head: &str,
11011 title: &str,
11012 body: &str,
11013) -> Result<String> {
11014 let out = tokio::process::Command::new("gh")
11015 .args([
11016 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11017 ])
11018 .current_dir(cwd)
11019 .quiet()
11020 .stdin(std::process::Stdio::null())
11021 .output()
11022 .await
11023 .context("spawn gh")?;
11024 if out.status.success() {
11025 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11026 } else {
11027 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11028 }
11029}
11030
11031pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11040 if crate::pr_recovery::sources_retained(home, &state.id) {
11041 return Ok(Vec::new());
11042 }
11043 let repo = state.repo.clone();
11044 let wait = repository_wait(state, None);
11047 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11048 Ok(lease) => lease,
11049 Err(e) => {
11050 state.save_under(home)?;
11051 return Err(e);
11052 }
11053 };
11054 let root = state.worktree_root();
11055 let winner = state.tally.as_ref().map(|t| t.winner);
11056 let mut removed = Vec::new();
11057
11058 for i in 0..state.candidates.len() {
11059 let c = state.candidates[i].clone();
11060 let is_winner = Some(c.label) == winner;
11061 if is_winner && !drop_winner {
11062 continue;
11063 }
11064 if c.worktree.exists() {
11065 git::worktree_remove(&repo, &c.worktree).await.ok();
11066 removed.push(c.worktree.to_string_lossy().into_owned());
11067 }
11068 let handed_over = state.released_branches.contains(&c.branch);
11071 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
11072 git::branch_delete(&repo, &c.branch).await.ok();
11073 removed.push(c.branch.clone());
11074 }
11075 state.candidates[i].folded = true;
11076 }
11077
11078 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11079 let path = name.path();
11080 let keep = !drop_winner
11081 && winner.is_some_and(|w| {
11082 path.file_name()
11083 .is_some_and(|n| n == format!("cand-{w}").as_str())
11084 });
11085 if keep {
11086 continue;
11087 }
11088 git::worktree_remove(&repo, &path).await.ok();
11089 removed.push(path.to_string_lossy().into_owned());
11090 }
11091
11092 remove_if_empty(&root);
11101
11102 if state.enabled_worktree_config && drop_winner {
11103 git::release_worktree_config(&repo).await.ok();
11107 state.enabled_worktree_config = false;
11108 }
11109 state.save_under(home)?;
11110 Ok(removed)
11111}
11112
11113fn remove_if_empty(dir: &Path) {
11124 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11125 std::fs::remove_dir(dir).ok();
11126 }
11127}
11128
11129pub fn worst_open(state: &RunState) -> Option<Severity> {
11131 state
11132 .reviews
11133 .last()?
11134 .reviews
11135 .iter()
11136 .flat_map(|r| r.findings.iter())
11137 .map(|f| f.severity)
11138 .max()
11139}
11140
11141async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11145 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11146 git::acquire_worktree_config(repo).await?;
11147 state.enabled_worktree_config = true;
11148 }
11149 Ok(())
11150}
11151
11152#[cfg(test)]
11153mod tests {
11154 #[test]
11155 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11156 let leaked = vec!["chore(deps): update a crate".to_owned()];
11157 let own = "fix(daemon): apply a chosen action";
11158 assert!(should_retitle("", own, &leaked));
11159 assert!(should_retitle(
11160 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11161 own,
11162 &leaked
11163 ));
11164 assert!(should_retitle(
11165 "chore: land candidate A of run 1",
11166 own,
11167 &leaked
11168 ));
11169 assert!(should_retitle(
11170 "magi: candidate A (uncommitted work)",
11171 own,
11172 &leaked
11173 ));
11174 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11175 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11176 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11177 }
11178
11179 #[test]
11180 fn pr_merge_plan_creates_adopts_or_stops() {
11181 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11182 assert_eq!(
11183 pr_merge_plan(Ok(land::OpenPr::One {
11184 url: "u".into(),
11185 title: "t".into()
11186 })),
11187 PrPlan::Adopt {
11188 url: "u".into(),
11189 title: "t".into()
11190 }
11191 );
11192 let PrPlan::Stop(many) =
11193 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11194 else {
11195 panic!("many must stop");
11196 };
11197 assert!(many.contains('a') && many.contains('b'));
11198 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11199 panic!("a failed lookup must stop");
11200 };
11201 assert!(err.contains("bad token"));
11202 }
11203
11204 use super::*;
11205 use crate::run::GateStatus;
11206 use std::collections::BTreeMap;
11207 use std::time::Duration;
11208
11209 fn conductor() -> AgentSpec {
11210 AgentSpec {
11211 id: "conductor".to_owned(),
11212 kind: crate::config::AgentKind::Command,
11213 model: None,
11214 command: vec!["true".to_owned()],
11215 extra_args: Vec::new(),
11216 env: BTreeMap::new(),
11217 prompt_delivery: None,
11218 }
11219 }
11220
11221 fn spec(id: &str) -> AgentSpec {
11222 AgentSpec {
11223 id: id.to_owned(),
11224 kind: crate::config::AgentKind::Command,
11225 model: None,
11226 command: vec!["true".to_owned()],
11227 extra_args: Vec::new(),
11228 env: BTreeMap::new(),
11229 prompt_delivery: None,
11230 }
11231 }
11232
11233 fn ids(xs: &[&str]) -> BTreeSet<String> {
11234 xs.iter().map(|s| (*s).to_owned()).collect()
11235 }
11236
11237 #[test]
11238 fn pick_successor_skips_an_agent_another_seat_holds() {
11239 let roster = [spec("a"), spec("b"), spec("c")];
11241 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11242 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11243 }
11244
11245 #[test]
11246 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11247 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11249 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11250 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11251 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11252 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11253 }
11254
11255 #[test]
11256 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11257 let roster = [spec("a"), spec("b")];
11258 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11259 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11260 let carried = ids(&["b"]);
11261 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11262 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11263 }
11264
11265 #[test]
11266 fn pick_successor_returns_none_without_any_untried_successor() {
11267 let roster = [spec("a"), spec("b")];
11268 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11269 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11270 let carried = ids(&["a"]);
11271 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11272 }
11273
11274 #[test]
11275 fn pick_successor_rescue_avoids_another_seats_occupant() {
11276 let roster = [spec("a"), spec("b"), spec("c")];
11278 let carried = ids(&["b", "c"]);
11279 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11280 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11281 }
11282
11283 #[test]
11284 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11285 let roster = [spec("a"), spec("b"), spec("c")];
11286 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11287 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11288 }
11289
11290 #[test]
11291 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11292 let roster = [spec("a"), spec("b"), spec("c")];
11293 let failed = ids(&["a", "b", "c"]);
11294 let mut tried = ids(&["b"]);
11296 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11297 assert_eq!(first.id, "a");
11298 tried.insert(first.id.clone());
11299 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11300 assert_eq!(second.id, "c");
11301 tried.insert(second.id.clone());
11302 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11303 }
11304
11305 #[test]
11306 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11307 let roster = [spec("a"), spec("b")];
11308 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11309 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11310 }
11311
11312 #[test]
11313 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11314 let roster = [spec("a"), spec("b"), spec("c")];
11315 let h = SeatHistory {
11316 failed: ids(&["a"]),
11317 last_ok: Some("b".to_owned()),
11318 last_fail: None,
11319 };
11320 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11321 let gone = SeatHistory {
11323 last_ok: Some("zzz".to_owned()),
11324 ..h.clone()
11325 };
11326 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11327 let failed = SeatHistory {
11328 failed: ids(&["a", "b"]),
11329 last_ok: Some("b".to_owned()),
11330 last_fail: None,
11331 };
11332 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11333 }
11334
11335 #[test]
11336 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11337 let first = SeatState::new("review-1", "alpha", 7);
11338 let next = handover_seat("review-1", "gamma", 7);
11339 assert_ne!(first.claude_session, next.claude_session);
11340 }
11341
11342 #[test]
11343 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11344 let mut state = state_with_summary("x", "y");
11345 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11346 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11347 assert_ne!(a.claude_session, b.claude_session);
11348 }
11349
11350 #[test]
11351 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11352 let roster = [spec("a"), spec("b")];
11353 let h = SeatHistory {
11354 failed: ids(&["a", "b"]),
11355 ..SeatHistory::default()
11356 };
11357 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11358 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11359 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11360 }
11361
11362 #[test]
11369 fn failure_signature_ignores_numbers_and_paths() {
11370 assert_eq!(
11371 failure_signature("exited with Some(2) and no usable output"),
11372 failure_signature("exited with Some(137) and no usable output")
11373 );
11374 assert_eq!(
11375 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11376 failure_signature("cannot open /var/x.txt: denied")
11377 );
11378 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11379 }
11380
11381 #[test]
11382 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11383 let other = FailClass::Other("x".into());
11384 assert!(should_hand_over(None, &FailClass::Quota));
11385 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11386 assert!(should_hand_over(
11387 Some(&FailClass::Timeout),
11388 &FailClass::Timeout
11389 ));
11390 assert!(should_hand_over(None, &other));
11391 assert!(!should_hand_over(Some(&other), &other));
11392 assert!(should_hand_over(
11393 Some(&other),
11394 &FailClass::Other("y".into())
11395 ));
11396 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11398 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11399 }
11400
11401 #[test]
11402 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11403 let a = SeatState::new("judge-1", "alpha", 7);
11404 let b = handover_seat("judge-1", "beta", 7);
11405 assert_ne!(a.claude_session, b.claude_session);
11406 assert_eq!(b.turns, 0);
11407 }
11408
11409 #[test]
11410 fn a_timeout_is_classified_apart_from_other_failures() {
11411 assert_eq!(
11412 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11413 Some(FailClass::Timeout)
11414 );
11415 assert!(matches!(
11416 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11417 Some(FailClass::Other(_))
11418 ));
11419 }
11420
11421 #[test]
11422 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11423 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11424 let tried = BTreeSet::from(["beta".to_owned()]);
11425 let next = next_untried_in_roster(&roster, 1, &tried);
11428 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11429 }
11430
11431 #[test]
11432 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11433 let roster = vec![spec("alpha"), spec("beta")];
11434 let tried = BTreeSet::from(["beta".to_owned()]);
11435 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11439 }
11440
11441 #[test]
11442 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11443 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11444 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11445 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11449 }
11450
11451 #[test]
11452 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11453 let roster = vec![spec("a"), spec("a"), spec("b")];
11454 let tried = BTreeSet::from(["a".to_owned()]);
11455 let next = next_untried_in_roster(&roster, 0, &tried);
11456 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11457 }
11458
11459 #[test]
11460 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11461 let roster = vec![spec("a"), spec("b")];
11462 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11463 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11464 }
11465
11466 #[test]
11467 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11468 let dir = tempfile::tempdir().unwrap();
11469 let bay = dir.path().join("ffff");
11470
11471 remove_if_empty(&bay);
11473 assert!(!bay.exists());
11474
11475 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11478 remove_if_empty(&bay);
11479 assert!(bay.exists(), "non-empty directory must survive");
11480
11481 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11483 remove_if_empty(&bay);
11484 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11485 }
11486
11487 #[test]
11496 fn a_full_panel_that_found_nothing_is_clean() {
11497 assert!(round_is_clean(
11498 0,
11499 true,
11500 2,
11501 2,
11502 0,
11503 IncompleteReviewPolicy::Block
11504 ));
11505 }
11506
11507 #[test]
11508 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11509 assert!(!round_is_clean(
11510 0,
11511 true,
11512 1,
11513 2,
11514 0,
11515 IncompleteReviewPolicy::Block
11516 ));
11517 }
11518
11519 #[test]
11520 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11521 assert!(!round_is_clean(
11522 1,
11523 true,
11524 1,
11525 2,
11526 0,
11527 IncompleteReviewPolicy::Warn
11528 ));
11529 }
11530
11531 #[test]
11532 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11533 assert!(round_is_clean(
11534 0,
11535 true,
11536 1,
11537 2,
11538 0,
11539 IncompleteReviewPolicy::Warn
11540 ));
11541 }
11542
11543 #[test]
11544 fn a_full_panel_with_an_open_finding_is_not_clean() {
11545 assert!(!round_is_clean(
11546 1,
11547 true,
11548 2,
11549 2,
11550 0,
11551 IncompleteReviewPolicy::Block
11552 ));
11553 }
11554
11555 #[test]
11556 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11557 assert!(!round_is_clean(
11558 0,
11559 false,
11560 2,
11561 2,
11562 0,
11563 IncompleteReviewPolicy::Block
11564 ));
11565 }
11566
11567 #[test]
11574 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11575 assert!(round_is_clean(
11578 0,
11579 true,
11580 1,
11581 2,
11582 1,
11583 IncompleteReviewPolicy::Block
11584 ));
11585 }
11586
11587 #[test]
11588 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11589 assert!(!round_is_clean(
11592 0,
11593 true,
11594 1,
11595 2,
11596 0,
11597 IncompleteReviewPolicy::Block
11598 ));
11599 }
11600
11601 #[test]
11602 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11603 assert!(!round_is_clean(
11604 1,
11605 true,
11606 1,
11607 2,
11608 1,
11609 IncompleteReviewPolicy::Block
11610 ));
11611 assert!(!round_is_clean(
11612 0,
11613 false,
11614 1,
11615 2,
11616 1,
11617 IncompleteReviewPolicy::Block
11618 ));
11619 }
11620
11621 #[test]
11622 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11623 assert!(!round_is_clean(
11627 0,
11628 true,
11629 0,
11630 2,
11631 2,
11632 IncompleteReviewPolicy::Block
11633 ));
11634 }
11635
11636 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11637 CommandOutcome {
11638 command: "test".to_owned(),
11639 code,
11640 output_tail: String::new(),
11641 duration_ms: 0,
11642 resource_blocked,
11643 }
11644 }
11645
11646 #[test]
11647 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11648 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11649 assert!(
11650 !verify_inconclusive(&[outcome(Some(1), false)]),
11651 "an ordinary failure is still evidence about the patch"
11652 );
11653 assert!(verify_inconclusive(&[outcome(None, true)]));
11654 assert!(
11655 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11656 "one inconclusive outcome taints the whole batch"
11657 );
11658 assert!(!verify_inconclusive(&[]));
11659 }
11660
11661 #[tokio::test]
11662 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11663 let calls = std::sync::atomic::AtomicUsize::new(0);
11667 let started = Instant::now();
11668 wait_for_pids_with(
11669 &[123],
11670 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11671 Duration::from_millis(5),
11672 Duration::from_secs(5),
11673 )
11674 .await;
11675 assert!(
11676 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11677 "must keep checking rather than deciding on the first answer"
11678 );
11679 assert!(
11680 started.elapsed() < Duration::from_secs(1),
11681 "must return the moment it is confirmed dead, not wait out the ceiling"
11682 );
11683 }
11684
11685 #[tokio::test]
11686 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11687 let started = Instant::now();
11688 wait_for_pids_with(
11689 &[123],
11690 |_| true, Duration::from_millis(5),
11692 Duration::from_millis(30),
11693 )
11694 .await;
11695 let elapsed = started.elapsed();
11696 assert!(
11697 elapsed >= Duration::from_millis(30),
11698 "must not give up before its own ceiling: {elapsed:?}"
11699 );
11700 assert!(
11701 elapsed < Duration::from_secs(1),
11702 "must not wait past its own ceiling either: {elapsed:?}"
11703 );
11704 }
11705
11706 #[tokio::test]
11707 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
11708 let started = Instant::now();
11709 wait_for_pids_with(
11710 &[],
11711 |_| true,
11712 Duration::from_secs(5),
11713 Duration::from_secs(5),
11714 )
11715 .await;
11716 assert!(
11717 started.elapsed() < Duration::from_millis(200),
11718 "an empty pid list has nothing to confirm"
11719 );
11720 }
11721
11722 fn review_round(
11728 clean: bool,
11729 blocking: usize,
11730 answered: usize,
11731 expected: usize,
11732 progressed: bool,
11733 e2e_ok: bool,
11734 ) -> ReviewRound {
11735 ReviewRound {
11736 round: 1,
11737 head: "h".to_owned(),
11738 verified_head: None,
11739 verified_at: None,
11740 reviews: Vec::new(),
11741 e2e: vec![CommandOutcome {
11742 command: "test".to_owned(),
11743 code: Some(if e2e_ok { 0 } else { 1 }),
11744 output_tail: String::new(),
11745 duration_ms: 0,
11746 resource_blocked: false,
11747 }],
11748 verify_retried: false,
11749 e2e_deferred: false,
11750 e2e_defer_reason: None,
11751 fix: None,
11752 blocking,
11753 answered,
11754 expected,
11755 clean,
11756 progressed,
11757 vote_split: false,
11758 reconsideration: Vec::new(),
11759 verdict: None,
11760 }
11761 }
11762
11763 #[test]
11764 fn review_conclusion_is_none_when_nothing_has_run() {
11765 assert_eq!(review_conclusion(&[], 3), None);
11766 }
11767
11768 #[test]
11769 fn review_conclusion_is_none_while_rounds_remain() {
11770 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
11771 assert_eq!(review_conclusion(&rounds, 3), None);
11772 }
11773
11774 #[test]
11775 fn review_conclusion_is_gating_once_a_round_is_clean() {
11776 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
11777 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
11778 }
11779
11780 #[test]
11781 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
11782 let rounds = vec![
11783 review_round(false, 1, 2, 2, true, true),
11784 review_round(false, 1, 2, 2, true, true),
11785 ];
11786 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
11787 }
11788
11789 #[test]
11790 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
11791 let rounds = vec![
11792 review_round(false, 1, 2, 2, true, true),
11793 review_round(false, 1, 2, 2, true, false),
11794 ];
11795 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
11796 }
11797
11798 #[test]
11799 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
11800 let mut blocked = review_round(false, 1, 2, 2, true, false);
11807 blocked.e2e[0].resource_blocked = true;
11808 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
11809 assert_eq!(review_conclusion(&rounds, 2), None);
11810 }
11811
11812 #[test]
11813 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
11814 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
11816 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
11817 }
11818
11819 #[test]
11820 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
11821 let rounds = vec![
11822 review_round(false, 1, 2, 2, false, true),
11823 review_round(false, 1, 2, 2, false, true),
11824 ];
11825 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
11826 }
11827
11828 fn secs(n: u64) -> Duration {
11829 Duration::from_secs(n)
11830 }
11831
11832 #[tokio::test]
11835 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
11836 ask_test_home();
11837 let tmp = tempfile::tempdir().unwrap();
11838 let repo = tmp.path().join("repo");
11839 std::fs::create_dir_all(&repo).unwrap();
11840 init_repo(&repo);
11841 let g = |args: &[&str]| {
11842 let out = std::process::Command::new("git")
11843 .args(args)
11844 .current_dir(&repo)
11845 .quiet()
11846 .output()
11847 .unwrap();
11848 assert!(
11849 out.status.success(),
11850 "{}",
11851 String::from_utf8_lossy(&out.stderr)
11852 );
11853 };
11854 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
11855 g(&["checkout", "-b", "historical"]);
11856 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
11857 g(&["add", "."]);
11858 g(&["commit", "-m", "old alternative"]);
11859 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
11860 g(&["checkout", "-b", "magi/task-selected", &base]);
11861 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
11862 g(&["add", "."]);
11863 g(&["commit", "-m", "measurement infrastructure"]);
11864 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
11865 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
11866 g(&["add", "."]);
11867 g(&["commit", "-m", "dependent implementation"]);
11868 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11869 g(&["checkout", "main"]);
11870 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
11871 g(&["add", "."]);
11872 g(&["commit", "-m", "unrelated base change"]);
11873 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
11874 let origin = tmp.path().join("origin.git");
11875 g(&[
11876 "clone",
11877 "--bare",
11878 repo.to_str().unwrap(),
11879 origin.to_str().unwrap(),
11880 ]);
11881 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
11882 g(&["fetch", "origin"]);
11883 let selection = crate::pr_recovery::Selection {
11884 url: "https://example.test/project/pull/1".into(),
11885 branch: "magi/task-selected".into(),
11886 head: head.clone(),
11887 instruction: "preserve measured behavior".into(),
11888 approved_at: jiff::Timestamp::now(),
11889 };
11890 let mut template = runner_at(RunStatus::Reviewing);
11891 template.state.repo = repo.clone();
11892 template.state.base_commit = current.clone();
11893 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
11894 template.state.origin = Some(crate::run::Origin::queue("selected"));
11895 template.state.pr_head = Some(selection.branch.clone());
11896 template.state.pr_recovery = Some(selection.clone());
11897 let mut runner =
11898 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
11899 .await
11900 .unwrap();
11901 let wt = runner.state.winner().unwrap().worktree.clone();
11902 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
11903 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
11904 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
11905 assert!(runner.state.seeds.is_empty());
11906 assert_eq!(runner.state.instruction, "preserve measured behavior");
11907 runner.sync_to_base().await.unwrap();
11908 assert_eq!(
11909 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
11910 "prerequisite"
11911 );
11912 assert_eq!(
11913 std::fs::read_to_string(wt.join("implementation")).unwrap(),
11914 "measured implementation"
11915 );
11916 assert_eq!(
11917 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
11918 "current main change"
11919 );
11920 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
11921 assert_eq!(
11922 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
11923 head
11924 );
11925
11926 let mut partial = runner_at(RunStatus::Reviewing);
11928 partial.state.repo = repo.clone();
11929 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
11930 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
11931 run: Some("exit 1".into()),
11932 ..Default::default()
11933 }];
11934 let selection = runner.state.pr_recovery.clone().unwrap();
11935 partial.state.pr_recovery = Some(selection.clone());
11936 partial.state.pr_head = Some(selection.branch.clone());
11937 partial.state.origin = Some(crate::run::Origin::queue("selected"));
11938 let id = partial.state.id.clone();
11939 let error = match Runner::open_selected(
11940 &repo,
11941 partial.state,
11942 partial.roles,
11943 1,
11944 runner.state.base_commit.clone(),
11945 selection,
11946 )
11947 .await
11948 {
11949 Ok(_) => panic!("setup must fail"),
11950 Err(e) => e,
11951 };
11952 assert_eq!(
11953 error
11954 .downcast_ref::<crate::pr_recovery::StartupFailed>()
11955 .unwrap()
11956 .run,
11957 id
11958 );
11959 let failed = RunState::load(&id).unwrap();
11960 assert_eq!(failed.status, RunStatus::Blocked);
11961 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
11962 assert!(Runner::resume(&id).is_err());
11963 runner.state.status = RunStatus::Ready;
11964 runner.state.driver_exited = true;
11965 runner.state.save().unwrap();
11966 let mut retry = runner_at(RunStatus::Reviewing);
11967 retry.state.repo = repo.clone();
11968 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
11969 let mut reselected = runner.state.pr_recovery.clone().unwrap();
11970 reselected.approved_at = jiff::Timestamp::now();
11971 retry.state.origin = Some(crate::run::Origin::queue("selected"));
11972 retry.state.pr_head = Some(reselected.branch.clone());
11973 retry.state.pr_recovery = Some(reselected.clone());
11974 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
11975 .await
11976 .unwrap();
11977 let restarted = Runner::open_selected(
11978 &repo,
11979 retry.state,
11980 retry.roles,
11981 1,
11982 runner.state.base_commit.clone(),
11983 reselected,
11984 )
11985 .await
11986 .unwrap();
11987 assert_eq!(
11988 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
11989 .await
11990 .unwrap(),
11991 head
11992 );
11993 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
11994 assert_eq!(
11995 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
11996 head
11997 );
11998 }
11999
12000 #[tokio::test]
12001 #[cfg(unix)]
12002 async fn fallback_and_interrupted_restoration_require_coordination() {
12003 use std::os::fd::AsRawFd as _;
12004 ask_test_home();
12005 let tmp = tempfile::tempdir().unwrap();
12006 let mut runner = registered_runner(tmp.path()).await;
12007 runner.state.config.graph.timeout_resource_wait = 1;
12008 let winner = runner.state.candidates[0].clone();
12009 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12010 let leftover = winner.worktree.join("leftover");
12011 std::fs::write(&leftover, "untrusted edits").unwrap();
12012 let next = tmp.path().join("next-fixer");
12013 let mut failed = spec("first");
12014 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12015 let mut successor = spec("second");
12016 successor.command = vec![
12017 "sh".into(),
12018 "-c".into(),
12019 format!("touch '{}'; echo done", next.display()),
12020 ];
12021 let attempts = vec![
12022 (failed.clone(), "first".into()),
12023 (successor.clone(), "second".into()),
12024 ];
12025 let lock_path = runner.state.repo.join(".git/kanade-sync.lock");
12026 let external = std::fs::File::create(&lock_path).unwrap();
12027 assert_eq!(
12028 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12029 0
12030 );
12031 let artifacts = tmp.path().join("artifacts");
12032 let build = |spec, seat| SeatJob {
12033 spec,
12034 seat,
12035 cwd: winner.worktree.clone(),
12036 prompt: "fixture".into(),
12037 timeout: Duration::from_secs(10),
12038 allow_write: true,
12039 sessions: false,
12040 artifacts: artifacts.clone(),
12041 stem: "fix".into(),
12042 handover: None,
12043 };
12044 let (_, _, out) = runner
12045 .ask_fixer_among(
12046 attempts.clone(),
12047 "graft",
12048 None,
12049 Some((&winner.worktree, &head)),
12050 build,
12051 )
12052 .await;
12053 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12054 assert!(leftover.exists(), "contention must prevent clean");
12055 assert!(!next.exists());
12056 assert!(runner.state.handovers.is_empty());
12057 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12059 from_run: "earlier".into(),
12060 generation: 1,
12061 mode: crate::recovery::Mode::VerifyRepair,
12062 reasons: Vec::new(),
12063 reviewed_head: None,
12064 rounds: Vec::new(),
12065 open_findings: Vec::new(),
12066 evidence: None,
12067 criteria_inherited: true,
12068 repair: crate::recovery::Repair::Started {
12069 before: head.clone(),
12070 },
12071 });
12072 assert!(
12073 runner
12074 .recovery_repair(&winner)
12075 .await
12076 .unwrap_err()
12077 .is::<crate::repo_lock::ResourceBlocked>()
12078 );
12079 assert!(matches!(
12080 runner.state.recovery.as_ref().unwrap().repair,
12081 crate::recovery::Repair::Started { .. }
12082 ));
12083 assert!(leftover.exists());
12084 drop(external);
12085 runner.recovery_repair(&winner).await.unwrap();
12086 assert!(!leftover.exists());
12087 assert!(matches!(
12088 runner.state.recovery.as_ref().unwrap().repair,
12089 crate::recovery::Repair::Done { .. }
12090 ));
12091 std::fs::remove_file(&lock_path).unwrap();
12093 std::fs::create_dir(&lock_path).unwrap();
12094 std::fs::write(&leftover, "untrusted edits").unwrap();
12095 let (_, _, out) = runner
12096 .ask_fixer_among(
12097 attempts,
12098 "graft",
12099 None,
12100 Some((&winner.worktree, &head)),
12101 build,
12102 )
12103 .await;
12104 assert!(matches!(out, AgentOutcome::Failed(_)));
12105 assert!(leftover.exists());
12106 assert!(!next.exists());
12107 assert_eq!(
12108 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12109 head
12110 );
12111 }
12112
12113 #[cfg(unix)]
12114 async fn registered_runner(root: &Path) -> Runner {
12115 ask_test_home();
12116 let repo = root.join("repo");
12117 std::fs::create_dir_all(&repo).unwrap();
12118 init_repo(&repo);
12119 std::fs::write(repo.join(".git/kanade-sync.contract"), "shared-lock-v1\n").unwrap();
12120 let mut runner = runner_at(RunStatus::Prep);
12121 runner.state.repo = repo.clone();
12122 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12123 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12124 runner.state.config.graph.advise = false;
12125 runner.roles.implementers = vec![spec("implement")];
12126 runner.roles.judges = vec![spec("judge")];
12127 runner.prep().await.unwrap();
12128 runner
12129 }
12130
12131 #[tokio::test]
12132 #[cfg(unix)]
12133 async fn parked_implementer_persists_completed_work() {
12134 for registered in [false, true] {
12135 let tmp = tempfile::tempdir().unwrap();
12136 let mut runner = registered_runner(tmp.path()).await;
12137 if !registered {
12138 std::fs::remove_file(runner.state.repo.join(".git/kanade-sync.contract")).unwrap();
12139 }
12140 let marker = tmp.path().join("started");
12141 let release = tmp.path().join("release");
12142 let mut implementer = spec("implement");
12143 implementer.command = vec![
12144 "sh".into(),
12145 "-c".into(),
12146 format!(
12147 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12148 marker.display(),
12149 release.display()
12150 ),
12151 ];
12152 runner.roles.implementers = vec![implementer.clone()];
12153 runner.state.config.agents = vec![implementer];
12154 let pause = runner.pause.clone();
12155 let (result, ()) = tokio::join!(runner.implement(), async {
12156 tokio::time::timeout(Duration::from_secs(10), async {
12157 while !marker.exists() {
12158 tokio::time::sleep(Duration::from_millis(10)).await;
12159 }
12160 })
12161 .await
12162 .unwrap();
12163 pause.park();
12164 std::fs::write(&release, "go").unwrap();
12165 });
12166 result.unwrap();
12167 let candidate = &runner.state.candidates[0];
12168 assert!(candidate.commits > 0, "{candidate:?}");
12169 assert!(candidate.summary.contains("saved-result"));
12170 assert_eq!(candidate.files, 1);
12171 let before = candidate.commits;
12172 runner.implement().await.unwrap();
12173 assert_eq!(runner.state.candidates[0].commits, before);
12174 }
12175 }
12176
12177 #[tokio::test]
12178 #[cfg(unix)]
12179 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12180 let tmp = tempfile::tempdir().unwrap();
12181 let mut runner = registered_runner(tmp.path()).await;
12182 let winner = runner.state.candidates[0].clone();
12183 let cache = tmp.path().join("cache");
12184 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12185 runner.state.config.graph.timeout_resource_wait = 0;
12186 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12187 let crate::cache::AcquireOutcome::Acquired(held) =
12188 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12189 else {
12190 panic!("cache must be free")
12191 };
12192 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12193 from_run: "earlier".into(),
12194 generation: 1,
12195 mode: crate::recovery::Mode::VerifyRepair,
12196 reasons: Vec::new(),
12197 reviewed_head: None,
12198 rounds: Vec::new(),
12199 open_findings: Vec::new(),
12200 evidence: None,
12201 criteria_inherited: true,
12202 repair: crate::recovery::Repair::NotStarted,
12203 });
12204 runner.recovery_repair(&winner).await.unwrap();
12205 assert!(matches!(
12206 runner.state.recovery.as_ref().unwrap().repair,
12207 crate::recovery::Repair::WaitingForRestore { .. }
12208 ));
12209 drop(held);
12210 runner.state.resource_block = None;
12211 runner.state.config.graph.timeout_resource_wait = 1;
12212 runner.recovery_repair(&winner).await.unwrap();
12213 assert!(matches!(
12214 runner.state.recovery.as_ref().unwrap().repair,
12215 crate::recovery::Repair::Done { .. }
12216 ));
12217 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12218 }
12219
12220 #[tokio::test]
12221 #[cfg(unix)]
12222 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12223 use std::os::fd::AsRawFd as _;
12224 for graft in [false, true] {
12225 let tmp = tempfile::tempdir().unwrap();
12226 let mut runner = registered_runner(tmp.path()).await;
12227 runner.state.config.graph.timeout_resource_wait = 1;
12228 let winner = runner.state.candidates[0].clone();
12229 let marker = tmp.path().join("fixer-started");
12230 let mut fixer = spec("fixer");
12231 fixer.command = vec![
12232 "sh".into(),
12233 "-c".into(),
12234 format!("echo started >> '{}'; echo done", marker.display()),
12235 ];
12236 runner.roles.fixer = Some(vec![fixer]);
12237 if graft {
12238 let mut loser = winner.clone();
12239 loser.label = 'B';
12240 loser.index = 1;
12241 runner.state.candidates.push(loser);
12242 runner.state.config.graph.graft_runner_up = true;
12243 runner.state.tally = Some(Tally {
12244 first_choice: BTreeMap::new(),
12245 borda: BTreeMap::new(),
12246 winner: winner.label,
12247 rankings: 1,
12248 unanimous_initial: true,
12249 deliberated: false,
12250 changed_votes: 0,
12251 unanimous_final: true,
12252 tie_break: None,
12253 judges: 1,
12254 present: 1,
12255 quorum: 1,
12256 met_quorum: true,
12257 uncontested: None,
12258 });
12259 runner.state.judgements.push(crate::run::Judgement {
12260 judge: 1,
12261 seat: "judge-1".into(),
12262 agent: "judge".into(),
12263 ranking: vec![winner.label, 'B'],
12264 reasons: BTreeMap::new(),
12265 confidence: None,
12266 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12267 order: vec![0, 1],
12268 failed: None,
12269 duration_ms: 0,
12270 });
12271 } else {
12272 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12273 from_run: "earlier".into(),
12274 generation: 1,
12275 mode: crate::recovery::Mode::VerifyRepair,
12276 reasons: Vec::new(),
12277 reviewed_head: None,
12278 rounds: Vec::new(),
12279 open_findings: Vec::new(),
12280 evidence: None,
12281 criteria_inherited: true,
12282 repair: crate::recovery::Repair::NotStarted,
12283 });
12284 }
12285 let reader = crate::repo_lock::SeatLease::for_checkout(&winner.worktree, false)
12286 .await
12287 .unwrap();
12288 let pause = runner.pause.clone();
12289 let (result, ()) = tokio::join!(
12290 async {
12291 if graft {
12292 runner.graft().await
12293 } else {
12294 runner.recovery_repair(&winner).await
12295 }
12296 },
12297 async {
12298 tokio::time::sleep(Duration::from_millis(100)).await;
12299 pause.park();
12300 }
12301 );
12302 result.unwrap();
12303 assert!(!marker.exists(), "a blocked seat must not spawn");
12304 runner.state = RunState::load(&runner.state.id).unwrap();
12306 if graft {
12307 assert_eq!(
12308 runner.state.graft.as_ref().unwrap().outcome,
12309 GraftOutcome::WaitingForRestore
12310 );
12311 } else {
12312 assert!(matches!(
12313 runner.state.recovery.as_ref().unwrap().repair,
12314 crate::recovery::Repair::WaitingForRestore { .. }
12315 ));
12316 }
12317 drop(reader);
12318 runner.pause = Pause::new();
12319 runner.state.resource_block = None;
12320 let external =
12322 std::fs::File::create(runner.state.repo.join(".git/kanade-sync.lock")).unwrap();
12323 assert_eq!(
12324 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12325 0
12326 );
12327 let leftover = winner.worktree.join("leftover");
12328 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12329 let result = if graft {
12330 runner.graft().await
12331 } else {
12332 runner.recovery_repair(&winner).await
12333 };
12334 assert!(
12335 result
12336 .unwrap_err()
12337 .is::<crate::repo_lock::ResourceBlocked>()
12338 );
12339 assert!(leftover.exists());
12340 assert!(!marker.exists());
12341 drop(external);
12342 runner.state.resource_block = None;
12343 let result = if graft {
12344 runner.graft().await
12345 } else {
12346 runner.recovery_repair(&winner).await
12347 };
12348 result.unwrap();
12349 assert!(!leftover.exists(), "restore must precede retry");
12350 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12351 if graft {
12352 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12353 } else {
12354 assert!(matches!(
12355 runner.state.recovery.as_ref().unwrap().repair,
12356 crate::recovery::Repair::Done { .. }
12357 ));
12358 }
12359 }
12360 }
12361
12362 #[tokio::test]
12363 #[cfg(unix)]
12364 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12365 use std::os::fd::AsRawFd as _;
12366 let tmp = tempfile::tempdir().unwrap();
12367 let mut runner = registered_runner(tmp.path()).await;
12368 runner.state.config.graph.timeout_resource_wait = 1;
12369 let winner = runner.state.candidates[0].clone();
12370 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12371 runner.state.tally = Some(Tally {
12372 first_choice: BTreeMap::new(),
12373 borda: BTreeMap::new(),
12374 winner: winner.label,
12375 rankings: 0,
12376 unanimous_initial: false,
12377 deliberated: false,
12378 changed_votes: 0,
12379 unanimous_final: false,
12380 tie_break: None,
12381 judges: 0,
12382 present: 0,
12383 quorum: 0,
12384 met_quorum: true,
12385 uncontested: Some("fixture".into()),
12386 });
12387 runner.state.graft = Some(GraftRecord {
12388 pre: pre.clone(),
12389 post: None,
12390 lines: None,
12391 limit: None,
12392 agent: None,
12393 outcome: GraftOutcome::Running,
12394 requested: Vec::new(),
12395 adoption: None,
12396 adoption_error: None,
12397 });
12398 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12399 git::commit_all(&winner.worktree, "unfinished graft")
12400 .await
12401 .unwrap();
12402 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12403 let leftover = winner.worktree.join("leftover");
12404 std::fs::write(&leftover, "untrusted edits").unwrap();
12405 let path = runner.state.repo.join(".git/kanade-sync.lock");
12406 let external = std::fs::File::create(&path).unwrap();
12407 assert_eq!(
12408 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12409 0
12410 );
12411 assert!(
12412 runner
12413 .graft()
12414 .await
12415 .unwrap_err()
12416 .is::<crate::repo_lock::ResourceBlocked>()
12417 );
12418 assert!(matches!(
12419 runner.state.graft.as_ref().unwrap().outcome,
12420 GraftOutcome::Running
12421 ));
12422 assert_eq!(
12423 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12424 unfinished
12425 );
12426 assert!(leftover.exists());
12427 drop(external);
12428 std::fs::remove_file(&path).unwrap();
12429 std::fs::create_dir(&path).unwrap();
12430 assert!(runner.graft().await.is_err());
12431 assert!(matches!(
12432 runner.state.graft.as_ref().unwrap().outcome,
12433 GraftOutcome::Running
12434 ));
12435 assert_eq!(
12436 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12437 unfinished
12438 );
12439 assert!(leftover.exists());
12440 std::fs::remove_dir(&path).unwrap();
12441 runner.graft().await.unwrap();
12442 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12443 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12444 assert!(!leftover.exists());
12445 }
12446
12447 #[tokio::test]
12448 #[cfg(unix)]
12449 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12450 use std::os::fd::AsRawFd as _;
12451 let tmp = tempfile::tempdir().unwrap();
12452 let mut runner = registered_runner(tmp.path()).await;
12453 runner.state.prep_complete = Some(false);
12454 runner.state.config.graph.timeout_resource_wait = 1;
12455 runner.state.save().unwrap();
12456 let external =
12457 std::fs::File::create(runner.state.repo.join(".git/kanade-sync.lock")).unwrap();
12458 assert_eq!(
12459 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12460 0
12461 );
12462 runner.execute().await.unwrap();
12463 assert_eq!(runner.state.status, RunStatus::Blocked);
12464 assert_eq!(
12465 runner.state.resource_block.as_ref().unwrap().kind,
12466 "repository coordination"
12467 );
12468 assert!(runner.state.prep_failure.is_none());
12469 assert!(runner.state.seats.is_empty());
12470 runner.pause.park();
12471 tokio::time::timeout(Duration::from_secs(1), runner.execute())
12472 .await
12473 .unwrap()
12474 .unwrap();
12475 assert!(runner.state.parked);
12476 assert!(runner.state.prep_failure.is_none());
12477 }
12478
12479 #[tokio::test]
12480 #[cfg(unix)]
12481 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12482 let tmp = tempfile::tempdir().unwrap();
12483 let mut runner = registered_runner(tmp.path()).await;
12484 let winner = runner.state.candidates[0].clone();
12485 let cache = tmp.path().join("cache");
12486 let started = tmp.path().join("pre-gate-started");
12487 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12488 runner.state.config.verify.gate =
12489 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12490 let home = crate::run::home();
12491 let owner =
12492 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12493 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12494 crate::cache::AcquireOutcome::Acquired(g) => g,
12495 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12496 };
12497 let mut gate = Box::pin(runner.run_gate(&winner));
12498 assert!(
12499 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12500 .await
12501 .is_err()
12502 );
12503 assert!(!started.exists(), "pre-gate must wait for the cache too");
12504 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12508 let editor = crate::repo_lock::SeatLease::checkout_with(&winner.worktree, true, &wait)
12509 .await
12510 .unwrap();
12511 drop(editor);
12512 guard.release();
12513 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12514 .await
12515 .unwrap()
12516 .unwrap();
12517 drop(gate);
12518 assert!(outcomes.iter().all(CommandOutcome::ok));
12519 assert!(started.exists());
12520 }
12521
12522 #[tokio::test]
12523 #[cfg(unix)]
12524 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12525 use std::os::fd::AsRawFd as _;
12526
12527 ask_test_home();
12528 let tmp = tempfile::tempdir().unwrap();
12529 let repo = tmp.path().join("repo");
12530 std::fs::create_dir_all(&repo).unwrap();
12531 init_repo(&repo);
12532 let common = repo.join(".git");
12533 std::fs::write(common.join("kanade-sync.contract"), "shared-lock-v1\n").unwrap();
12534 let mut runner = runner_at(RunStatus::Prep);
12535 runner.state.repo = repo.clone();
12536 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12537 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12538 runner.state.config.graph.advise = false;
12539 runner.roles.implementers = vec![spec("implement")];
12540 runner.roles.judges = vec![spec("judge")];
12541 runner.prep().await.unwrap();
12542 let winner = runner.state.candidates[0].clone();
12543 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12546 let completed = tmp.path().join("seat-completed");
12547 runner.roles.implementers[0].command = vec![
12548 "sh".into(),
12549 "-c".into(),
12550 format!("touch '{}'; echo completed", completed.display()),
12551 ];
12552 let outsider = std::fs::File::create(common.join("kanade-sync.lock")).unwrap();
12553 assert_eq!(
12554 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12555 0
12556 );
12557 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12558 let mut implement = Box::pin(runner.implement());
12559 tokio::select! {
12560 result = &mut implement => panic!("rescue must wait: {result:?}"),
12561 result = tokio::time::timeout(Duration::from_secs(10), async {
12562 while !completed.exists() {
12563 tokio::time::sleep(Duration::from_millis(20)).await;
12564 }
12565 }) => result.unwrap(),
12566 }
12567 assert!(
12568 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12569 .await
12570 .is_err()
12571 );
12572 assert_eq!(
12573 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12574 before
12575 );
12576 assert_eq!(
12577 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12578 0
12579 );
12580 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12581 .await
12582 .unwrap()
12583 .unwrap();
12584 drop(implement);
12585 assert_ne!(
12586 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12587 before
12588 );
12589 assert!(git::is_clean(&winner.worktree).await.unwrap());
12590
12591 let started = tmp.path().join("pre-gate-started");
12592 let proceed = tmp.path().join("pre-gate-proceed");
12593 let checked = tmp.path().join("gate-ran");
12594 runner.state.config.verify.pre_gate = vec![format!(
12595 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12596 started.display(),
12597 proceed.display()
12598 )];
12599 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12600 assert_eq!(
12601 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12602 0
12603 );
12604 let mut gate = Box::pin(runner.run_gate(&winner));
12605 assert!(
12606 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12607 .await
12608 .is_err()
12609 );
12610 assert!(!started.exists());
12611 assert!(!checked.exists());
12612 assert!(git::is_clean(&winner.worktree).await.unwrap());
12613 assert_eq!(
12614 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12615 0
12616 );
12617 tokio::select! {
12618 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12619 result = tokio::time::timeout(Duration::from_secs(10), async {
12620 while !started.exists() {
12621 tokio::time::sleep(Duration::from_millis(20)).await;
12622 }
12623 }) => result.unwrap(),
12624 }
12625 assert_eq!(
12626 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12627 -1,
12628 "mechanical edits must hold the real lock"
12629 );
12630 std::fs::write(proceed, "ready").unwrap();
12631 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12632 .await
12633 .unwrap()
12634 .unwrap();
12635 drop(gate);
12636 assert!(outcomes.iter().all(CommandOutcome::ok));
12637 assert!(checked.exists());
12638 assert!(runner.state.pre_gate_commit.is_some());
12639 assert!(git::is_clean(&winner.worktree).await.unwrap());
12640 assert_eq!(
12641 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12642 0,
12643 "gate completion must release coordination"
12644 );
12645 drop(outsider);
12646 std::fs::remove_file(common.join("kanade-sync.lock")).unwrap();
12647 std::fs::create_dir(common.join("kanade-sync.lock")).unwrap();
12648 std::fs::remove_file(&started).unwrap();
12649 std::fs::remove_file(&checked).unwrap();
12650 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12651 let error = runner.run_gate(&winner).await.unwrap_err();
12652 assert!(
12653 error
12654 .to_string()
12655 .contains("acquire mandatory repository lock")
12656 );
12657 assert!(
12658 !started.exists(),
12659 "failed acquisition must prevent pre-gate"
12660 );
12661 assert!(!checked.exists(), "failed acquisition must prevent gate");
12662 assert!(git::is_clean(&winner.worktree).await.unwrap());
12663 assert_eq!(
12664 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12665 head
12666 );
12667 }
12668
12669 #[tokio::test]
12670 #[cfg(unix)]
12671 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
12672 use std::os::fd::AsRawFd as _;
12673 use std::time::Duration;
12674
12675 ask_test_home();
12676 let tmp = tempfile::tempdir().unwrap();
12677 let repo = tmp.path().join("repo");
12678 std::fs::create_dir_all(&repo).unwrap();
12679 init_repo(&repo);
12680 let common = repo.join(".git");
12681 std::fs::write(common.join("kanade-sync.contract"), "shared-lock-v1\n").unwrap();
12682 let lock = std::fs::File::create(common.join("kanade-sync.lock")).unwrap();
12683 assert_eq!(
12684 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12685 0
12686 );
12687 let mut runner = runner_at(RunStatus::Prep);
12688 runner.state.repo = repo.clone();
12689 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12690 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12691 runner.state.config.graph.advise = true;
12692 runner.state.config.graph.advisors = 1;
12693 runner.roles.implementers = vec![spec("implement")];
12694 runner.roles.judges = vec![spec("judge")];
12695 let root = runner.state.worktree_root();
12696 let started = tmp.path().join("setup-started");
12697 let proceed = tmp.path().join("setup-proceed");
12698 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
12699 run: Some(format!(
12700 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12701 started.display(),
12702 proceed.display()
12703 )),
12704 ..Default::default()
12705 }];
12706 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
12707 let config = git::git(&repo, &["config", "--local", "--list"])
12708 .await
12709 .unwrap();
12710 let mut prep = Box::pin(runner.prep());
12711 assert!(
12712 tokio::time::timeout(Duration::from_millis(200), &mut prep)
12713 .await
12714 .is_err()
12715 );
12716 assert!(
12717 !root.exists(),
12718 "contention must prevent even checkout creation"
12719 );
12720 assert!(!started.exists(), "contention must prevent setup");
12721 assert_eq!(
12722 git::git(&repo, &["branch", "--list"]).await.unwrap(),
12723 branches
12724 );
12725 assert_eq!(
12726 git::git(&repo, &["config", "--local", "--list"])
12727 .await
12728 .unwrap(),
12729 config
12730 );
12731 drop(lock);
12732 tokio::select! {
12733 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
12734 result = tokio::time::timeout(Duration::from_secs(10), async {
12735 while !started.exists() {
12736 tokio::time::sleep(Duration::from_millis(20)).await;
12737 }
12738 }) => result.unwrap(),
12739 }
12740 let outsider = std::fs::File::open(common.join("kanade-sync.lock")).unwrap();
12741 assert_eq!(
12742 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12743 -1
12744 );
12745 assert!(
12746 tokio::time::timeout(
12747 Duration::from_millis(100),
12748 crate::repo_lock::SeatLease::for_checkout(&repo, true)
12749 )
12750 .await
12751 .is_err(),
12752 "preparation must also exclude local editors"
12753 );
12754 std::fs::write(proceed, "ready").unwrap();
12755 tokio::time::timeout(Duration::from_secs(10), &mut prep)
12756 .await
12757 .unwrap()
12758 .unwrap();
12759 drop(prep);
12760 assert_eq!(runner.state.prep_complete, Some(true));
12761 assert!(root.join("judge-1").exists());
12762 assert!(root.join("advisor-1").exists());
12763 assert_eq!(
12764 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12765 0
12766 );
12767 }
12768
12769 #[tokio::test]
12770 #[cfg(unix)]
12771 async fn preparation_lock_failure_prevents_repository_changes() {
12772 ask_test_home();
12773 let tmp = tempfile::tempdir().unwrap();
12774 let repo = tmp.path().join("repo");
12775 std::fs::create_dir_all(&repo).unwrap();
12776 init_repo(&repo);
12777 std::fs::write(repo.join(".git/kanade-sync.contract"), "shared-lock-v1\n").unwrap();
12778 std::fs::create_dir(repo.join(".git/kanade-sync.lock")).unwrap();
12781 let mut runner = runner_at(RunStatus::Prep);
12782 runner.state.repo = repo.clone();
12783 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12784 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12785 runner.roles.implementers = vec![spec("implement")];
12786 let error = runner.prep().await.unwrap_err();
12787 assert!(
12788 error
12789 .to_string()
12790 .contains("acquire mandatory repository lock")
12791 );
12792 assert!(!runner.state.worktree_root().exists());
12793 assert!(runner.state.candidates.is_empty());
12794 assert!(!repo.join(".git/config.worktree").exists());
12795 assert_eq!(
12796 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
12797 "* main"
12798 );
12799 }
12800
12801 #[tokio::test]
12802 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
12803 ask_test_home();
12804 let tmp = tempfile::tempdir().unwrap();
12805 let repo = tmp.path().join("repo");
12806 std::fs::create_dir_all(&repo).unwrap();
12807 init_repo(&repo);
12808 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12809 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
12810 std::fs::write(repo.join("referenced"), "required commit").unwrap();
12811 git::git(&repo, &["add", "."]).await.unwrap();
12812 git::git(&repo, &["commit", "-m", "required seed"])
12813 .await
12814 .unwrap();
12815 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
12816 git::git(&repo, &["checkout", "main"]).await.unwrap();
12817 let mut runner = runner_at(RunStatus::Prep);
12818 runner.state.repo = repo.clone();
12819 runner.state.base_commit = base.clone();
12820 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12821 runner.state.config.graph.advise = false;
12822 runner.state.config.blind.commit_msg_hook = false;
12823 runner.state.seeds = vec![refs::Seed {
12824 token: seed.clone(),
12825 sha: seed.clone(),
12826 kind: refs::SeedKind::Unmerged,
12827 branch: false,
12828 detail: String::new(),
12829 }];
12830 runner.roles.implementers = vec![spec("one"), spec("two")];
12831 runner.roles.judges = vec![spec("judge")];
12832 let label = blind::assign_labels(2, runner.state.seed)[0];
12833 let branch = runner.state.branch_for(label);
12834 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
12835 git::worktree_add_branch(&repo, &worktree, &branch, &base)
12836 .await
12837 .unwrap();
12838 runner.state.candidates.push(Candidate {
12839 index: 0,
12840 label,
12841 agent: "one".into(),
12842 branch,
12843 worktree,
12844 summary: String::new(),
12845 stat: String::new(),
12846 files: 0,
12847 commits: 0,
12848 empty: false,
12849 failed: None,
12850 verified_noop: None,
12851 duration_ms: 0,
12852 folded: false,
12853 });
12854 runner.state.save().unwrap();
12855 runner.state = RunState::load(&runner.state.id).unwrap();
12857 assert_eq!(runner.state.prep_complete, Some(false));
12858 runner.prep().await.unwrap();
12859 assert_eq!(runner.state.prep_complete, Some(true));
12860 assert_eq!(runner.state.candidates.len(), 2);
12861 for candidate in &runner.state.candidates {
12862 assert_eq!(
12863 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
12864 git::tree_of(&repo, &seed).await.unwrap()
12865 );
12866 assert_eq!(
12867 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
12868 "required commit"
12869 );
12870 }
12871 assert!(runner.state.worktree_root().join("judge-1").exists());
12872 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
12873 runner.prep().await.unwrap();
12874 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
12875
12876 runner.state.prep_complete = Some(false);
12877 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
12878 run: Some(format!(
12879 "test -f '{}'",
12880 tmp.path().join("dependency-ready").display()
12881 )),
12882 ..Default::default()
12883 }];
12884 assert!(runner.prep().await.is_err());
12885 assert!(runner.state.prep_failure.is_some());
12886 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
12887 runner.state = RunState::load(&runner.state.id).unwrap();
12888 assert!(
12889 runner.prep().await.is_err(),
12890 "correcting the cause alone must not auto-retry"
12891 );
12892 let failed_at = runner.state.events.last().unwrap().at;
12893 assert!(
12894 runner.retry_preparation(failed_at).is_err(),
12895 "old approvals cannot unlock the hold"
12896 );
12897 let mut task = crate::queue::Task::new(
12898 "retry".into(),
12899 "retry setup".into(),
12900 repo.clone(),
12901 crate::queue::Source::Human,
12902 );
12903 task.start(runner.state.id.clone());
12904 task.hold_machine(Some("setup failed".into()));
12905 task.release_operator("cli", None);
12906 runner
12907 .retry_preparation(task.resumption.as_ref().unwrap().at)
12908 .unwrap();
12909 runner.prep().await.unwrap();
12910 assert!(runner.state.prep_failure.is_none());
12911 assert_eq!(runner.state.prep_complete, Some(true));
12912 assert_eq!(runner.state.candidates.len(), 2);
12913 assert!(runner.state.worktree_root().join("judge-1").exists());
12914 for candidate in &runner.state.candidates {
12915 assert_eq!(
12916 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
12917 git::tree_of(&repo, &seed).await.unwrap()
12918 );
12919 }
12920 }
12921
12922 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
12923 let mut heads = Vec::new();
12924 for candidate in candidates {
12925 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
12926 }
12927 heads
12928 }
12929
12930 fn init_repo(dir: &Path) {
12931 let run = |args: &[&str]| {
12932 let out = std::process::Command::new("git")
12933 .args(args)
12934 .current_dir(dir)
12935 .quiet()
12936 .output()
12937 .expect("spawn git");
12938 assert!(
12939 out.status.success(),
12940 "git {args:?} failed: {}",
12941 String::from_utf8_lossy(&out.stderr)
12942 );
12943 };
12944 run(&["init", "-b", "main"]);
12945 run(&["config", "user.name", "magi test"]);
12946 run(&["config", "user.email", "magi@example.com"]);
12947 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
12948 run(&["add", "-A"]);
12949 run(&["commit", "-m", "init"]);
12950 }
12951
12952 fn ask_test_home() {
12960 crate::run::pin_test_home();
12961 }
12962
12963 fn runner_at(status: RunStatus) -> Runner {
12966 let mut state = RunState::new(
12967 PathBuf::from("/nonexistent/repo"),
12968 "main".to_owned(),
12969 "deadbeef".to_owned(),
12970 "task".to_owned(),
12971 Config::default(),
12972 );
12973 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
12976 state.id = format!(
12977 "20000101-000000-{:04x}",
12978 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
12979 );
12980 state.status = status;
12981 Runner {
12982 state,
12983 roles: ResolvedRoles {
12984 implementers: Vec::new(),
12985 judges: Vec::new(),
12986 reviewers: Vec::new(),
12987 fixer: None,
12988 conductor: conductor(),
12989 implementer_roster: Vec::new(),
12990 judge_roster: Vec::new(),
12991 reviewer_roster: Vec::new(),
12992 },
12993 sem: Arc::new(Semaphore::new(1)),
12994 pause: Pause::new(),
12995 interrupt: Pause::new(),
12996 }
12997 }
12998
12999 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13000 crate::run::MergeRepair {
13001 question: "repair-question".into(),
13002 head: "old-head".into(),
13003 pr: "https://github.com/o/r/pull/42".into(),
13004 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13005 gate: vec![outcome(Some(0), false)],
13006 gate_head: Some("old-head".into()),
13007 gate_fixes: Vec::new(),
13008 pre_gate: Vec::new(),
13009 phase,
13010 failure: None,
13011 }
13012 }
13013
13014 #[tokio::test]
13015 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13016 crate::run::pin_test_home();
13017 let mut runner = runner_at(RunStatus::Landing);
13018 runner.state.merge_repairs.push(merge_repair_request(
13019 crate::run::MergeRepairPhase::Requested,
13020 ));
13021 runner.state.save().unwrap();
13022 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13023 assert!(
13024 runner
13025 .execute()
13026 .await
13027 .unwrap_err()
13028 .to_string()
13029 .contains("already running")
13030 );
13031 assert_eq!(
13032 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13033 crate::run::MergeRepairPhase::Requested
13034 );
13035 drop(claim);
13036 let mut disk = runner.state.clone();
13037 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13038 disk.save().unwrap();
13039 assert!(
13040 runner
13041 .execute()
13042 .await
13043 .unwrap_err()
13044 .to_string()
13045 .contains("decision changed")
13046 );
13047 assert_eq!(
13048 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13049 crate::run::MergeRepairPhase::Complete
13050 );
13051 }
13052
13053 #[test]
13054 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13055 crate::run::pin_test_home();
13056 let mut runner = runner_at(RunStatus::Landing);
13057 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13058 runner.state.reviews = request.reviews.clone();
13059 runner.state.gate = request.gate.clone();
13060 runner.state.gate_ran = true;
13061 runner.state.land_approval = Some(crate::run::LandApproval {
13062 question: "old".into(),
13063 head: "old-head".into(),
13064 });
13065 runner.state.merge_repairs.push(request);
13066 runner.begin_merge_repair_review(0).unwrap();
13067 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13068 assert!(!runner.state.gate_status().ok());
13069 assert!(runner.state.land_approval.is_none());
13070 let disk = RunState::load(&runner.state.id).unwrap();
13071 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13072 assert!(disk.merge_repairs[0].gate[0].ok());
13073 assert_eq!(
13074 disk.merge_repairs[0].phase,
13075 crate::run::MergeRepairPhase::Reviewing
13076 );
13077 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13078 assert!(disk.reviews.is_empty());
13079 }
13080
13081 #[tokio::test]
13082 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13083 crate::run::pin_test_home();
13084 for phase in [
13085 crate::run::MergeRepairPhase::Fixing,
13086 crate::run::MergeRepairPhase::Requested,
13087 ] {
13088 let mut runner = runner_at(RunStatus::Landing);
13089 runner.state.config.graph.land_rounds = 0;
13090 runner.state.merge_repairs.push(merge_repair_request(phase));
13091 runner.state.save().unwrap();
13092 assert!(!runner.repair_merge_review().await.unwrap());
13093 assert_eq!(runner.state.status, RunStatus::Blocked);
13094 assert_eq!(
13095 runner.state.merge_repairs[0].phase,
13096 crate::run::MergeRepairPhase::Failed
13097 );
13098 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
13099 assert!(
13100 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
13101 "interrupted"
13102 } else {
13103 "budget exhausted"
13104 })
13105 );
13106 assert!(runner.state.merge.is_none());
13107 }
13108 }
13109
13110 #[test]
13114 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
13115 crate::run::pin_test_home();
13116 let mut runner = runner_at(RunStatus::Implementing);
13117 let interrupt = Pause::new();
13118 runner.watch_interrupt(interrupt.clone());
13119
13120 interrupt.park_because("task a1b2 asked to run first");
13121
13122 assert!(runner.park_here().expect("park_here"));
13123 assert!(runner.state.parked);
13124 let last = runner.state.events.last().expect("a park event");
13125 assert_eq!(last.node, "park");
13126 assert!(
13127 last.message.contains("task a1b2 asked to run first"),
13128 "expected the interrupt reason in {:?}",
13129 last.message
13130 );
13131 }
13132
13133 #[test]
13136 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
13137 crate::run::pin_test_home();
13138 let mut runner = runner_at(RunStatus::Implementing);
13139 runner.state.begin_pass();
13140 let shutdown = Pause::new();
13141 runner.on_pause(shutdown.clone());
13142 let upgrade = ParkCause::Upgrade {
13143 from: "0.121.0".to_owned(),
13144 to: Some("0.121.1".to_owned()),
13145 };
13146 shutdown.park_for(upgrade.clone());
13147 shutdown.park_for(ParkCause::Operator);
13148
13149 assert!(runner.park_here().expect("park_here"));
13150 assert_eq!(
13151 runner.state.passes[0].end,
13152 Some(crate::run::PassEnd::Parked { cause: upgrade })
13153 );
13154 let last = runner.state.events.last().expect("a park event");
13155 assert!(last.message.contains("0.121.0"), "{}", last.message);
13156 }
13157
13158 #[test]
13166 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
13167 crate::run::pin_test_home();
13168 let mut runner = runner_at(RunStatus::Implementing);
13169 let shutdown = Pause::new();
13170 runner.on_pause(shutdown.clone());
13171 let interrupt = Pause::new();
13172 runner.watch_interrupt(interrupt.clone());
13173
13174 assert!(!runner.park_here().expect("park_here"));
13176 assert!(!runner.state.parked);
13177
13178 interrupt.park_because("test");
13180 assert!(!shutdown.parked());
13181 assert!(runner.park_here().expect("park_here"));
13182 }
13183
13184 #[tokio::test]
13198 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
13199 crate::run::pin_test_home();
13200 let mut runner = runner_at(RunStatus::Implementing);
13201 let interrupt = Pause::new();
13202 runner.watch_interrupt(interrupt.clone());
13203
13204 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
13205 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
13206
13207 let node = async move {
13211 started_tx.send(()).expect("send started");
13212 finish_rx.await.expect("recv finish");
13213 "node finished"
13214 };
13215
13216 let interrupter = async move {
13217 started_rx.await.expect("recv started");
13218 interrupt.park_because("higher-priority task waiting");
13220 tokio::task::yield_now().await;
13224 finish_tx.send(()).expect("send finish");
13225 };
13226
13227 let (node_result, ()) = tokio::join!(node, interrupter);
13228 assert_eq!(
13229 node_result, "node finished",
13230 "the in-flight call ran to completion"
13231 );
13232
13233 assert!(runner.park_here().expect("park_here"));
13236 assert!(runner.state.parked);
13237 }
13238
13239 #[test]
13245 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
13246 crate::run::pin_test_home();
13247 let mut runner = runner_at(RunStatus::Judging);
13248 runner.state.config.agents = vec![conductor()];
13252 runner.state.candidates = vec![Candidate {
13253 index: 0,
13254 label: 'A',
13255 agent: "alpha".to_owned(),
13256 branch: "magi/x/A".to_owned(),
13257 worktree: PathBuf::from("/nonexistent/worktree"),
13258 summary: "did the thing".to_owned(),
13259 stat: "1 file changed".to_owned(),
13260 files: 1,
13261 commits: 1,
13262 empty: false,
13263 failed: None,
13264 verified_noop: None,
13265 duration_ms: 1234,
13266 folded: false,
13267 }];
13268 let run_id = runner.state.id.clone();
13269
13270 let interrupt = Pause::new();
13271 runner.watch_interrupt(interrupt.clone());
13272 interrupt.park_because("task c3d4 asked to run first");
13273 assert!(runner.park_here().expect("park_here"));
13274
13275 let resumed = Runner::resume(&run_id).expect("resume");
13276 assert_eq!(resumed.state.candidates.len(), 1);
13277 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
13278 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
13279 assert_eq!(resumed.state.status, runner.state.status);
13280 assert!(
13281 resumed.state.parked,
13282 "still parked until `execute` actually walks the graph again"
13283 );
13284 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
13285 }
13286
13287 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
13289 let mut q = ask::Question::new(
13290 run.to_owned(),
13291 "implement".to_owned(),
13292 "impl-A".to_owned(),
13293 "Which storage backend should the cache use?".to_owned(),
13294 String::new(),
13295 vec!["SQLite".to_owned(), "Redis".to_owned()],
13296 );
13297 store.put(&mut q).unwrap();
13298 q
13299 }
13300
13301 #[test]
13302 fn a_failed_runs_open_question_is_abandoned() {
13303 ask_test_home();
13304 let store = ask::Questions::open();
13305 let mut runner = runner_at(RunStatus::Failed);
13306 let run = runner.state.id.clone();
13307 let q = ask_open_question(&store, &run);
13308
13309 runner.settle_questions();
13310
13311 let back = store.get(&q.id).unwrap();
13312 assert!(
13313 !back.status.open(),
13314 "the seat that asked died with the run; nobody is left to read an answer"
13315 );
13316 assert!(
13317 back.detail.contains(&run) && back.detail.contains("failed"),
13318 "the reason names what the run became, not just that it is gone: {}",
13319 back.detail
13320 );
13321 }
13322
13323 #[test]
13324 fn a_merged_runs_open_question_is_abandoned_too() {
13325 ask_test_home();
13326 let store = ask::Questions::open();
13327 for status in [RunStatus::Merged, RunStatus::Ready] {
13330 let mut runner = runner_at(status);
13331 let run = runner.state.id.clone();
13332 let q = ask_open_question(&store, &run);
13333
13334 runner.settle_questions();
13335
13336 let back = store.get(&q.id).unwrap();
13337 assert!(
13338 !back.status.open(),
13339 "{status:?} run's question must not outlive the run"
13340 );
13341 }
13342 }
13343
13344 #[test]
13345 fn a_still_resumable_runs_open_question_is_left_alone() {
13346 ask_test_home();
13347 let store = ask::Questions::open();
13348 for status in [RunStatus::Blocked, RunStatus::Stalled] {
13354 let mut runner = runner_at(status);
13355 let run = runner.state.id.clone();
13356 let q = ask_open_question(&store, &run);
13357
13358 runner.settle_questions();
13359
13360 let back = store.get(&q.id).unwrap();
13361 assert!(
13362 back.status.open(),
13363 "{status:?} is still alive; the question must still be waiting"
13364 );
13365 }
13366 }
13367
13368 #[test]
13369 fn settle_questions_never_touches_an_already_answered_question() {
13370 ask_test_home();
13371 let store = ask::Questions::open();
13372 let mut runner = runner_at(RunStatus::Failed);
13373 let run = runner.state.id.clone();
13374 let mut q = ask_open_question(&store, &run);
13375 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
13376 .unwrap();
13377 store.put(&mut q).unwrap();
13378
13379 runner.settle_questions();
13384 runner.settle_questions();
13385
13386 let back = store.get(&q.id).unwrap();
13387 assert_eq!(
13388 back.status,
13389 ask::QuestionStatus::Answered,
13390 "a real answer is a decision on record, never overwritten by a sweep"
13391 );
13392 }
13393
13394 #[tokio::test]
13405 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
13406 crate::run::pin_test_home();
13407 let tmp = tempfile::tempdir().expect("tempdir");
13408 let repo = tmp.path().join("repo");
13409 std::fs::create_dir_all(&repo).unwrap();
13410 init_repo(&repo);
13411
13412 let mut config = Config::default();
13413 config.graph.worktree_root = Some(tmp.path().join("wt"));
13414
13415 let mut state = RunState::new(
13416 repo.clone(),
13417 "main".to_owned(),
13418 "deadbeef".to_owned(),
13419 "task".to_owned(),
13420 config,
13421 );
13422 let root = state.worktree_root();
13423 let wt_a = root.join("cand-A");
13424 let wt_b = root.join("cand-B");
13425 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
13426 .await
13427 .expect("worktree A");
13428 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
13429 .await
13430 .expect("worktree B");
13431
13432 state.candidates = vec![
13433 Candidate {
13434 index: 0,
13435 label: 'A',
13436 agent: "alpha".to_owned(),
13437 branch: "magi/x/A".to_owned(),
13438 worktree: wt_a.clone(),
13439 summary: String::new(),
13440 stat: String::new(),
13441 files: 0,
13442 commits: 0,
13443 empty: false,
13444 failed: None,
13445 verified_noop: None,
13446 duration_ms: 0,
13447 folded: false,
13448 },
13449 Candidate {
13450 index: 1,
13451 label: 'B',
13452 agent: "beta".to_owned(),
13453 branch: "magi/x/B".to_owned(),
13454 worktree: wt_b.clone(),
13455 summary: String::new(),
13456 stat: String::new(),
13457 files: 0,
13458 commits: 0,
13459 empty: false,
13460 failed: None,
13461 verified_noop: None,
13462 duration_ms: 0,
13463 folded: false,
13464 },
13465 ];
13466 state.tally = Some(Tally {
13467 first_choice: BTreeMap::from([('A', 1)]),
13468 borda: BTreeMap::new(),
13469 winner: 'A',
13470 rankings: 1,
13471 unanimous_initial: true,
13472 deliberated: false,
13473 changed_votes: 0,
13474 unanimous_final: true,
13475 tie_break: None,
13476 judges: 1,
13477 present: 1,
13478 quorum: 1,
13479 met_quorum: true,
13480 uncontested: None,
13481 });
13482 state.status = RunStatus::Ready;
13483
13484 fold_run(&mut state, false, &crate::run::home())
13485 .await
13486 .expect("fold_run");
13487
13488 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
13489 assert!(
13490 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
13491 "the unmerged winner's branch survives"
13492 );
13493 assert!(
13494 !state.candidates[0].folded,
13495 "the winner is not marked folded"
13496 );
13497
13498 assert!(!wt_b.exists(), "the loser's worktree is removed");
13499 assert!(
13500 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
13501 "the loser's branch is removed"
13502 );
13503 assert!(state.candidates[1].folded, "the loser is marked folded");
13504 }
13505
13506 #[tokio::test]
13507 #[cfg(unix)]
13508 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
13509 use std::os::fd::AsRawFd as _;
13510 use std::time::Duration;
13511
13512 let tmp = tempfile::tempdir().unwrap();
13513 let repo = tmp.path().join("repo");
13514 std::fs::create_dir_all(&repo).unwrap();
13515 init_repo(&repo);
13516 let common = repo.join(".git");
13517 std::fs::write(common.join("kanade-sync.contract"), "shared-lock-v1\n").unwrap();
13518 let home = tmp.path().join("home");
13519 let mut config = Config::default();
13520 config.graph.worktree_root = Some(tmp.path().join("wt"));
13521 config.graph.timeout_resource_wait = 1;
13522 let mut state = RunState::new(
13523 repo.clone(),
13524 "main".into(),
13525 "deadbeef".into(),
13526 "task".into(),
13527 config,
13528 );
13529 let root = state.worktree_root();
13530 let candidate = root.join("cand-A");
13531 let reviewer = root.join("review-1");
13532 let branch = state.branch_for('A');
13533 git::worktree_add_branch(&repo, &candidate, &branch, "main")
13534 .await
13535 .unwrap();
13536 git::worktree_add_detached(&repo, &reviewer, "main")
13537 .await
13538 .unwrap();
13539 git::acquire_worktree_config(&repo).await.unwrap();
13540 state.enabled_worktree_config = true;
13541 state.status = RunStatus::Ready;
13542 state.candidates.push(Candidate {
13543 index: 0,
13544 label: 'A',
13545 agent: "implement".into(),
13546 branch: branch.clone(),
13547 worktree: candidate.clone(),
13548 summary: String::new(),
13549 stat: String::new(),
13550 files: 0,
13551 commits: 0,
13552 empty: false,
13553 failed: None,
13554 verified_noop: None,
13555 duration_ms: 0,
13556 folded: false,
13557 });
13558 state.save_under(&home).unwrap();
13559 let config_before = std::fs::read(common.join("config")).unwrap();
13560 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
13561 .await
13562 .unwrap();
13563 let external = std::fs::File::create(common.join("kanade-sync.lock")).unwrap();
13566 assert_eq!(
13567 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13568 0
13569 );
13570 let mut fold = Box::pin(fold_run(&mut state, true, &home));
13571 assert!(
13572 tokio::time::timeout(Duration::from_millis(200), &mut fold)
13573 .await
13574 .is_err()
13575 );
13576 assert!(candidate.exists());
13577 assert!(reviewer.exists());
13578 assert!(git::branch_exists(&repo, &branch).await.unwrap());
13579 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
13580 assert_eq!(
13581 git::git(&repo, &["worktree", "list", "--porcelain"])
13582 .await
13583 .unwrap(),
13584 worktrees_before
13585 );
13586 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
13587 .await
13588 .unwrap()
13589 .unwrap_err();
13590 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
13591 drop(fold);
13592 assert!(candidate.exists() && reviewer.exists());
13593 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
13594 assert_eq!(
13595 state.resource_block.as_ref().unwrap().kind,
13596 "repository coordination"
13597 );
13598 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
13599 drop(external);
13600 let mut fold = Box::pin(fold_run(&mut state, true, &home));
13601 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
13602 .await
13603 .unwrap()
13604 .unwrap();
13605 drop(fold);
13606 assert!(removed.contains(&branch));
13607 assert!(!root.exists());
13608 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
13609 assert!(state.candidates[0].folded);
13610 assert!(!state.enabled_worktree_config);
13611 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
13612 let enabled = git::git_raw(
13613 &repo,
13614 &["config", "--local", "--get", "extensions.worktreeConfig"],
13615 )
13616 .await
13617 .unwrap();
13618 assert!(!enabled.ok(), "the last config reference was released");
13619 }
13620
13621 #[tokio::test]
13624 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
13625 let tmp = tempfile::tempdir().expect("tempdir");
13626 let repo = tmp.path().join("repo");
13627 std::fs::create_dir_all(&repo).unwrap();
13628 init_repo(&repo);
13629 let home = tmp.path().join("home");
13630
13631 let mut config = Config::default();
13632 config.graph.worktree_root = Some(tmp.path().join("wt"));
13633 let mut state = RunState::new(
13634 repo.clone(),
13635 "main".to_owned(),
13636 "deadbeef".to_owned(),
13637 "task".to_owned(),
13638 config,
13639 );
13640 git::git(&repo, &["branch", "magi/x/A", "main"])
13642 .await
13643 .expect("branch");
13644 state.candidates = vec![Candidate {
13645 index: 0,
13646 label: 'A',
13647 agent: "alpha".to_owned(),
13648 branch: "magi/x/A".to_owned(),
13649 worktree: state.worktree_root().join("cand-A"),
13650 summary: String::new(),
13651 stat: String::new(),
13652 files: 0,
13653 commits: 0,
13654 empty: false,
13655 failed: None,
13656 verified_noop: None,
13657 duration_ms: 0,
13658 folded: true,
13659 }];
13660 state.released_to = Some("20260901-000000-new1".to_owned());
13661 state.released_branches = vec!["magi/x/A".to_owned()];
13662
13663 fold_run(&mut state, true, &home).await.expect("fold_run");
13664
13665 assert!(
13666 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
13667 "the handed-over branch survives a fold"
13668 );
13669 }
13670
13671 #[tokio::test]
13675 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
13676 let tmp = tempfile::tempdir().expect("tempdir");
13677 let repo = tmp.path().join("repo");
13678 std::fs::create_dir_all(&repo).unwrap();
13679 init_repo(&repo);
13680 let run = |args: &[&str]| {
13681 let out = std::process::Command::new("git")
13682 .quiet()
13683 .args(args)
13684 .current_dir(&repo)
13685 .output()
13686 .expect("spawn git");
13687 assert!(out.status.success(), "git {args:?}");
13688 };
13689 run(&["branch", "magi/x/A"]);
13690 run(&["checkout", "-q", "-b", "magi/x/B"]);
13691 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
13692 run(&["add", "-A"]);
13693 run(&["commit", "-q", "-m", "work"]);
13694 run(&["checkout", "-q", "main"]);
13695
13696 let probe = RunState::new(
13700 repo.clone(),
13701 "main".to_owned(),
13702 "deadbeef".to_owned(),
13703 "task".to_owned(),
13704 Config::default(),
13705 );
13706 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
13707 let bare = tmp.path().join("origin.git");
13708 let out = std::process::Command::new("git")
13709 .quiet()
13710 .args(["init", "-q", "--bare"])
13711 .arg(&bare)
13712 .output()
13713 .expect("spawn git");
13714 assert!(out.status.success(), "git init --bare");
13715 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
13716 run(&["push", "-q", "origin", "main"]);
13717
13718 let mut state = RunState::new(
13719 repo.clone(),
13720 "main".to_owned(),
13721 "deadbeef".to_owned(),
13722 "task".to_owned(),
13723 Config::default(),
13724 );
13725 state.seeds = vec![refs::Seed {
13726 token: "magi/27b2/A".to_owned(),
13727 kind: refs::SeedKind::Unresolved,
13728 sha: String::new(),
13729 branch: true,
13730 detail: "no branch or commit named magi/27b2/A".to_owned(),
13731 }];
13732
13733 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
13734 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
13735 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
13736 let detail = empty_candidate_detail(&state, "main");
13737 assert!(detail.starts_with("empty candidate"), "{detail}");
13738 assert!(detail.contains("magi/27b2/A"), "{detail}");
13739 }
13740
13741 #[tokio::test]
13750 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
13751 let tmp = tempfile::tempdir().expect("tempdir");
13752 let repo = tmp.path().join("repo");
13753 std::fs::create_dir_all(&repo).unwrap();
13754 init_repo(&repo);
13755
13756 let mut config = Config::default();
13757 config.merge.mode = MergeMode::Local;
13758
13759 let mut state = RunState::new(
13760 repo.clone(),
13761 "main".to_owned(),
13762 "deadbeef".to_owned(),
13763 "task".to_owned(),
13764 config,
13765 );
13766 state.candidates = vec![Candidate {
13767 index: 0,
13768 label: 'A',
13769 agent: "alpha".to_owned(),
13770 branch: "does-not-exist".to_owned(),
13771 worktree: repo.clone(),
13772 summary: String::new(),
13773 stat: String::new(),
13774 files: 0,
13775 commits: 0,
13776 empty: false,
13777 failed: None,
13778 verified_noop: None,
13779 duration_ms: 0,
13780 folded: false,
13781 }];
13782 state.tally = Some(Tally {
13783 first_choice: BTreeMap::from([('A', 1)]),
13784 borda: BTreeMap::new(),
13785 winner: 'A',
13786 rankings: 1,
13787 unanimous_initial: true,
13788 deliberated: false,
13789 changed_votes: 0,
13790 unanimous_final: true,
13791 tie_break: None,
13792 judges: 0,
13793 present: 0,
13794 quorum: 0,
13795 met_quorum: true,
13796 uncontested: Some("only candidate A produced a change".to_owned()),
13797 });
13798 state.reviews = vec![ReviewRound {
13799 round: 1,
13800 head: "deadbeef".to_owned(),
13801 verified_head: None,
13802 verified_at: None,
13803 reviews: Vec::new(),
13804 e2e: Vec::new(),
13805 fix: None,
13806 blocking: 0,
13807 answered: 0,
13808 expected: 0,
13809 clean: true,
13810 verify_retried: false,
13811 e2e_deferred: false,
13812 e2e_defer_reason: None,
13813 progressed: false,
13814 vote_split: false,
13815 reconsideration: Vec::new(),
13816 verdict: None,
13817 }];
13818 state.gate = vec![CommandOutcome {
13819 command: "test".to_owned(),
13820 code: Some(0),
13821 output_tail: String::new(),
13822 duration_ms: 0,
13823 resource_blocked: false,
13824 }];
13825 state.gate_ran = true;
13826 state.status = RunStatus::Ready;
13831 state.merge = Some(MergeOutcome {
13832 mode: MergeMode::Local,
13833 ok: false,
13834 detail: "already concluded".to_owned(),
13835 empty: false,
13836 });
13837
13838 let mut runner = Runner {
13839 state,
13840 roles: ResolvedRoles {
13841 implementers: Vec::new(),
13842 judges: Vec::new(),
13843 reviewers: Vec::new(),
13844 fixer: None,
13845 conductor: conductor(),
13846 implementer_roster: Vec::new(),
13847 judge_roster: Vec::new(),
13848 reviewer_roster: Vec::new(),
13849 },
13850 sem: Arc::new(Semaphore::new(1)),
13851 pause: Pause::new(),
13852 interrupt: Pause::new(),
13853 };
13854
13855 runner.merge().await.expect("merge");
13856
13857 assert_eq!(
13858 runner.state.status,
13859 RunStatus::Ready,
13860 "a concluded run's status must not change on reentry"
13861 );
13862 assert_eq!(
13863 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
13864 Some("already concluded"),
13865 "merge must not run again once the node already recorded an outcome"
13866 );
13867 }
13868
13869 #[tokio::test]
13878 async fn merge_refuses_a_gate_that_has_not_actually_run() {
13879 let tmp = tempfile::tempdir().expect("tempdir");
13880 let repo = tmp.path().join("repo");
13881 std::fs::create_dir_all(&repo).unwrap();
13882 init_repo(&repo);
13883
13884 let mut config = Config::default();
13885 config.merge.mode = MergeMode::Local;
13886
13887 let mut state = RunState::new(
13888 repo.clone(),
13889 "main".to_owned(),
13890 "deadbeef".to_owned(),
13891 "task".to_owned(),
13892 config,
13893 );
13894 state.candidates = vec![Candidate {
13895 index: 0,
13896 label: 'A',
13897 agent: "alpha".to_owned(),
13898 branch: "does-not-exist".to_owned(),
13899 worktree: repo.clone(),
13900 summary: String::new(),
13901 stat: String::new(),
13902 files: 0,
13903 commits: 0,
13904 empty: false,
13905 failed: None,
13906 verified_noop: None,
13907 duration_ms: 0,
13908 folded: false,
13909 }];
13910 state.tally = Some(Tally {
13911 first_choice: BTreeMap::from([('A', 1)]),
13912 borda: BTreeMap::new(),
13913 winner: 'A',
13914 rankings: 1,
13915 unanimous_initial: true,
13916 deliberated: false,
13917 changed_votes: 0,
13918 unanimous_final: true,
13919 tie_break: None,
13920 judges: 0,
13921 present: 0,
13922 quorum: 0,
13923 met_quorum: true,
13924 uncontested: Some("only candidate A produced a change".to_owned()),
13925 });
13926 state.reviews = vec![ReviewRound {
13927 round: 1,
13928 head: "deadbeef".to_owned(),
13929 verified_head: None,
13930 verified_at: None,
13931 reviews: Vec::new(),
13932 e2e: Vec::new(),
13933 fix: None,
13934 blocking: 0,
13935 answered: 0,
13936 expected: 0,
13937 clean: true,
13938 verify_retried: false,
13939 e2e_deferred: false,
13940 e2e_defer_reason: None,
13941 progressed: false,
13942 vote_split: false,
13943 reconsideration: Vec::new(),
13944 verdict: None,
13945 }];
13946 state.gate = Vec::new();
13948 state.gate_ran = false;
13949 state.status = RunStatus::Gating;
13950
13951 let mut runner = Runner {
13952 state,
13953 roles: ResolvedRoles {
13954 implementers: Vec::new(),
13955 judges: Vec::new(),
13956 reviewers: Vec::new(),
13957 fixer: None,
13958 conductor: conductor(),
13959 implementer_roster: Vec::new(),
13960 judge_roster: Vec::new(),
13961 reviewer_roster: Vec::new(),
13962 },
13963 sem: Arc::new(Semaphore::new(1)),
13964 pause: Pause::new(),
13965 interrupt: Pause::new(),
13966 };
13967
13968 runner.merge().await.expect("merge");
13969
13970 assert!(
13971 runner.state.merge.is_none(),
13972 "an empty gate must never be read as a passing one: {:?}",
13973 runner.state.merge
13974 );
13975 }
13976
13977 #[tokio::test]
13984 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
13985 let tmp = tempfile::tempdir().expect("tempdir");
13986 let repo = tmp.path().join("repo");
13987 std::fs::create_dir_all(&repo).unwrap();
13988 init_repo(&repo);
13989
13990 let config = Config::default();
13992
13993 let mut state = RunState::new(
13994 repo.clone(),
13995 "main".to_owned(),
13996 "deadbeef".to_owned(),
13997 "task".to_owned(),
13998 config,
13999 );
14000 state.candidates = vec![Candidate {
14001 index: 0,
14002 label: 'A',
14003 agent: "alpha".to_owned(),
14004 branch: "does-not-exist".to_owned(),
14005 worktree: repo.clone(),
14006 summary: String::new(),
14007 stat: String::new(),
14008 files: 0,
14009 commits: 0,
14010 empty: false,
14011 failed: None,
14012 verified_noop: None,
14013 duration_ms: 0,
14014 folded: false,
14015 }];
14016 state.tally = Some(Tally {
14017 first_choice: BTreeMap::from([('A', 1)]),
14018 borda: BTreeMap::new(),
14019 winner: 'A',
14020 rankings: 1,
14021 unanimous_initial: true,
14022 deliberated: false,
14023 changed_votes: 0,
14024 unanimous_final: true,
14025 tie_break: None,
14026 judges: 0,
14027 present: 0,
14028 quorum: 0,
14029 met_quorum: true,
14030 uncontested: Some("only candidate A produced a change".to_owned()),
14031 });
14032 state.reviews = vec![ReviewRound {
14033 round: 1,
14034 head: "deadbeef".to_owned(),
14035 verified_head: None,
14036 verified_at: None,
14037 reviews: Vec::new(),
14038 e2e: Vec::new(),
14039 fix: None,
14040 blocking: 0,
14041 answered: 0,
14042 expected: 0,
14043 clean: true,
14044 verify_retried: false,
14045 e2e_deferred: false,
14046 e2e_defer_reason: None,
14047 progressed: false,
14048 vote_split: false,
14049 reconsideration: Vec::new(),
14050 verdict: None,
14051 }];
14052
14053 let mut runner = Runner {
14054 state,
14055 roles: ResolvedRoles {
14056 implementers: Vec::new(),
14057 judges: Vec::new(),
14058 reviewers: Vec::new(),
14059 fixer: None,
14060 conductor: conductor(),
14061 implementer_roster: Vec::new(),
14062 judge_roster: Vec::new(),
14063 reviewer_roster: Vec::new(),
14064 },
14065 sem: Arc::new(Semaphore::new(1)),
14066 pause: Pause::new(),
14067 interrupt: Pause::new(),
14068 };
14069
14070 runner.gate().await.expect("gate");
14071 assert!(
14072 runner.state.gate_ran,
14073 "zero configured commands is still a real attempt, not an unrun gate"
14074 );
14075 assert!(runner.state.gate.is_empty());
14076 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
14077 assert_ne!(
14078 runner.state.status,
14079 RunStatus::Blocked,
14080 "a gate with nothing to check must not read as failed"
14081 );
14082
14083 runner.merge().await.expect("merge");
14084 assert_eq!(
14085 runner.state.status,
14086 RunStatus::Ready,
14087 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14088 );
14089 }
14090
14091 #[tokio::test]
14102 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
14103 crate::run::pin_test_home();
14104 let home = crate::run::home();
14105
14106 let tmp = tempfile::tempdir().expect("tempdir");
14107 let repo = tmp.path().join("repo");
14108 std::fs::create_dir_all(&repo).unwrap();
14109 init_repo(&repo);
14110 let cache_dir = tmp.path().join("target");
14113
14114 let mut config = Config::default();
14115 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
14116 config.graph.timeout_verify = Some(2);
14119 config.graph.timeout_resource_wait = 2;
14120
14121 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14122 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14123 .expect("no io error acquiring directly")
14124 {
14125 crate::cache::AcquireOutcome::Acquired(g) => g,
14126 crate::cache::AcquireOutcome::Busy(b) => {
14127 panic!("expected the direct acquire to win the lease first: {b:?}")
14128 }
14129 };
14130
14131 let mut state = RunState::new(
14132 repo.clone(),
14133 "main".to_owned(),
14134 "deadbeef".to_owned(),
14135 "task".to_owned(),
14136 config,
14137 );
14138 state.candidates = vec![Candidate {
14139 index: 0,
14140 label: 'A',
14141 agent: "alpha".to_owned(),
14142 branch: "does-not-exist".to_owned(),
14143 worktree: repo.clone(),
14144 summary: String::new(),
14145 stat: String::new(),
14146 files: 0,
14147 commits: 0,
14148 empty: false,
14149 failed: None,
14150 verified_noop: None,
14151 duration_ms: 0,
14152 folded: false,
14153 }];
14154 state.tally = Some(Tally {
14155 first_choice: BTreeMap::from([('A', 1)]),
14156 borda: BTreeMap::new(),
14157 winner: 'A',
14158 rankings: 1,
14159 unanimous_initial: true,
14160 deliberated: false,
14161 changed_votes: 0,
14162 unanimous_final: true,
14163 tie_break: None,
14164 judges: 0,
14165 present: 0,
14166 quorum: 0,
14167 met_quorum: true,
14168 uncontested: Some("only candidate A produced a change".to_owned()),
14169 });
14170 state.reviews = vec![ReviewRound {
14171 round: 1,
14172 head: "deadbeef".to_owned(),
14173 verified_head: None,
14174 verified_at: None,
14175 reviews: Vec::new(),
14176 e2e: Vec::new(),
14177 fix: None,
14178 blocking: 0,
14179 answered: 0,
14180 expected: 0,
14181 clean: true,
14182 verify_retried: false,
14183 e2e_deferred: false,
14184 e2e_defer_reason: None,
14185 progressed: false,
14186 vote_split: false,
14187 reconsideration: Vec::new(),
14188 verdict: None,
14189 }];
14190
14191 let mut runner = Runner {
14192 state,
14193 roles: ResolvedRoles {
14194 implementers: Vec::new(),
14195 judges: Vec::new(),
14196 reviewers: Vec::new(),
14197 fixer: None,
14198 conductor: conductor(),
14199 implementer_roster: Vec::new(),
14200 judge_roster: Vec::new(),
14201 reviewer_roster: Vec::new(),
14202 },
14203 sem: Arc::new(Semaphore::new(1)),
14204 pause: Pause::new(),
14205 interrupt: Pause::new(),
14206 };
14207
14208 let started = std::time::Instant::now();
14209 runner.gate().await.expect("gate");
14210 assert!(
14211 started.elapsed() < Duration::from_secs(1),
14212 "a gate with nothing to run must never wait on a lease it never needed"
14213 );
14214 assert!(
14215 runner.state.gate_ran,
14216 "zero commands is still a real, immediate attempt"
14217 );
14218 assert!(runner.state.gate.is_empty());
14219 assert_ne!(
14220 runner.state.status,
14221 RunStatus::Blocked,
14222 "must not read as resource-blocked on a lease it never asked for"
14223 );
14224 }
14225
14226 #[tokio::test]
14236 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
14237 crate::run::pin_test_home();
14238
14239 let tmp = tempfile::tempdir().expect("tempdir");
14240 let repo = tmp.path().join("repo");
14241 std::fs::create_dir_all(&repo).unwrap();
14242 init_repo(&repo);
14243
14244 let mut config = Config::default();
14245 config.verify.gate = vec![
14246 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
14247 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
14248 .to_owned(),
14249 ];
14250
14251 let mut state = RunState::new(
14252 repo.clone(),
14253 "main".to_owned(),
14254 "deadbeef".to_owned(),
14255 "task".to_owned(),
14256 config,
14257 );
14258 let run_id = state.id.clone();
14259 state.candidates = vec![Candidate {
14260 index: 0,
14261 label: 'A',
14262 agent: "alpha".to_owned(),
14263 branch: "does-not-exist".to_owned(),
14264 worktree: repo.clone(),
14265 summary: String::new(),
14266 stat: String::new(),
14267 files: 0,
14268 commits: 0,
14269 empty: false,
14270 failed: None,
14271 verified_noop: None,
14272 duration_ms: 0,
14273 folded: false,
14274 }];
14275 state.tally = Some(Tally {
14276 first_choice: BTreeMap::from([('A', 1)]),
14277 borda: BTreeMap::new(),
14278 winner: 'A',
14279 rankings: 1,
14280 unanimous_initial: true,
14281 deliberated: false,
14282 changed_votes: 0,
14283 unanimous_final: true,
14284 tie_break: None,
14285 judges: 0,
14286 present: 0,
14287 quorum: 0,
14288 met_quorum: true,
14289 uncontested: Some("only candidate A produced a change".to_owned()),
14290 });
14291 state.reviews = vec![ReviewRound {
14292 round: 1,
14293 head: "deadbeef".to_owned(),
14294 verified_head: None,
14295 verified_at: None,
14296 reviews: Vec::new(),
14297 e2e: Vec::new(),
14298 fix: None,
14299 blocking: 0,
14300 answered: 0,
14301 expected: 0,
14302 clean: true,
14303 verify_retried: false,
14304 e2e_deferred: false,
14305 e2e_defer_reason: None,
14306 progressed: false,
14307 vote_split: false,
14308 reconsideration: Vec::new(),
14309 verdict: None,
14310 }];
14311
14312 let mut runner = Runner {
14313 state,
14314 roles: ResolvedRoles {
14315 implementers: Vec::new(),
14316 judges: Vec::new(),
14317 reviewers: Vec::new(),
14318 fixer: None,
14319 conductor: conductor(),
14320 implementer_roster: Vec::new(),
14321 judge_roster: Vec::new(),
14322 reviewer_roster: Vec::new(),
14323 },
14324 sem: Arc::new(Semaphore::new(1)),
14325 pause: Pause::new(),
14326 interrupt: Pause::new(),
14327 };
14328
14329 let started_marker = repo.join("started.marker");
14330 let release_marker = repo.join("release.marker");
14331 let poller = tokio::spawn(async move {
14332 for _ in 0..100 {
14337 if started_marker.exists()
14338 && let Ok(s) = crate::run::RunState::load(&run_id)
14339 && let Some(a) = s.active.get("gate")
14340 {
14341 std::fs::write(&release_marker, b"go").expect("release marker");
14342 return Some(a.clone());
14343 }
14344 tokio::time::sleep(Duration::from_millis(50)).await;
14345 }
14346 None
14347 });
14348
14349 runner.gate().await.expect("gate");
14350 let captured = poller.await.expect("poller task");
14351 let captured = captured.expect(
14352 "the poller never saw a `gate` task entry in run.json while the command was \
14353 still blocked on its own release marker",
14354 );
14355
14356 assert_eq!(captured.task.as_deref(), Some("gate"));
14357 assert_eq!(captured.node, "gate");
14358 assert_eq!(captured.index, Some(1));
14359 assert_eq!(captured.total, Some(1));
14360 assert!(
14361 captured
14362 .command
14363 .as_deref()
14364 .is_some_and(|c| c.contains("started.marker")),
14365 "{captured:?}"
14366 );
14367
14368 assert!(
14369 runner.state.active.is_empty(),
14370 "the entry must be cleared once the command actually finished: {:?}",
14371 runner.state.active
14372 );
14373 assert!(runner.state.gate_ran);
14374 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
14375 }
14376
14377 #[tokio::test]
14380 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
14381 use crate::verdict::{Finding, ReviewVote, Severity};
14382 crate::run::pin_test_home();
14383 let tmp = tempfile::tempdir().expect("tempdir");
14384 let repo = tmp.path().join("repo");
14385 std::fs::create_dir_all(&repo).unwrap();
14386 init_repo(&repo);
14387
14388 for (severity, vote, expect) in [
14389 (Severity::Major, ReviewVote::Reject, true),
14390 (Severity::Minor, ReviewVote::Reject, false),
14391 (Severity::Major, ReviewVote::Approve, false),
14392 ] {
14393 let mut round = review_round(false, 1, 1, 1, false, true);
14394 round.reviews = vec![ReviewRecord {
14395 reviewer: 1,
14396 agent: "alpha".to_owned(),
14397 summary: String::new(),
14398 findings: vec![Finding {
14399 id: "R1-1-1".to_owned(),
14400 severity,
14401 file: None,
14402 line: None,
14403 title: "t".to_owned(),
14404 detail: String::new(),
14405 }],
14406 vote: Some(vote),
14407 failed: None,
14408 duration_ms: 0,
14409 attempts: 0,
14410 }];
14411 let mut state = RunState::new(
14412 repo.clone(),
14413 "main".to_owned(),
14414 "deadbeef".to_owned(),
14415 "task".to_owned(),
14416 Config::default(),
14417 );
14418 state.reviews = vec![round];
14419 let mut runner = Runner {
14420 state,
14421 roles: ResolvedRoles {
14422 implementers: Vec::new(),
14423 judges: Vec::new(),
14424 reviewers: Vec::new(),
14425 fixer: None,
14426 conductor: conductor(),
14427 implementer_roster: Vec::new(),
14428 judge_roster: Vec::new(),
14429 reviewer_roster: Vec::new(),
14430 },
14431 sem: Arc::new(Semaphore::new(1)),
14432 pause: Pause::new(),
14433 interrupt: Pause::new(),
14434 };
14435 let shell = runner.state.config.shell();
14436 runner
14437 .stop_reviewing("round budget spent", &shell, &repo)
14438 .await
14439 .expect("stop_reviewing");
14440 assert_eq!(runner.state.status, RunStatus::Gating);
14441 assert_eq!(
14442 runner.state.contested_handoff.is_some(),
14443 expect,
14444 "{severity:?} + {vote:?}"
14445 );
14446 }
14447 }
14448
14449 #[tokio::test]
14462 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
14463 crate::run::pin_test_home();
14464 let home = crate::run::home();
14465
14466 let tmp = tempfile::tempdir().expect("tempdir");
14467 let repo = tmp.path().join("repo");
14468 std::fs::create_dir_all(&repo).unwrap();
14469 init_repo(&repo);
14470 let head = crate::git::rev_parse(&repo, "HEAD")
14471 .await
14472 .expect("rev-parse");
14473 let cache_dir = tmp.path().join("target");
14476
14477 let mut config = Config::default();
14478 config.verify.e2e = vec![format!(
14479 "CARGO_TARGET_DIR='{}' test -f README.md",
14480 cache_dir.display()
14481 )];
14482 config.graph.review_rounds = 1;
14483 config.graph.timeout_verify = Some(2);
14486 config.graph.timeout_resource_wait = 2;
14487
14488 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14489 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14490 .expect("no io error acquiring directly")
14491 {
14492 crate::cache::AcquireOutcome::Acquired(g) => g,
14493 crate::cache::AcquireOutcome::Busy(b) => {
14494 panic!("expected the direct acquire to win the lease first: {b:?}")
14495 }
14496 };
14497
14498 let mut state = RunState::new(
14499 repo.clone(),
14500 "main".to_owned(),
14501 head.clone(),
14502 "task".to_owned(),
14503 config,
14504 );
14505 state.candidates = vec![Candidate {
14506 index: 0,
14507 label: 'A',
14508 agent: "alpha".to_owned(),
14509 branch: "does-not-exist".to_owned(),
14510 worktree: repo.clone(),
14511 summary: String::new(),
14512 stat: String::new(),
14513 files: 0,
14514 commits: 0,
14515 empty: false,
14516 failed: None,
14517 verified_noop: None,
14518 duration_ms: 0,
14519 folded: false,
14520 }];
14521 state.tally = Some(Tally {
14522 first_choice: BTreeMap::from([('A', 1)]),
14523 borda: BTreeMap::new(),
14524 winner: 'A',
14525 rankings: 1,
14526 unanimous_initial: true,
14527 deliberated: false,
14528 changed_votes: 0,
14529 unanimous_final: true,
14530 tie_break: None,
14531 judges: 0,
14532 present: 0,
14533 quorum: 0,
14534 met_quorum: true,
14535 uncontested: Some("only candidate A produced a change".to_owned()),
14536 });
14537 state.reviews = vec![ReviewRound {
14541 round: 1,
14542 head: head.clone(),
14543 verified_head: None,
14544 verified_at: None,
14545 reviews: Vec::new(),
14546 e2e: Vec::new(),
14547 fix: None,
14548 blocking: 1,
14549 answered: 1,
14550 expected: 1,
14551 clean: false,
14552 verify_retried: false,
14553 e2e_deferred: true,
14554 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
14555 progressed: false,
14556 vote_split: false,
14557 reconsideration: Vec::new(),
14558 verdict: None,
14559 }];
14560
14561 let mut runner = Runner {
14562 state,
14563 roles: ResolvedRoles {
14564 implementers: Vec::new(),
14565 judges: Vec::new(),
14566 reviewers: Vec::new(),
14567 fixer: None,
14568 conductor: conductor(),
14569 implementer_roster: Vec::new(),
14570 judge_roster: Vec::new(),
14571 reviewer_roster: Vec::new(),
14572 },
14573 sem: Arc::new(Semaphore::new(1)),
14574 pause: Pause::new(),
14575 interrupt: Pause::new(),
14576 };
14577
14578 let shell = runner.state.config.shell();
14579 runner
14580 .stop_reviewing("round budget spent", &shell, &repo)
14581 .await
14582 .expect("stop_reviewing");
14583
14584 let last = runner.state.reviews.last().expect("round record");
14585 assert_eq!(
14586 last.e2e_status(),
14587 E2eStatus::ResourceBlocked,
14588 "the shared cache is still held; the attempt must read as blocked, not deferred or \
14589 failed: {last:?}"
14590 );
14591 assert_eq!(
14592 last.verified_head.as_deref(),
14593 Some(head.as_str()),
14594 "which commit this attempt targeted is known even though nothing finished checking \
14595 it"
14596 );
14597 let first_attempt_at = last
14598 .verified_at
14599 .expect("when this attempt ran is known too");
14600 assert_ne!(
14601 runner.state.status,
14602 RunStatus::Blocked,
14603 "contention is evidence about the machine, not the patch — it must not settle the \
14604 run as blocked: {:?}",
14605 runner.state.status
14606 );
14607 assert!(
14608 !runner
14609 .state
14610 .events
14611 .iter()
14612 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
14613 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
14614 runner.state.events
14615 );
14616
14617 runner
14622 .stop_reviewing("round budget spent", &shell, &repo)
14623 .await
14624 .expect("stop_reviewing retry");
14625 assert_eq!(
14626 runner.state.reviews.len(),
14627 1,
14628 "no new round was started: {:?}",
14629 runner.state.reviews
14630 );
14631 let last = runner.state.reviews.last().expect("round record");
14632 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
14633 assert!(
14634 last.verified_at.expect("still known") > first_attempt_at,
14635 "a second reentry must be a fresh attempt, not a stale copy of the first"
14636 );
14637 assert_ne!(runner.state.status, RunStatus::Blocked);
14638
14639 held.release();
14640 }
14641
14642 #[tokio::test]
14654 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
14655 crate::run::pin_test_home();
14656 let home = crate::run::home();
14657
14658 let tmp = tempfile::tempdir().expect("tempdir");
14659 let repo = tmp.path().join("repo");
14660 std::fs::create_dir_all(&repo).unwrap();
14661 init_repo(&repo);
14662 let head = crate::git::rev_parse(&repo, "HEAD")
14663 .await
14664 .expect("rev-parse");
14665 let cache_dir = tmp.path().join("target");
14666
14667 let mut config = Config::default();
14668 config.verify.e2e = vec![format!(
14669 "CARGO_TARGET_DIR='{}' test -f README.md",
14670 cache_dir.display()
14671 )];
14672 config.graph.review_rounds = 1;
14673 config.graph.timeout_verify = Some(2);
14674 config.graph.timeout_resource_wait = 2;
14675
14676 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
14677 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
14678 .expect("no io error acquiring directly")
14679 {
14680 crate::cache::AcquireOutcome::Acquired(g) => g,
14681 crate::cache::AcquireOutcome::Busy(b) => {
14682 panic!("expected the direct acquire to win the lease first: {b:?}")
14683 }
14684 };
14685
14686 let mut state = RunState::new(
14687 repo.clone(),
14688 "main".to_owned(),
14689 head.clone(),
14690 "task".to_owned(),
14691 config,
14692 );
14693 state.candidates = vec![Candidate {
14694 index: 0,
14695 label: 'A',
14696 agent: "alpha".to_owned(),
14697 branch: "does-not-exist".to_owned(),
14698 worktree: repo.clone(),
14699 summary: String::new(),
14700 stat: String::new(),
14701 files: 0,
14702 commits: 0,
14703 empty: false,
14704 failed: None,
14705 verified_noop: None,
14706 duration_ms: 0,
14707 folded: false,
14708 }];
14709 state.tally = Some(Tally {
14710 first_choice: BTreeMap::from([('A', 1)]),
14711 borda: BTreeMap::new(),
14712 winner: 'A',
14713 rankings: 1,
14714 unanimous_initial: true,
14715 deliberated: false,
14716 changed_votes: 0,
14717 unanimous_final: true,
14718 tie_break: None,
14719 judges: 0,
14720 present: 0,
14721 quorum: 0,
14722 met_quorum: true,
14723 uncontested: Some("only candidate A produced a change".to_owned()),
14724 });
14725 state.reviews = vec![ReviewRound {
14729 round: 1,
14730 head: head.clone(),
14731 verified_head: Some(head.clone()),
14732 verified_at: Some(jiff::Timestamp::now()),
14733 reviews: Vec::new(),
14734 e2e: vec![CommandOutcome {
14735 command: format!(
14736 "CARGO_TARGET_DIR='{}' test -f README.md",
14737 cache_dir.display()
14738 ),
14739 code: None,
14740 output_tail: "waiting for the shared build cache".to_owned(),
14741 duration_ms: 0,
14742 resource_blocked: true,
14743 }],
14744 fix: None,
14745 blocking: 1,
14746 answered: 1,
14747 expected: 1,
14748 clean: false,
14749 verify_retried: false,
14750 e2e_deferred: false,
14751 e2e_defer_reason: None,
14752 progressed: false,
14753 vote_split: false,
14754 reconsideration: Vec::new(),
14755 verdict: None,
14756 }];
14757
14758 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
14759 let mut runner = Runner {
14760 state,
14761 roles: ResolvedRoles {
14762 implementers: Vec::new(),
14763 judges: Vec::new(),
14764 reviewers: Vec::new(),
14765 fixer: None,
14766 conductor: conductor(),
14767 implementer_roster: Vec::new(),
14768 judge_roster: Vec::new(),
14769 reviewer_roster: Vec::new(),
14770 },
14771 sem: Arc::new(Semaphore::new(1)),
14772 pause: Pause::new(),
14773 interrupt: Pause::new(),
14774 };
14775
14776 runner.review_loop().await.expect("review_loop");
14781
14782 assert_eq!(
14783 runner.state.reviews.len(),
14784 1,
14785 "no new round was started on top of the unresolved one: {:?}",
14786 runner.state.reviews
14787 );
14788 let last = &runner.state.reviews[0];
14789 assert_eq!(
14790 last.e2e_status(),
14791 E2eStatus::ResourceBlocked,
14792 "still contended: {last:?}"
14793 );
14794 assert!(
14795 last.verified_at.expect("still known") > first_attempt_at,
14796 "review_loop must have actually retried the check, not left it exactly as found"
14797 );
14798 assert_ne!(
14799 runner.state.status,
14800 RunStatus::Blocked,
14801 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
14802 runner.state.status
14803 );
14804
14805 held.release();
14806 }
14807
14808 #[tokio::test]
14809 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
14810 crate::run::pin_test_home();
14811 let tmp = tempfile::tempdir().expect("tempdir");
14812 let repo = tmp.path().join("repo");
14813 std::fs::create_dir_all(&repo).unwrap();
14814 init_repo(&repo);
14815
14816 let mut config = Config::default();
14817 config.merge.mode = MergeMode::Pr;
14818 config.graph.land = true;
14819 config.graph.land_approval = false;
14820
14821 let mut state = RunState::new(
14822 repo.clone(),
14823 "main".to_owned(),
14824 "deadbeef".to_owned(),
14825 "task".to_owned(),
14826 config,
14827 );
14828 state.candidates = vec![Candidate {
14829 index: 0,
14830 label: 'A',
14831 agent: "alpha".to_owned(),
14832 branch: "does-not-exist".to_owned(),
14833 worktree: repo.clone(),
14834 summary: String::new(),
14835 stat: String::new(),
14836 files: 0,
14837 commits: 0,
14838 empty: false,
14839 failed: None,
14840 verified_noop: None,
14841 duration_ms: 0,
14842 folded: false,
14843 }];
14844 state.tally = Some(Tally {
14845 first_choice: BTreeMap::from([('A', 1)]),
14846 borda: BTreeMap::new(),
14847 winner: 'A',
14848 rankings: 1,
14849 unanimous_initial: true,
14850 deliberated: false,
14851 changed_votes: 0,
14852 unanimous_final: true,
14853 tie_break: None,
14854 judges: 0,
14855 present: 0,
14856 quorum: 0,
14857 met_quorum: true,
14858 uncontested: Some("only candidate A produced a change".to_owned()),
14859 });
14860 state.reviews = vec![ReviewRound {
14861 round: 1,
14862 head: "deadbeef".to_owned(),
14863 verified_head: None,
14864 verified_at: None,
14865 reviews: Vec::new(),
14866 e2e: Vec::new(),
14867 fix: None,
14868 blocking: 0,
14869 answered: 0,
14870 expected: 0,
14871 clean: true,
14872 verify_retried: false,
14873 e2e_deferred: false,
14874 e2e_defer_reason: None,
14875 progressed: false,
14876 vote_split: false,
14877 reconsideration: Vec::new(),
14878 verdict: None,
14879 }];
14880 state.gate = vec![CommandOutcome {
14881 command: "test".to_owned(),
14882 code: Some(0),
14883 output_tail: String::new(),
14884 duration_ms: 0,
14885 resource_blocked: false,
14886 }];
14887 state.gate_ran = true;
14888 state.status = RunStatus::Landing;
14892 state.merge = Some(MergeOutcome {
14893 mode: MergeMode::Pr,
14894 ok: true,
14895 detail: "https://example.invalid/x/y/pull/1".to_owned(),
14896 empty: false,
14897 });
14898
14899 ask_test_home();
14903 state.save().expect("save landing run");
14906 let store = ask::Questions::open();
14907 let q = ask_open_question(&store, &state.id);
14908
14909 let mut runner = Runner {
14910 state,
14911 roles: ResolvedRoles {
14912 implementers: Vec::new(),
14913 judges: Vec::new(),
14914 reviewers: Vec::new(),
14915 fixer: None,
14916 conductor: conductor(),
14917 implementer_roster: Vec::new(),
14918 judge_roster: Vec::new(),
14919 reviewer_roster: Vec::new(),
14920 },
14921 sem: Arc::new(Semaphore::new(1)),
14922 pause: Pause::new(),
14923 interrupt: Pause::new(),
14924 };
14925
14926 runner.execute().await.expect("execute");
14931
14932 assert_eq!(
14933 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14934 Some("https://example.invalid/x/y/pull/1"),
14935 "reentry must not push again or open a second pull request over the \
14936 one `land` is already watching"
14937 );
14938 assert_ne!(
14939 runner.state.status,
14940 RunStatus::Landing,
14941 "land could not actually reach the fake pull request, so it must \
14942 have given up rather than left the run silently parked forever"
14943 );
14944 assert_eq!(runner.state.status, RunStatus::Blocked);
14948 assert!(
14949 store.get(&q.id).unwrap().status.open(),
14950 "Blocked is still alive; settle_questions must have been a no-op here"
14951 );
14952 }
14953
14954 fn state_with_round(round: ReviewRound) -> RunState {
14955 let mut s = RunState::new(
14956 PathBuf::from("/repo"),
14957 "main".to_owned(),
14958 "abc1234".to_owned(),
14959 "add retries".to_owned(),
14960 Config::default(),
14961 );
14962 s.reviews = vec![round];
14963 s
14964 }
14965
14966 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
14967 crate::verdict::Finding {
14968 id: id.to_owned(),
14969 severity,
14970 file: None,
14971 line: None,
14972 title: title.to_owned(),
14973 detail: String::new(),
14974 }
14975 }
14976
14977 #[test]
14978 fn pr_body_names_open_findings_and_declined_ones() {
14979 let round = ReviewRound {
14980 round: 2,
14981 head: "deadbee".to_owned(),
14982 verified_head: None,
14983 verified_at: None,
14984 reviews: vec![ReviewRecord {
14985 attempts: 0,
14986 reviewer: 1,
14987 agent: "alpha".to_owned(),
14988 summary: String::new(),
14989 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
14990 vote: None,
14991 failed: None,
14992 duration_ms: 0,
14993 }],
14994 e2e: vec![CommandOutcome {
14995 command: "cargo test".to_owned(),
14996 code: Some(0),
14997 output_tail: String::new(),
14998 duration_ms: 0,
14999 resource_blocked: false,
15000 }],
15001 verify_retried: false,
15002 e2e_deferred: false,
15003 e2e_defer_reason: None,
15004 fix: Some(FixRecord {
15005 agent: "alpha".to_owned(),
15006 addressed: Vec::new(),
15007 rejected: vec![crate::verdict::Rejection {
15008 id: "R1-1-1".to_owned(),
15009 why: "not reachable from any caller".to_owned(),
15010 }],
15011 notes: String::new(),
15012 committed: true,
15013 failed: None,
15014 duration_ms: 0,
15015 continuation: None,
15016 }),
15017 blocking: 0,
15018 answered: 1,
15019 expected: 1,
15020 clean: false,
15021 progressed: true,
15022 vote_split: false,
15023 reconsideration: Vec::new(),
15024 verdict: None,
15025 };
15026 let state = state_with_round(round);
15027 let body = pr_message(&state, 'A').body;
15028
15029 assert!(body.contains("add retries"), "the task must still be there");
15030 assert!(body.contains("R2-1-1"), "{body}");
15031 assert!(!body.contains("unused import"), "{body}");
15032 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15033 assert!(
15034 !body.contains("not reachable from any caller"),
15035 "the reason it was declined: {body}"
15036 );
15037 }
15038
15039 #[test]
15040 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15041 let round = ReviewRound {
15042 round: 1,
15043 head: "deadbee".to_owned(),
15044 verified_head: None,
15045 verified_at: None,
15046 reviews: vec![ReviewRecord {
15047 attempts: 0,
15048 reviewer: 1,
15049 agent: "alpha".to_owned(),
15050 summary: String::new(),
15051 findings: Vec::new(),
15052 vote: None,
15053 failed: None,
15054 duration_ms: 0,
15055 }],
15056 e2e: Vec::new(),
15057 verify_retried: false,
15058 e2e_deferred: false,
15059 e2e_defer_reason: None,
15060 fix: None,
15061 blocking: 0,
15062 answered: 1,
15063 expected: 1,
15064 clean: true,
15065 progressed: false,
15066 vote_split: false,
15067 reconsideration: Vec::new(),
15068 verdict: None,
15069 };
15070 let state = state_with_round(round);
15071 let body = pr_message(&state, 'A').body;
15072 assert!(!body.contains("Open review findings"), "{body}");
15073 assert!(!body.contains("Declined"), "{body}");
15074 }
15075
15076 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15077 let mut state = RunState::new(
15078 PathBuf::from("/repo"),
15079 "main".to_owned(),
15080 "abc1234".to_owned(),
15081 instruction.to_owned(),
15082 Config::default(),
15083 );
15084 state.candidates.push(Candidate {
15085 index: 0,
15086 label: 'A',
15087 agent: "alpha".to_owned(),
15088 branch: "magi/x/A".to_owned(),
15089 worktree: PathBuf::from("/wt"),
15090 summary: summary.to_owned(),
15091 stat: String::new(),
15092 files: 1,
15093 commits: 1,
15094 empty: false,
15095 failed: None,
15096 verified_noop: None,
15097 folded: false,
15098 duration_ms: 0,
15099 });
15100 state
15101 }
15102
15103 fn review_state(subjects: &[&str]) -> RunState {
15104 let mut state = state_with_summary(
15105 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
15106 "",
15107 );
15108 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
15109 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
15110 state
15111 }
15112
15113 #[tokio::test]
15116 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
15117 ask_test_home();
15118 let tmp = tempfile::tempdir().unwrap();
15119 let repo = tmp.path().join("repo");
15120 std::fs::create_dir_all(&repo).unwrap();
15121 init_repo(&repo);
15122 let origin = tmp.path().join("origin.git");
15123 let g = |dir: &Path, args: &[&str]| {
15124 let out = std::process::Command::new("git")
15125 .args(args)
15126 .current_dir(dir)
15127 .quiet()
15128 .output()
15129 .expect("spawn git");
15130 assert!(
15131 out.status.success(),
15132 "git {args:?}: {}",
15133 String::from_utf8_lossy(&out.stderr)
15134 );
15135 };
15136 g(
15137 tmp.path(),
15138 &[
15139 "clone",
15140 "--bare",
15141 "-q",
15142 repo.to_str().unwrap(),
15143 origin.to_str().unwrap(),
15144 ],
15145 );
15146 g(
15147 &repo,
15148 &["remote", "add", "origin", origin.to_str().unwrap()],
15149 );
15150 let c1 = git::rev_parse(&repo, "main").await.unwrap();
15151
15152 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
15154 g(&repo, &["add", "-A"]);
15155 g(
15156 &repo,
15157 &["commit", "-q", "-m", "chore(deps): update a crate"],
15158 );
15159 g(&repo, &["push", "-q", "origin", "main"]);
15160 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
15161 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
15162 g(&repo, &["add", "-A"]);
15163 g(
15164 &repo,
15165 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
15166 );
15167 g(&repo, &["checkout", "-q", "main"]);
15168
15169 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
15170 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
15171 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
15173 assert_eq!(fallback, c1);
15174
15175 let mut state = review_state(&[
15176 "chore(deps): update a crate",
15177 "fix(daemon): apply a chosen action",
15178 ]);
15179 state.repo = repo.clone();
15180 state.base_branch = "main".to_owned();
15181 state.base_commit = c1;
15182 refresh_reviewed_commits(&mut state, "feat/own").await;
15183 assert_eq!(
15184 state.reviewed_commits,
15185 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
15186 );
15187 assert_eq!(
15188 review_title(&state).as_deref(),
15189 Some("fix(daemon): apply a chosen action")
15190 );
15191 assert_eq!(
15192 leaked_subjects(&state, "feat/own").await,
15193 Some(vec!["chore(deps): update a crate".to_owned()])
15194 );
15195 state.config.merge.remote = "nowhere".to_owned();
15198 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
15199 }
15200
15201 struct SummaryFixture {
15205 _tmp: tempfile::TempDir,
15206 repo: PathBuf,
15207 reply: PathBuf,
15208 runner: Runner,
15209 }
15210
15211 fn summary_fixture() -> SummaryFixture {
15212 ask_test_home();
15213 let tmp = tempfile::tempdir().unwrap();
15214 let repo = tmp.path().join("repo");
15215 std::fs::create_dir_all(&repo).unwrap();
15216 init_repo(&repo);
15217 let origin = tmp.path().join("origin.git");
15218 let g = |args: &[&str]| {
15219 let out = std::process::Command::new("git")
15220 .args(args)
15221 .current_dir(&repo)
15222 .quiet()
15223 .output()
15224 .expect("spawn git");
15225 assert!(
15226 out.status.success(),
15227 "git {args:?}: {}",
15228 String::from_utf8_lossy(&out.stderr)
15229 );
15230 };
15231 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
15232 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
15233 g(&["push", "-q", "origin", "main"]);
15234 g(&["checkout", "-q", "-b", "feat/own"]);
15235 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
15236 g(&["add", "-A"]);
15237 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
15238 let reply = tmp.path().join("reply.txt");
15239 std::fs::write(
15240 &reply,
15241 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
15242 )
15243 .unwrap();
15244 let mut runner = runner_at(RunStatus::Gating);
15245 let mut state = review_state(&["feat(own): add the own file"]);
15246 state.repo = repo.clone();
15247 state.base_branch = "main".to_owned();
15248 state.candidates[0].branch = "feat/own".to_owned();
15249 state.candidates[0].worktree = repo.clone();
15250 let mut author = spec("alpha");
15251 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
15252 author
15253 .env
15254 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
15255 state.config.agents = vec![author];
15256 runner.state = state;
15257 SummaryFixture {
15258 _tmp: tmp,
15259 repo,
15260 reply,
15261 runner,
15262 }
15263 }
15264
15265 async fn posted(f: &mut SummaryFixture) -> PrMessage {
15266 let winner = f.runner.state.candidates[0].clone();
15267 f.runner
15268 .guarded_pr_message(&winner, None, true)
15269 .await
15270 .unwrap()
15271 .expect("an English description never parks")
15272 }
15273
15274 fn clean_round_with_minor_findings() -> ReviewRound {
15275 let mut round = passing_round(2, None);
15276 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
15277 round.reviews = vec![ReviewRecord {
15278 attempts: 0,
15279 reviewer: 1,
15280 agent: "alpha".into(),
15281 summary: "内部レビュー".into(),
15282 findings: vec![
15283 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
15284 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
15285 ],
15286 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
15287 failed: None,
15288 duration_ms: 0,
15289 }];
15290 round
15291 }
15292
15293 #[tokio::test]
15294 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
15295 for review_only in [false, true] {
15296 let mut f = summary_fixture();
15297 if !review_only {
15298 f.runner.state.candidates[0].agent = "alpha".into();
15299 }
15300 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
15301 assert!(f.runner.state.reviews[0].clean);
15302 assert!(f.runner.state.reviews[0].fix.is_none());
15303 assert!(f.runner.state.open_findings().is_empty());
15304 assert_eq!(
15305 crate::pr_outcome::required_finding_ids(&f.runner.state),
15306 ["R2-1-1", "R2-1-2"]
15307 );
15308 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15309 let evidence = crate::pr_outcome::summary_evidence(
15310 &f.runner.state,
15311 crate::pr_outcome::FinalHead::At(&head),
15312 );
15313 assert!(
15314 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
15315 );
15316 let omitted = posted(&mut f).await;
15318 assert!(omitted.body.contains("No description"));
15319 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
15320 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
15321 assert!(f.runner.state.change_summary.is_none());
15322 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
15323 let described = posted(&mut f).await;
15324 assert!(described.body.contains("improve documentation for callers"));
15325 assert!(described.body.contains("remaining edge case"));
15326 assert!(described.body.contains("## Open review findings"));
15327 assert!(!described.body.contains("未修正"));
15328 }
15329 }
15330
15331 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
15332 let mut f = summary_fixture();
15333 if !review_only {
15334 f.runner.state.candidates[0].agent = "alpha".into();
15335 }
15336 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15337 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
15338 fixed
15339 .fix
15340 .as_mut()
15341 .unwrap()
15342 .rejected
15343 .push(crate::verdict::Rejection {
15344 id: "R1-1-1".into(),
15345 why: "呼び出されないコードのため辞退".into(),
15346 });
15347 let mut clean = clean_round_with_minor_findings();
15348 clean.verified_head = Some(head.clone());
15349 f.runner.state.reviews = vec![fixed, clean];
15350 ok_gate(&mut f.runner.state, Some(&head));
15351 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
15352 f.runner.state.config.agents.push(scripted("beta", reply));
15353 f.runner.state.config.roles.language_judge =
15354 Some(crate::config::AgentChoice::One("beta".into()));
15355 f
15356 }
15357
15358 #[tokio::test]
15359 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
15360 for review_only in [false, true] {
15361 let mut f = rewrite_fixture_with_findings(review_only,
15362 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
15363 let m = posted(&mut f).await;
15364 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15365 let expected =
15366 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
15367 assert!(m.body.contains(&expected));
15368 assert!(m.body.contains("## Review rounds"));
15369 assert!(
15370 m.body
15371 .contains("Gate (`verify.gate`): passed on the final commit")
15372 );
15373 assert!(
15374 m.body
15375 .contains("Tests (`verify.e2e`): passed on the final commit")
15376 );
15377 let c = f.runner.state.change_summary.as_ref().unwrap();
15378 assert!(
15379 m.body
15380 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
15381 );
15382 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15383 assert!(crate::pr_outcome::covers_required_findings(
15384 &f.runner.state,
15385 &m.body
15386 ));
15387 assert!(m.body.contains("documentation remains incomplete"));
15388 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
15389 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
15390 assert!(f.runner.state.github_text.is_none());
15391 }
15392 }
15393
15394 #[tokio::test]
15395 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
15396 for review_only in [false, true] {
15397 for reply in [
15398 "TITLE: fix: public description\nA short English description.",
15399 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
15400 ] {
15401 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
15402 let winner = f.runner.state.candidates[0].clone();
15403 let posted = f
15404 .runner
15405 .guarded_pr_message(&winner, None, true)
15406 .await
15407 .unwrap();
15408 assert!(posted.is_none());
15409 assert!(f.runner.state.parked);
15410 let withheld = f.runner.state.github_text.as_ref().unwrap();
15411 assert!(
15412 withheld.rewrite.is_none(),
15413 "an incomplete rewrite must not be selectable"
15414 );
15415 let q = ask::Questions::open()
15416 .get(withheld.question.as_ref().unwrap())
15417 .unwrap();
15418 assert!(
15419 !q.choices
15420 .contains(&crate::github_text::USE_REWRITE.to_owned())
15421 );
15422 assert!(
15423 q.detail.contains("R1-1-1")
15424 && q.detail.contains("R2-1-1")
15425 && q.detail.contains("R2-1-2")
15426 );
15427 }
15428 }
15429 }
15430
15431 #[tokio::test]
15432 async fn japanese_reports_stay_local_in_both_public_run_modes() {
15433 for review_only in [false, true] {
15434 let mut f = summary_fixture();
15435 if !review_only {
15436 f.runner.state.candidates[0].agent = "alpha".into();
15437 }
15438 f.runner.state.instruction = "内部タスクの説明".into();
15439 f.runner.state.candidates[0].summary = "内部の候補報告".into();
15440 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15441 f.runner.state.reviews = vec![
15442 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
15443 round_with_notes(2, Some("二回目の内部修正報告です。")),
15444 passing_round(3, Some(&head)),
15445 ];
15446 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
15447 ok_gate(&mut f.runner.state, Some(&head));
15448 let captured = f.repo.join("public-prompt.txt");
15449 let author = &mut f.runner.state.config.agents[0];
15450 author.command = vec![
15451 "sh".into(),
15452 "-c".into(),
15453 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
15454 ];
15455 author.env.insert(
15456 "PROMPT_FILE".into(),
15457 captured.to_string_lossy().into_owned(),
15458 );
15459 let m = posted(&mut f).await;
15460 assert!(m.body.contains("Adds `own.txt`"));
15461 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
15462 assert!(m.body.contains("R1-1-1"));
15463 assert!(
15464 m.body
15465 .contains("Gate (`verify.gate`): passed on the final commit")
15466 );
15467 assert!(
15468 m.body
15469 .contains("Tests (`verify.e2e`): passed on the final commit")
15470 );
15471 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15472 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
15473 let prompt = std::fs::read_to_string(captured).unwrap();
15474 assert!(prompt.contains(&format!(
15475 "git diff {} {head}",
15476 f.runner.state.change_summary.as_ref().unwrap().base
15477 )));
15478 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
15479 assert_eq!(
15480 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
15481 "二回目の内部修正報告です。"
15482 );
15483 }
15484 }
15485
15486 #[tokio::test]
15487 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
15488 for review_only in [false, true] {
15489 let mut f = summary_fixture();
15490 if !review_only {
15491 f.runner.state.candidates[0].agent = "alpha".into();
15492 }
15493 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
15494 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
15495 std::fs::write(&f.reply, "no usable description").unwrap();
15496 let failed = posted(&mut f).await;
15497 assert!(
15498 failed
15499 .body
15500 .contains("No description of what this change does could be generated")
15501 );
15502 assert!(failed.body.is_ascii());
15503 assert!(f.runner.state.github_text.is_none());
15504 assert!(f.runner.state.change_summary.is_none());
15505 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15506 let evidence = crate::pr_outcome::summary_evidence(
15507 &f.runner.state,
15508 crate::pr_outcome::FinalHead::At(&head),
15509 );
15510 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
15511 assert!(
15512 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
15513 .exists()
15514 );
15515 f.runner.state =
15516 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
15517 std::fs::write(
15518 &f.reply,
15519 "TITLE: feat(own): add the own file\nAdds the committed own file.",
15520 )
15521 .unwrap();
15522 let retried = posted(&mut f).await;
15523 assert!(retried.body.contains("Adds the committed own file."));
15524 assert_eq!(
15525 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
15526 "日本語の修正報告"
15527 );
15528 }
15529 }
15530
15531 #[tokio::test]
15532 async fn same_head_verification_and_findings_invalidate_public_prose() {
15533 let mut f = summary_fixture();
15534 posted(&mut f).await;
15535 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15536 let original = f
15537 .runner
15538 .state
15539 .change_summary
15540 .as_ref()
15541 .unwrap()
15542 .evidence
15543 .clone();
15544 ok_gate(&mut f.runner.state, Some(&head));
15545 std::fs::write(
15546 &f.reply,
15547 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
15548 )
15549 .unwrap();
15550 let verified = posted(&mut f).await;
15551 assert!(verified.body.contains("now verified"));
15552 assert_ne!(
15553 f.runner.state.change_summary.as_ref().unwrap().evidence,
15554 original
15555 );
15556 let mut round = round_with_notes(1, Some("内部報告"));
15557 round.reviews.push(ReviewRecord {
15558 attempts: 0,
15559 reviewer: 1,
15560 agent: "alpha".into(),
15561 summary: "内部レビュー".into(),
15562 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
15563 vote: None,
15564 failed: None,
15565 duration_ms: 0,
15566 });
15567 round
15568 .fix
15569 .as_mut()
15570 .unwrap()
15571 .rejected
15572 .push(crate::verdict::Rejection {
15573 id: "R1-1-1".into(),
15574 why: "呼び出し元から到達できない".into(),
15575 });
15576 f.runner.state.reviews = vec![round];
15577 let omitted = posted(&mut f).await;
15579 assert!(omitted.body.contains("No description"));
15580 assert!(omitted.body.contains("R1-1-1"));
15581 assert!(!omitted.body.contains("到達"));
15582 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
15583 let described = posted(&mut f).await;
15584 assert!(described.body.contains("unreachable from callers"));
15585 assert!(described.body.contains("R1-1-1"));
15586 assert!(!described.body.contains("到達"));
15587 let evidence = f
15588 .runner
15589 .state
15590 .change_summary
15591 .as_ref()
15592 .unwrap()
15593 .evidence
15594 .clone();
15595 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
15596 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
15597 assert!(posted(&mut f).await.body.contains("rationale updated"));
15598 assert_ne!(
15599 f.runner.state.change_summary.as_ref().unwrap().evidence,
15600 evidence
15601 );
15602 f.runner
15604 .state
15605 .change_summary
15606 .as_mut()
15607 .unwrap()
15608 .evidence
15609 .clear();
15610 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
15611 assert!(posted(&mut f).await.body.contains("current specification"));
15612 }
15613
15614 #[tokio::test]
15615 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
15616 let mut f = summary_fixture();
15617 let m = posted(&mut f).await;
15618 assert_eq!(m.title, "feat(own): add the own file");
15619 assert!(
15620 m.body
15621 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
15622 "{}",
15623 m.body
15624 );
15625 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
15626 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
15627 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
15628 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
15629 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
15630 let kept = f.runner.state.change_summary.as_ref().unwrap();
15631 assert_eq!(
15632 (kept.head.as_str(), kept.base.as_str()),
15633 (head.as_str(), base.as_str())
15634 );
15635 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
15636 }
15637
15638 #[tokio::test]
15639 async fn review_only_summary_receives_the_configured_github_language() {
15640 let mut f = summary_fixture();
15641 f.runner.state.config.graph.github_language = "ja".into();
15642 let captured = f.repo.join("summary-prompt.txt");
15643 let author = &mut f.runner.state.config.agents[0];
15644 author.command = vec![
15645 "sh".into(),
15646 "-c".into(),
15647 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
15648 ];
15649 author.env.insert(
15650 "PROMPT_FILE".into(),
15651 captured.to_string_lossy().into_owned(),
15652 );
15653 std::fs::write(
15654 &f.reply,
15655 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
15656 )
15657 .unwrap();
15658 let m = posted(&mut f).await;
15659 let prompt = std::fs::read_to_string(captured).unwrap();
15660 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
15661 assert!(!prompt.contains("in English"));
15662 assert_eq!(m.title, "feat(own): ファイルを追加");
15663 assert!(m.body.contains("own.txt を追加します。"));
15664 }
15665
15666 #[tokio::test]
15667 async fn review_fixes_come_after_the_whole_change_summary() {
15668 let mut f = summary_fixture();
15669 f.runner.state.reviews = vec![
15670 round_with_notes(1, Some("handled the PATH case")),
15671 round_with_notes(2, None),
15672 ];
15673 let m = posted(&mut f).await;
15674 let summary = m.body.find("## Summary").expect("summary");
15675 let fixes = m.body.find("## Review rounds").expect("review rounds");
15676 assert!(summary < fixes, "{}", m.body);
15677 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
15678 assert!(m.body.contains("## Final verification"), "{}", m.body);
15679 }
15680
15681 #[tokio::test]
15682 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
15683 let mut f = summary_fixture();
15684 let first = posted(&mut f).await;
15685 let first_head = f.runner.state.change_summary.clone().unwrap().head;
15686 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
15688 let again = posted(&mut f).await;
15689 assert_eq!((again.title, again.body), (first.title, first.body));
15690
15691 let g = |args: &[&str]| {
15692 let out = std::process::Command::new("git")
15693 .args(args)
15694 .current_dir(&f.repo)
15695 .quiet()
15696 .output()
15697 .expect("spawn git");
15698 assert!(out.status.success(), "git {args:?}");
15699 };
15700 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
15701 g(&["add", "-A"]);
15702 g(&["commit", "-q", "-m", "fix(own): handle more"]);
15703 let second = posted(&mut f).await;
15704 let kept = f.runner.state.change_summary.clone().unwrap();
15705 assert_ne!(kept.head, first_head);
15706 assert!(second.body.contains("Not read yet."), "{}", second.body);
15707 assert!(!second.body.contains("so the first summary is read"));
15708 assert_eq!(second.title, "feat(own): other");
15709 }
15710
15711 #[tokio::test]
15712 async fn an_unavailable_summary_is_stated_and_never_invented() {
15713 let mut f = summary_fixture();
15714 std::fs::write(&f.reply, "no title line here").unwrap();
15715 let m = posted(&mut f).await;
15716 assert!(
15717 m.body
15718 .contains("No description of what this change does could be generated"),
15719 "{}",
15720 m.body
15721 );
15722 assert!(m.body.contains("internal reports remain in local records"));
15723 assert!(f.runner.state.change_summary.is_none());
15724 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
15725 let winner = f.runner.state.candidates[0].clone();
15727 let preview = f
15728 .runner
15729 .guarded_pr_message(&winner, None, false)
15730 .await
15731 .unwrap()
15732 .unwrap();
15733 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
15734 }
15735
15736 #[test]
15737 fn pr_message_review_single_commit_uses_its_subject() {
15738 let state = review_state(&["feat(nats): per-role user"]);
15739 let m = pr_message(&state, 'A');
15740 assert_eq!(m.title, "feat(nats): per-role user");
15741 assert!(!m.body.contains("Review the work already"), "{}", m.body);
15742 assert!(m.body.contains("## Commits under review"), "{}", m.body);
15743 }
15744
15745 #[test]
15746 fn pr_message_review_multi_commit_takes_the_oldest() {
15747 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
15748 let m = pr_message(&state, 'A');
15749 assert_eq!(m.title, "feat: the change");
15750 for s in ["feat: the change", "fix: typo", "fix: again"] {
15751 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
15752 }
15753 }
15754
15755 #[test]
15756 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
15757 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
15758 let mut state = review_state(&[first, "fix: later fixup"]);
15759 state.candidates[0].branch = "機能/ログイン".to_owned();
15760 let m = pr_message(&state, 'A');
15761 assert!(
15762 m.title.starts_with("chore: land candidate A of run"),
15763 "{}",
15764 m.title
15765 );
15766 }
15767 }
15768
15769 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
15770 BranchFacts {
15771 commits: commits
15772 .iter()
15773 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
15774 .collect(),
15775 stat: stat.to_owned(),
15776 }
15777 }
15778
15779 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
15780 let mut r = review_round(true, 0, 1, 1, true, true);
15781 r.round = round;
15782 r.fix = notes.map(|n| FixRecord {
15783 agent: "fixer".to_owned(),
15784 addressed: Vec::new(),
15785 rejected: Vec::new(),
15786 notes: n.to_owned(),
15787 committed: true,
15788 failed: None,
15789 duration_ms: 0,
15790 continuation: None,
15791 });
15792 r
15793 }
15794
15795 #[test]
15796 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
15797 let state = review_state(&["ignored"]);
15798 let f = facts(
15799 &[
15800 (
15801 "fix(login): resolve PATH on macOS",
15802 "Login shells skip rc files.",
15803 ),
15804 ("fix: address review", ""),
15805 ],
15806 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
15807 );
15808 let m = pr_message_with(&state, 'A', Some(&f));
15809 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
15810 assert!(
15811 m.body
15812 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
15813 "{}",
15814 m.body
15815 );
15816 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
15817 assert!(m.body.contains("## Diff stat"), "{}", m.body);
15818 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
15819 for banned in [
15820 "Review the work already",
15821 "no task statement",
15822 "Original task",
15823 ] {
15824 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
15825 }
15826 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
15827 }
15828
15829 #[test]
15830 fn pr_message_review_truncates_a_huge_first_commit_body() {
15831 let state = review_state(&["ignored"]);
15832 let f = facts(
15833 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
15834 "s",
15835 );
15836 let m = pr_message_with(&state, 'A', Some(&f));
15837 assert!(m.body.len() < 30_000, "{}", m.body.len());
15838 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
15839 assert!(
15840 m.body.contains("- ... 1 more commit(s)"),
15841 "{}",
15842 m.body.len()
15843 );
15844 assert!(m.body.ends_with("magi:candidate-a\n"));
15845 }
15846
15847 #[test]
15848 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
15849 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
15850 assert_eq!(m.title, "feat: x");
15851 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
15852 assert!(!m.body.contains("Diff stat"), "{}", m.body);
15853 assert!(!m.body.contains("no task statement"), "{}", m.body);
15854 }
15855
15856 #[test]
15857 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
15858 let mut state = review_state(&["日本語の件名"]);
15859 state.candidates[0].branch = "fix/macos-login-path".to_owned();
15860 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
15861 state.candidates[0].branch = "機能/ログイン".to_owned();
15862 assert!(
15863 pr_message(&state, 'A')
15864 .title
15865 .starts_with("chore: land candidate A")
15866 );
15867 }
15868
15869 #[test]
15870 fn pr_message_review_fixes_survive_a_clean_final_round() {
15871 let mut state = review_state(&["feat: x"]);
15872 state.reviews = vec![
15873 round_with_notes(1, Some("handled the PATH case")),
15874 round_with_notes(2, None),
15875 ];
15876 let body = pr_message(&state, 'A').body;
15877 assert!(
15878 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
15879 "{body}"
15880 );
15881 assert!(body.contains("### Round 1"), "{body}");
15882
15883 state.reviews = vec![
15884 round_with_notes(1, Some("first fix")),
15885 round_with_notes(2, Some("")),
15886 round_with_notes(3, Some("second fix")),
15887 round_with_notes(4, None),
15888 ];
15889 let body = pr_message(&state, 'A').body;
15890 assert!(
15891 body.contains("### Round 1") && !body.contains("first fix"),
15892 "{body}"
15893 );
15894 assert!(
15895 body.contains("### Round 3") && !body.contains("second fix"),
15896 "{body}"
15897 );
15898 assert!(body.contains("### Round 2"), "{body}");
15899 }
15900
15901 const FINAL: &str = "abcdef1234567890";
15902
15903 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
15904 let mut r = round_with_notes(round, None);
15905 r.verified_head = verified.map(str::to_owned);
15906 r
15907 }
15908
15909 fn ok_gate(state: &mut RunState, head: Option<&str>) {
15910 state.gate = vec![CommandOutcome {
15911 command: "gate".to_owned(),
15912 code: Some(0),
15913 output_tail: String::new(),
15914 duration_ms: 0,
15915 resource_blocked: false,
15916 }];
15917 state.gate_ran = true;
15918 state.gate_head = head.map(str::to_owned);
15919 }
15920
15921 fn body_at(state: &RunState) -> String {
15922 pr_message_raw(
15923 state,
15924 'A',
15925 None,
15926 crate::pr_outcome::FinalHead::At(FINAL),
15927 ChangeNote::NotAsked,
15928 )
15929 .body
15930 }
15931
15932 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
15933
15934 #[test]
15935 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
15936 let mut state = review_state(&["feat: x"]);
15937 state.reviews = vec![
15938 round_with_notes(1, Some(SANDBOXED)),
15939 passing_round(2, Some(FINAL)),
15940 ];
15941 ok_gate(&mut state, Some(FINAL));
15942 let body = body_at(&state);
15943 assert!(!body.contains("Handled the PATH case."), "{body}");
15944 assert!(!body.contains("could not commit"), "{body}");
15945 assert!(!body.contains("could not run"), "{body}");
15946 assert!(
15947 body.contains("The fix is committed on the branch"),
15948 "{body}"
15949 );
15950 assert!(!body.contains("sandbox"), "{body}");
15951 assert!(
15952 body.contains("Tests (`verify.e2e`): passed on the final commit"),
15953 "{body}"
15954 );
15955 assert!(
15956 body.contains("Gate (`verify.gate`): passed on the final commit"),
15957 "{body}"
15958 );
15959 assert!(
15961 state.reviews[0]
15962 .fix
15963 .as_ref()
15964 .unwrap()
15965 .notes
15966 .contains("could not commit")
15967 );
15968 }
15969
15970 #[test]
15971 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
15972 let mut state = review_state(&["feat: x"]);
15973 let mut r1 = round_with_notes(1, Some(SANDBOXED));
15974 r1.fix.as_mut().unwrap().committed = false;
15975 let mut r2 = passing_round(2, Some(FINAL));
15976 r2.e2e[0].code = Some(1);
15977 state.reviews = vec![r1, r2];
15978 let body = body_at(&state);
15979 assert!(!body.contains("could not commit"), "{body}");
15980 assert!(!body.contains("could not run"), "{body}");
15981 assert!(body.contains("This round produced no commit"), "{body}");
15982 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
15983 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
15984 assert!(!body.contains("passed"), "{body}");
15985 }
15986
15987 #[test]
15988 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
15989 let mut state = review_state(&["feat: x"]);
15990 state.reviews = vec![
15991 round_with_notes(1, Some(SANDBOXED)),
15992 passing_round(2, Some("0123456789abcdef")),
15993 ];
15994 ok_gate(&mut state, Some("0123456789abcdef"));
15995 let body = body_at(&state);
15996 assert!(!body.contains("HTTP tests could not run"), "{body}");
15997 assert!(!body.contains("could not commit"), "{body}");
15999 assert!(
16000 body.contains("passed on an earlier commit (`0123456`)"),
16001 "{body}"
16002 );
16003 assert!(!body.contains("passed on the final commit"), "{body}");
16004
16005 ok_gate(&mut state, None);
16007 let body = body_at(&state);
16008 assert!(
16009 body.contains("passed on a commit that was not recorded"),
16010 "{body}"
16011 );
16012 }
16013
16014 #[test]
16015 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16016 let state = state_with_summary(
16017 "add retries to the client",
16018 "TITLE: feat: retries\n\nDid it.",
16019 );
16020 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16021 assert_eq!(m.title, "feat: retries");
16022 assert!(
16023 m.body.contains("<summary>Original task</summary>"),
16024 "{}",
16025 m.body
16026 );
16027 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16028 assert!(
16029 m.body
16030 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16031 "{}",
16032 m.body
16033 );
16034 }
16035
16036 #[test]
16037 fn pr_message_review_bounds_a_long_english_subject() {
16038 let long = format!("feat: {}", "word ".repeat(100));
16039 let m = pr_message(&review_state(&[&long]), 'A');
16040 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16041 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16042 }
16043
16044 #[test]
16045 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16046 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16047 let m = pr_message(&state, 'A');
16048 assert_eq!(m.title, "add retries");
16049 assert!(m.body.contains("<summary>Original task</summary>"));
16050 assert!(!m.body.contains("Commits under review"));
16051 assert_eq!(landing_subject_source(&state), state.instruction);
16052 }
16053
16054 #[test]
16055 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16056 let state = review_state(&["feat: the change", "fix: typo"]);
16057 let source = landing_subject_source(&state);
16058 assert_eq!(land::merge_subject("", &source), "feat: the change");
16059 assert_eq!(
16060 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16061 "feat: the change"
16062 );
16063 assert_eq!(
16065 land::merge_subject("feat: renamed by hand", &source),
16066 "feat: renamed by hand"
16067 );
16068 let blank = review_state(&["日本語"]);
16069 assert!(
16070 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16071 );
16072 }
16073
16074 #[test]
16075 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16076 let state = review_state(&["feat: the change"]);
16077 let source = landing_subject_source(&state);
16078 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16079 assert_eq!(
16080 land::merge_subject(landing_title(&state, old), &source),
16081 "feat: the change"
16082 );
16083 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16084 let task = state_with_summary("add retries", "");
16085 assert_eq!(landing_title(&task, old), old);
16086 }
16087
16088 #[test]
16089 fn pr_message_describes_the_change_not_the_task() {
16090 let state = state_with_summary(
16091 "今回やってほしいこと: results projector を直す",
16092 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
16093 );
16094 let m = pr_message(&state, 'A');
16095 assert_eq!(m.title, "fix(web): batch the runs list reads");
16096 assert!(
16097 m.body.starts_with("## Summary\n\n- reads run.json once"),
16098 "{}",
16099 m.body
16100 );
16101 assert!(!m.body.contains("TITLE:"), "{}", m.body);
16102 let task_at = m.body.find("今回やってほしいこと").unwrap();
16103 let details_at = m.body.find("<details>").unwrap();
16104 assert!(
16105 details_at < task_at,
16106 "the task lives inside <details>: {}",
16107 m.body
16108 );
16109 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
16110 assert!(m.body.contains("magi:candidate-a"));
16111 }
16112
16113 #[test]
16114 fn pr_message_falls_back_to_the_task_without_a_title_line() {
16115 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
16116 let m = pr_message(&state, 'A');
16117 assert_eq!(m.title, "add retries");
16118 assert!(
16119 m.body.contains("## Summary\n\n- did some things"),
16120 "{}",
16121 m.body
16122 );
16123
16124 let none = RunState::new(
16125 PathBuf::from("/repo"),
16126 "main".to_owned(),
16127 "abc1234".to_owned(),
16128 "add retries".to_owned(),
16129 Config::default(),
16130 );
16131 let m = pr_message(&none, 'A');
16132 assert_eq!(m.title, "add retries");
16133 assert!(!m.body.contains("## Summary"), "{}", m.body);
16134 }
16135
16136 #[test]
16137 fn pr_message_refuses_the_candidate_commit_subject() {
16138 for bad in [
16139 "TITLE: magi: candidate A (uncommitted work)",
16140 "TITLE: chore: stuff (uncommitted work)",
16141 "TITLE: ",
16142 ] {
16143 let state = state_with_summary("add retries", bad);
16144 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
16145 }
16146 }
16147
16148 #[test]
16149 fn pr_message_bounds_a_very_long_task_and_title() {
16150 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
16151 let state = state_with_summary(&long, "- nothing");
16152 let m = pr_message(&state, 'A');
16153 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16154 assert!(!m.title.contains('\n'));
16155
16156 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
16157 let m = pr_message(&state, 'A');
16158 assert!(m.title.starts_with("feat: "));
16159 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16160 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
16161 }
16162
16163 fn long_title_of(instruction: &str) -> String {
16164 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
16165 }
16166
16167 #[test]
16168 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
16169 let first = "Make the landing path keep a readable title for long tasks";
16170 let line = format!(
16171 "{first}. {}",
16172 "And then keep going with more words ".repeat(20)
16173 );
16174 let t = long_title_of(&line);
16175 assert_eq!(t, first);
16176 assert!(!t.starts_with("chore: land"));
16177 }
16178
16179 #[test]
16180 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
16181 let line = "word ".repeat(200);
16182 let t = long_title_of(&line);
16183 assert!(t.ends_with("word..."), "{t}");
16184 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
16185 }
16186
16187 #[test]
16188 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
16189 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
16190 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
16191 }
16192 }
16193
16194 #[test]
16195 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
16196 let mut state = state_with_summary("再試行を追加する", "- nothing");
16197 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
16198 state.config.graph.github_language = "ja".into();
16199 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
16200 }
16201
16202 #[test]
16203 fn pr_message_title_limit_is_exact() {
16204 let at = "a".repeat(PR_TITLE_MAX);
16205 assert_eq!(long_title_of(&at), at);
16206 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
16207 assert!(over.ends_with("..."), "{over}");
16208 assert_eq!(over.chars().count(), PR_TITLE_MAX);
16209 }
16210
16211 #[test]
16212 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
16213 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
16214 let t = long_title_of(&line);
16215 assert!(t.ends_with("..."), "{t}");
16216 assert!(t.is_ascii(), "{t}");
16217 }
16218
16219 #[test]
16220 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
16221 let line = format!(
16222 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
16223 "plus more filler words ".repeat(20)
16224 );
16225 let t = long_title_of(&line);
16226 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
16227 }
16228
16229 #[test]
16230 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
16231 let line = format!("\n\n# {}", "title words ".repeat(40));
16232 let state = state_with_summary(&line, "- nothing");
16233 let m = pr_message(&state, 'A');
16234 assert!(m.title.starts_with("title words"), "{}", m.title);
16235 assert_eq!(
16236 land::merge_subject(&m.title, &landing_subject_source(&state)),
16237 m.title
16238 );
16239 assert_eq!(
16241 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
16242 "feat: renamed by hand"
16243 );
16244 }
16245
16246 #[tokio::test]
16247 async fn github_text_rewrite_is_bounded_and_falls_back() {
16248 crate::run::pin_test_home();
16249 for (reply, accepted) in [
16250 (
16251 "TITLE: fix: retries\nAdd retries for failed requests.",
16252 true,
16253 ),
16254 (
16255 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
16256 false,
16257 ),
16258 (
16259 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
16260 true,
16261 ),
16262 (
16263 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
16264 true,
16265 ),
16266 (
16267 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
16268 true,
16269 ),
16270 (
16271 "TITLE: fix: retries\n日本語の説明です。 token=secret",
16272 false,
16273 ),
16274 ("TITLE: fix: retries\n", false),
16275 ] {
16276 let dir = tempfile::tempdir().unwrap();
16277 let mut runner = runner_at(RunStatus::Gating);
16278 runner.state = state_with_summary(
16279 "add retries",
16280 "TITLE: fix: retries\n日本語の説明を書きます。",
16281 );
16282 runner.state.repo = dir.path().to_owned();
16283 runner.state.candidates[0].worktree = dir.path().to_owned();
16284 let mut author = spec("alpha");
16285 author.command = vec![
16286 "sh".into(),
16287 "-c".into(),
16288 "printf '%s' \"$REWRITE_REPLY\"".into(),
16289 ];
16290 author.env.insert("REWRITE_REPLY".into(), reply.into());
16291 runner.state.config.agents = vec![author];
16292 seed_public_summary(&mut runner);
16293 let winner = runner.state.candidates[0].clone();
16294 let first = runner
16295 .guarded_pr_message(&winner, None, true)
16296 .await
16297 .unwrap();
16298 let message = if accepted {
16299 first.expect("a passing rewrite never asks")
16300 } else {
16301 assert!(first.is_none());
16303 assert!(runner.state.parked);
16304 let store = ask::Questions::open();
16305 let asked: Vec<_> = store
16306 .list()
16307 .into_iter()
16308 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16309 .collect();
16310 assert_eq!(asked.len(), 1);
16311 assert!(!asked[0].detail.contains("secret"));
16312 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
16313 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
16314 assert_eq!(
16315 asked[0]
16316 .choices
16317 .contains(&crate::github_text::USE_REWRITE.to_owned()),
16318 rewrite.is_some()
16319 );
16320 store
16323 .update(&asked[0].id, |q| {
16324 q.abandon("test");
16325 Ok(())
16326 })
16327 .unwrap();
16328 let again = runner
16329 .guarded_pr_message(&winner, None, true)
16330 .await
16331 .unwrap()
16332 .expect("an abandoned question falls back");
16333 let n = store
16334 .list()
16335 .into_iter()
16336 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16337 .count();
16338 assert_eq!(n, 1, "asked once per rejected text");
16339 again
16340 };
16341 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
16342 assert!(!message.body.contains("secret"));
16343 assert!(!message.body.contains("prefix"));
16344 assert!(!message.title.contains("secret"));
16345 assert_eq!(
16346 message.body.contains("Add retries for failed requests."),
16347 accepted
16348 );
16349 assert_eq!(
16350 runner.state.seats["github-text-rewrite"].turns, 1,
16351 "only one rewrite invocation"
16352 );
16353 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
16354 if accepted {
16355 assert!(runner.state.github_text.is_none());
16356 assert!(!runner.state.parked);
16357 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
16358 assert!(message.body.contains("## Final verification"));
16359 assert!(message.body.contains("Evidence: final diff"));
16360 assert!(
16361 message
16362 .body
16363 .contains(&format!("magi:run/{}", runner.state.id))
16364 );
16365 }
16366 if !accepted {
16367 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
16368 assert!(
16369 message
16370 .body
16371 .contains(&format!("magi:run/{}", runner.state.id))
16372 );
16373 }
16374 }
16375 }
16376
16377 fn scripted(id: &str, reply: &str) -> AgentSpec {
16378 let mut a = spec(id);
16379 a.command = vec![
16380 "sh".into(),
16381 "-c".into(),
16382 "printf '%s' \"$REWRITE_REPLY\"".into(),
16383 ];
16384 a.env.insert("REWRITE_REPLY".into(), reply.into());
16385 a
16386 }
16387
16388 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
16389 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
16390
16391 fn seed_public_summary(runner: &mut Runner) {
16393 let repo = &runner.state.repo;
16394 let g = |args: &[&str]| {
16395 let out = std::process::Command::new("git")
16396 .args(args)
16397 .current_dir(repo)
16398 .quiet()
16399 .output()
16400 .unwrap();
16401 assert!(out.status.success(), "git {args:?}");
16402 String::from_utf8_lossy(&out.stdout).trim().to_owned()
16403 };
16404 g(&["init", "-q", "-b", "main"]);
16405 g(&[
16406 "-c",
16407 "user.name=Test",
16408 "-c",
16409 "user.email=test@example.invalid",
16410 "commit",
16411 "-q",
16412 "--allow-empty",
16413 "-m",
16414 "feat: fixture",
16415 ]);
16416 let head = g(&["rev-parse", "HEAD"]);
16417 g(&["update-ref", "refs/remotes/origin/main", &head]);
16418 let evidence = crate::pr_outcome::summary_evidence(
16419 &runner.state,
16420 crate::pr_outcome::FinalHead::At(&head),
16421 );
16422 let generated = &runner.state.candidates[0].summary;
16423 runner.state.change_summary = Some(crate::run::ChangeSummary {
16424 base: head.clone(),
16425 head,
16426 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
16427 title: summary_title(generated).unwrap(),
16428 body: summary_without_title(generated),
16429 });
16430 }
16431
16432 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
16433 let mut runner = runner_at(RunStatus::Gating);
16434 runner.state = state_with_summary("add retries", JA_SUMMARY);
16435 if review_only {
16436 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16437 }
16438 runner.state.repo = dir.to_owned();
16439 runner.state.candidates[0].worktree = dir.to_owned();
16440 runner.state.config.agents = agents;
16441 seed_public_summary(&mut runner);
16442 runner
16443 }
16444
16445 fn gate_questions(runner: &Runner) -> usize {
16446 ask::Questions::open()
16447 .list()
16448 .into_iter()
16449 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
16450 .count()
16451 }
16452
16453 #[tokio::test]
16454 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
16455 crate::run::pin_test_home();
16456 let dir = tempfile::tempdir().unwrap();
16457 let mut runner = rewrite_runner(
16458 dir.path(),
16459 true,
16460 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
16461 );
16462 runner.state.config.roles.language_judge =
16463 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
16464 let winner = runner.state.candidates[0].clone();
16465 let msg = runner
16466 .guarded_pr_message(&winner, None, true)
16467 .await
16468 .unwrap()
16469 .expect("an English rewrite posts without asking");
16470 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
16471 assert!(msg.body.contains("<details>kept</details>"));
16472 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
16473 assert_eq!(gate_questions(&runner), 0);
16474 assert!(runner.state.github_text.is_none());
16475 assert!(!runner.state.seats.contains_key("impl-A"));
16476 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16477 }
16478
16479 #[tokio::test]
16480 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
16481 crate::run::pin_test_home();
16482 let dir = tempfile::tempdir().unwrap();
16483 let mut runner = rewrite_runner(
16484 dir.path(),
16485 false,
16486 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
16487 );
16488 runner.state.config.roles.language_judge =
16489 Some(crate::config::AgentChoice::One("beta".into()));
16490 runner.state.seats.insert(
16491 "impl-A".into(),
16492 SeatState::new("impl-A", "alpha", runner.state.seed),
16493 );
16494 let winner = runner.state.candidates[0].clone();
16495 let msg = runner
16496 .guarded_pr_message(&winner, None, true)
16497 .await
16498 .unwrap()
16499 .expect("rewritten");
16500 assert!(msg.body.contains("<details>kept</details>"));
16501 assert_eq!(runner.state.seats["impl-A"].turns, 0);
16502 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16503 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
16504 }
16505
16506 #[tokio::test]
16507 async fn rewrite_reuses_a_resumable_implementer_session() {
16508 crate::run::pin_test_home();
16509 let dir = tempfile::tempdir().unwrap();
16510 let mut runner = rewrite_runner(
16511 dir.path(),
16512 false,
16513 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
16514 );
16515 runner.state.config.roles.language_judge =
16516 Some(crate::config::AgentChoice::One("beta".into()));
16517 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
16518 seat.turns = 1;
16519 runner.state.seats.insert("impl-A".into(), seat);
16520 let winner = runner.state.candidates[0].clone();
16521 runner
16522 .guarded_pr_message(&winner, None, true)
16523 .await
16524 .unwrap()
16525 .expect("rewritten");
16526 assert_eq!(runner.state.seats["impl-A"].turns, 2);
16527 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
16528 }
16529
16530 #[tokio::test]
16531 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
16532 crate::run::pin_test_home();
16533 let dir = tempfile::tempdir().unwrap();
16534 let mut broken = spec("bad");
16535 broken.command = vec!["false".into()];
16536 let mut runner = rewrite_runner(
16537 dir.path(),
16538 true,
16539 vec![
16540 broken,
16541 scripted("noformat", "Add retries."),
16542 scripted("good", EN_REWRITE),
16543 ],
16544 );
16545 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
16546 "bad".into(),
16547 "noformat".into(),
16548 "good".into(),
16549 ]));
16550 let winner = runner.state.candidates[0].clone();
16551 let msg = runner
16552 .guarded_pr_message(&winner, None, true)
16553 .await
16554 .unwrap()
16555 .expect("the third agent rewrites");
16556 assert!(msg.body.contains("`src/net.rs`"));
16557 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
16558 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
16559 }
16560
16561 #[tokio::test]
16562 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
16563 crate::run::pin_test_home();
16564 for (agents, role, saved) in [
16566 (
16567 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
16568 Some("beta"),
16569 true,
16570 ),
16571 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
16572 (Vec::new(), None, false),
16573 ] {
16574 let dir = tempfile::tempdir().unwrap();
16575 let mut runner = rewrite_runner(dir.path(), true, agents);
16576 runner.state.config.roles.language_judge =
16577 role.map(|r| crate::config::AgentChoice::One(r.into()));
16578 let winner = runner.state.candidates[0].clone();
16579 let first = runner
16580 .guarded_pr_message(&winner, None, true)
16581 .await
16582 .unwrap();
16583 assert!(first.is_none(), "withheld text parks the run");
16584 assert!(runner.state.parked);
16585 assert_eq!(gate_questions(&runner), 1);
16586 let g = runner.state.github_text.as_ref().unwrap();
16587 assert_eq!(g.rewrite.is_some(), saved);
16588 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
16589 }
16590 }
16591
16592 #[tokio::test]
16593 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
16594 crate::run::pin_test_home();
16595 let dir = tempfile::tempdir().unwrap();
16596 let mut runner = runner_at(RunStatus::Gating);
16597 runner.state =
16598 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
16599 runner.state.repo = dir.path().to_owned();
16600 runner.state.candidates[0].worktree = dir.path().to_owned();
16601 let mut author = spec("alpha");
16602 author.command = vec![
16603 "sh".into(),
16604 "-c".into(),
16605 "printf '%s' \"$REWRITE_REPLY\"".into(),
16606 ];
16607 author.env.insert(
16608 "REWRITE_REPLY".into(),
16609 format!(
16610 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
16611 "English sentence. ".repeat(600)
16612 ),
16613 );
16614 runner.state.config.agents = vec![author];
16615 seed_public_summary(&mut runner);
16616 let winner = runner.state.candidates[0].clone();
16617 assert!(
16618 runner
16619 .guarded_pr_message(&winner, None, true)
16620 .await
16621 .unwrap()
16622 .is_none()
16623 );
16624 let record = runner.state.github_text.clone().unwrap();
16625 let rewrite = record.rewrite.clone().unwrap();
16626 assert_eq!(rewrite.failed, ["body-language"]);
16627 assert!(
16628 rewrite
16629 .body
16630 .ends_with(&format!("magi:run/{}", runner.state.id))
16631 );
16632 let store = ask::Questions::open();
16633 let q = store.get(record.question.as_ref().unwrap()).unwrap();
16634 assert!(
16635 q.choices
16636 .contains(&crate::github_text::USE_REWRITE.to_owned())
16637 );
16638 for label in [
16639 "ORIGINAL",
16640 "FALLBACK",
16641 "REWRITE",
16642 "Rules still failed: body-language",
16643 "truncated",
16644 ] {
16645 assert!(q.detail.contains(label), "{label}");
16646 }
16647 let artifact = std::fs::read_to_string(crate::run::artifact_path(
16648 &runner.state,
16649 &crate::github_text::artifact_name(&record.fingerprint),
16650 ))
16651 .unwrap();
16652 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
16653 runner.state =
16655 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
16656 store
16657 .update(&q.id, |q| {
16658 q.say("まだ日本語")?;
16659 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
16660 })
16661 .unwrap();
16662 assert!(
16663 runner
16664 .guarded_pr_message(&winner, None, true)
16665 .await
16666 .unwrap()
16667 .is_none()
16668 );
16669 let retry = runner.state.github_text.as_ref().unwrap();
16670 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
16671 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
16672 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
16673 assert!(
16674 q.posting_comparison
16675 .as_ref()
16676 .unwrap()
16677 .matches_detail(&q.detail)
16678 );
16679 store
16680 .update(&q.id, |q| {
16681 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
16682 })
16683 .unwrap();
16684 let message = runner
16685 .guarded_pr_message(&winner, None, true)
16686 .await
16687 .unwrap()
16688 .unwrap();
16689 assert_eq!(message.title, rewrite.title);
16690 assert_eq!(message.body, rewrite.body);
16691 assert_eq!(
16692 runner.state.github_text.as_ref().unwrap().choice,
16693 Some(GithubTextChoice::Rewrite)
16694 );
16695 let again = runner
16696 .guarded_pr_message(&winner, None, true)
16697 .await
16698 .unwrap()
16699 .unwrap();
16700 assert_eq!(again.title, message.title);
16701 assert_eq!(again.body, message.body);
16702 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
16703 assert!(
16704 !runner
16705 .state
16706 .events
16707 .iter()
16708 .any(|e| e.message.contains(&rewrite.title)
16709 || e.message.contains("English sentence."))
16710 );
16711 }
16712
16713 #[tokio::test]
16714 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
16715 crate::run::pin_test_home();
16716 let dir = tempfile::tempdir().unwrap();
16717 let mut runner = runner_at(RunStatus::Gating);
16718 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
16719 runner.state.repo = dir.path().to_owned();
16720 runner.state.candidates[0].worktree = dir.path().to_owned();
16721 seed_public_summary(&mut runner);
16722 let winner = runner.state.candidates[0].clone();
16723 assert!(
16724 runner
16725 .guarded_pr_message(&winner, None, true)
16726 .await
16727 .unwrap()
16728 .is_none()
16729 );
16730 let record = runner.state.github_text.as_ref().unwrap();
16731 assert!(record.rewrite.is_none());
16732 let mut old = serde_json::to_value(record).unwrap();
16734 old.as_object_mut().unwrap().remove("rewrite");
16735 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
16736 assert!(old.rewrite.is_none());
16737 let id = record.question.clone().unwrap();
16738 ask::Questions::open()
16739 .update(&id, |q| {
16740 q.choices.push(crate::github_text::USE_REWRITE.into());
16741 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
16742 })
16743 .unwrap();
16744 let message = runner
16745 .guarded_pr_message(&winner, None, true)
16746 .await
16747 .unwrap()
16748 .unwrap();
16749 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
16750 assert_eq!(
16751 runner.state.github_text.as_ref().unwrap().choice,
16752 Some(GithubTextChoice::Fallback)
16753 );
16754 }
16755
16756 #[tokio::test]
16757 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
16758 crate::run::pin_test_home();
16759 let dir = tempfile::tempdir().unwrap();
16760 let mut runner = runner_at(RunStatus::Gating);
16761 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
16762 runner.state.repo = dir.path().to_owned();
16763 runner.state.candidates[0].worktree = dir.path().to_owned();
16764 seed_public_summary(&mut runner);
16765 let winner = runner.state.candidates[0].clone();
16766 assert!(
16767 runner
16768 .guarded_pr_message(&winner, None, true)
16769 .await
16770 .unwrap()
16771 .is_none()
16772 );
16773 let store = ask::Questions::open();
16774 let find = |run: &str| {
16775 store
16776 .list()
16777 .into_iter()
16778 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
16779 .unwrap()
16780 };
16781 let run = runner.state.id.clone();
16782 let q = find(&run);
16783 let fp = runner
16784 .state
16785 .github_text
16786 .as_ref()
16787 .unwrap()
16788 .fingerprint
16789 .clone();
16790 let name = crate::github_text::artifact_name(&fp);
16791 let path = crate::run::artifact_path(&runner.state, &name);
16792 assert!(q.detail.contains(&path.display().to_string()));
16793 assert!(q.detail.contains("title-language: title"));
16794 let saved = std::fs::read_to_string(&path).expect("artifact written");
16795 assert!(saved.contains("日本語のタイトル"));
16796 assert!(q.cwd.is_none());
16797 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
16798 let say = |id: &str, text: &str, choice: &str| {
16800 store
16801 .update(id, |q| {
16802 q.thread.push(ask::Turn {
16803 who: ask::Who::Operator,
16804 body: text.to_owned(),
16805 at: jiff::Timestamp::now(),
16806 note: None,
16807 });
16808 q.answer(ask::Answer::Choice(choice.to_owned()))
16809 })
16810 .unwrap();
16811 };
16812 say(&q.id, "まだ日本語", "use my text");
16813 assert!(
16814 runner
16815 .guarded_pr_message(&winner, None, true)
16816 .await
16817 .unwrap()
16818 .is_none()
16819 );
16820 let second = find(&run);
16821 assert_ne!(second.id, q.id);
16822 assert!(second.detail.contains("did not pass"));
16823 assert!(second.detail.contains("日本語のタイトル"));
16826 assert!(!second.detail.contains("まだ日本語"));
16827 assert!(second.detail.contains(&path.display().to_string()));
16828 say(&second.id, "fix: retry failed requests", "use my text");
16829 let message = runner
16830 .guarded_pr_message(&winner, None, true)
16831 .await
16832 .unwrap()
16833 .expect("a vetted replacement is posted");
16834 assert_eq!(message.title, "fix: retry failed requests");
16835 assert_eq!(
16836 runner
16837 .state
16838 .github_text
16839 .as_ref()
16840 .unwrap()
16841 .chosen_title
16842 .as_deref(),
16843 Some("fix: retry failed requests")
16844 );
16845 assert!(runner.state.github_text.as_ref().unwrap().resolved);
16846 let again = runner
16848 .guarded_pr_message(&winner, None, true)
16849 .await
16850 .unwrap()
16851 .unwrap();
16852 assert_eq!(again.title, message.title);
16853 assert_eq!(
16854 store
16855 .list()
16856 .into_iter()
16857 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
16858 .count(),
16859 2
16860 );
16861 }
16862
16863 #[test]
16864 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
16865 let mut state = state_with_summary(
16869 "add retries",
16870 "TITLE: fix(web): batch reads\n- reads run.json once",
16871 );
16872 state.config.graph.language = "ja".to_owned();
16873 let m = pr_message(&state, 'A');
16874 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
16875 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
16876
16877 let task = "今回やってほしいこと: results projector を直す";
16880 let mut state = state_with_summary(task, "- no title line");
16881 state.config.graph.language = "ja".to_owned();
16882 let m = pr_message(&state, 'A');
16883 assert_eq!(
16884 m.title,
16885 format!("chore: land candidate A of run {}", state.id)
16886 );
16887 assert!(
16888 m.body.contains(&format!(
16889 "<summary>Original task</summary>\n\n{task}\n\n</details>"
16890 )),
16891 "{}",
16892 m.body
16893 );
16894 }
16895
16896 #[test]
16897 fn pr_message_scrubs_home_paths_and_addresses() {
16898 let state = state_with_summary(
16899 "fix it in /Users/someone/src/x",
16900 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
16901 );
16902 let m = pr_message(&state, 'A');
16903 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
16904 assert!(!m.body.contains(leak), "{}", m.body);
16905 }
16906 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
16907 }
16908
16909 #[test]
16910 fn pr_message_survives_a_task_that_closes_details() {
16911 let state = state_with_summary("a </details> b", "TITLE: fix: x");
16912 let m = pr_message(&state, 'A');
16913 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
16914 }
16915
16916 #[test]
16917 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
16918 let cmd = manual_merge_command(
16919 MergeStyle::Squash,
16920 Path::new("/repo"),
16921 "b",
16922 "fix: \"quoted\" $(x) `y`\n\nbody",
16923 );
16924 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
16925 }
16926
16927 #[test]
16928 fn manual_merge_command_matches_the_configured_style() {
16929 let repo = Path::new("/repo");
16930 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
16931
16932 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
16933 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
16934
16935 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
16936 assert_eq!(
16937 squash,
16938 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
16939 \"Merge magi run 0832 (candidate A)\""
16940 );
16941
16942 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
16943 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
16944 }
16945
16946 #[test]
16947 fn a_nudge_gets_a_quarter_of_the_budget() {
16948 assert_eq!(retry_budget(secs(1200), true), secs(300));
16950 assert_eq!(retry_budget(secs(3600), true), secs(900));
16951 }
16952
16953 #[test]
16954 fn a_resent_prompt_keeps_the_whole_budget() {
16955 assert_eq!(retry_budget(secs(1200), false), secs(1200));
16958 assert_eq!(retry_budget(secs(60), false), secs(60));
16959 }
16960
16961 #[test]
16962 fn the_floor_never_exceeds_the_original_budget() {
16963 assert_eq!(retry_budget(secs(60), true), secs(60));
16967 assert_eq!(retry_budget(secs(480), true), secs(120));
16968 assert_eq!(retry_budget(secs(0), true), secs(0));
16969 }
16970
16971 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
16972 agent::CommandEvidence {
16973 id: "item1".to_owned(),
16974 description: "cargo test".to_owned(),
16975 exit_code,
16976 result_summary: String::new(),
16977 source: "codex".to_owned(),
16978 }
16979 }
16980
16981 #[test]
16982 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
16983 assert!(!has_unconfirmed_command(&[]));
16987 }
16988
16989 #[test]
16990 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
16991 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
16995 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
16996 assert!(!has_unconfirmed_command(&[
16997 evidence(Some(0)),
16998 evidence(Some(101))
16999 ]));
17000 }
17001
17002 #[test]
17003 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17004 assert!(has_unconfirmed_command(&[
17005 evidence(Some(0)),
17006 evidence(None)
17007 ]));
17008 }
17009
17010 #[test]
17011 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17012 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17013 assert_eq!(
17014 verified_noop_claim(true, &[], text).as_deref(),
17015 Some("already fixed by b32cfc4, on main.")
17016 );
17017 }
17018
17019 #[test]
17020 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17021 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17024 assert!(verified_noop_claim(false, &[], text).is_none());
17025 }
17026
17027 #[test]
17028 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17029 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17030 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17031 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17033 }
17034
17035 #[test]
17036 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17037 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17038 }
17039
17040 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17043 runner.state.candidates = shape
17044 .iter()
17045 .enumerate()
17046 .map(|(i, &(empty, verified))| Candidate {
17047 index: i,
17048 label: (b'A' + i as u8) as char,
17049 agent: "sonnet".to_owned(),
17050 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17051 worktree: PathBuf::from(format!("/wt/{i}")),
17052 summary: String::new(),
17053 stat: String::new(),
17054 files: 0,
17055 commits: 0,
17056 empty,
17057 failed: None,
17058 verified_noop: verified.map(str::to_owned),
17059 duration_ms: 0,
17060 folded: false,
17061 })
17062 .collect();
17063 }
17064
17065 #[test]
17066 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17067 ask_test_home();
17068 let mut runner = runner_at(RunStatus::Implementing);
17069 set_candidates(
17070 &mut runner,
17071 &[
17072 (true, Some("already on main at b32cfc4")),
17073 (true, Some("same fix, see the existing test")),
17074 ],
17075 );
17076
17077 runner
17078 .after_implement()
17079 .expect("a verified no-op is not an error");
17080
17081 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17082 }
17083
17084 #[test]
17085 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17086 ask_test_home();
17087 let mut runner = runner_at(RunStatus::Implementing);
17088 set_candidates(
17092 &mut runner,
17093 &[(true, Some("already on main at b32cfc4")), (true, None)],
17094 );
17095
17096 let err = runner
17097 .after_implement()
17098 .expect_err("an unverified empty candidate must still fail the run");
17099
17100 assert!(
17101 err.to_string().contains("no candidate produced a change"),
17102 "{err}"
17103 );
17104 assert_eq!(runner.state.status, RunStatus::Failed);
17105 }
17106
17107 #[test]
17108 fn after_implement_still_fails_an_ordinary_all_empty_run() {
17109 ask_test_home();
17110 let mut runner = runner_at(RunStatus::Implementing);
17111 set_candidates(&mut runner, &[(true, None), (true, None)]);
17112
17113 let err = runner
17114 .after_implement()
17115 .expect_err("no candidate declared anything; this is an ordinary failure");
17116
17117 assert!(
17118 err.to_string().contains("no candidate produced a change"),
17119 "{err}"
17120 );
17121 assert_eq!(runner.state.status, RunStatus::Failed);
17122 }
17123}
17124
17125#[derive(Debug)]
17128struct GraftRestoreFailed(String);
17129
17130impl std::fmt::Display for GraftRestoreFailed {
17131 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
17132 write!(
17133 f,
17134 "could not restore the winner to its pre-graft commit: {}",
17135 self.0
17136 )
17137 }
17138}
17139
17140impl std::error::Error for GraftRestoreFailed {}
17141
17142#[cfg(test)]
17146mod resource_wait_tests {
17147 use super::*;
17148 use std::collections::BTreeMap;
17149
17150 fn sh(id: &str, script: &str) -> AgentSpec {
17151 AgentSpec {
17152 id: id.to_owned(),
17153 kind: crate::config::AgentKind::Command,
17154 model: None,
17155 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
17156 extra_args: Vec::new(),
17157 env: BTreeMap::new(),
17158 prompt_delivery: None,
17159 }
17160 }
17161
17162 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
17163 SeatJob {
17164 seat: SeatState::new("impl-A", &spec.id, 1),
17165 spec,
17166 cwd: dir.to_path_buf(),
17167 prompt: "do it".to_owned(),
17168 timeout,
17169 allow_write: true,
17170 sessions: false,
17171 artifacts: dir.join("art"),
17172 stem: "impl-A".to_owned(),
17173 handover: None,
17174 }
17175 }
17176
17177 fn state(resource_wait_secs: u64) -> RunState {
17178 let mut config = Config::default();
17179 config.graph.timeout_resource_wait = resource_wait_secs;
17180 RunState::new(
17181 PathBuf::from("/repo"),
17182 "main".to_owned(),
17183 "abc1234".to_owned(),
17184 "task".to_owned(),
17185 config,
17186 )
17187 }
17188
17189 fn hold(cache: &Path) -> crate::cache::Guard {
17190 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
17191 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
17192 crate::cache::AcquireOutcome::Acquired(g) => g,
17193 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
17194 }
17195 }
17196
17197 async fn run(
17198 jobs: Vec<SeatJob>,
17199 sem: Arc<Semaphore>,
17200 cache: Option<&Path>,
17201 st: &mut RunState,
17202 ) -> Vec<(usize, SeatState, AgentOutcome)> {
17203 let prompts = Prompts::default();
17204 let ctx = WaveCtx {
17205 run: "r",
17206 node: "implement",
17207 prompts: &prompts,
17208 cache,
17209 round: None,
17210 carry_seats: false,
17211 pauses: None,
17212 };
17213 wave(jobs, sem, &ctx, st, 0).await
17214 }
17215
17216 #[tokio::test]
17217 async fn verification_cache_wait_preserves_the_command_budget() {
17218 crate::run::pin_test_home();
17219 let tmp = tempfile::tempdir().unwrap();
17220 let cache = tmp.path().join("target");
17221 let guard = hold(&cache);
17222 let release = tokio::spawn(async move {
17223 tokio::time::sleep(Duration::from_millis(100)).await;
17224 guard.release();
17225 });
17226 let mut st = state(10);
17227 let budget = Duration::from_millis(50);
17228 let wait = repository_wait(&st, None);
17229 let (outcomes, _) = with_cache_lease(
17230 &mut st,
17231 Some(&cache),
17232 "gate",
17233 "gate",
17234 tmp.path(),
17235 "abc1234",
17236 budget,
17237 "test gate",
17238 &wait,
17239 |_, available| async move {
17240 assert_eq!(
17241 available, budget,
17242 "contention must not shorten verification"
17243 );
17244 (Vec::new(), false, Vec::new())
17245 },
17246 )
17247 .await
17248 .unwrap();
17249 release.await.unwrap();
17250 assert!(outcomes.is_empty(), "{outcomes:?}");
17251 assert!(
17252 st.events
17253 .iter()
17254 .any(|event| event.message.contains("waiting for"))
17255 );
17256 assert!(st.resource_wait.is_none());
17257 hold(&cache).release();
17259 }
17260
17261 #[tokio::test]
17262 async fn verification_cache_wait_expires_without_starting_commands() {
17263 crate::run::pin_test_home();
17264 let tmp = tempfile::tempdir().unwrap();
17265 let cache = tmp.path().join("target");
17266 let _held = hold(&cache);
17267 let mut st = state(0);
17268 let wait = repository_wait(&st, None);
17269 let (outcomes, _) = with_cache_lease(
17270 &mut st,
17271 Some(&cache),
17272 "gate",
17273 "gate",
17274 tmp.path(),
17275 "abc1234",
17276 Duration::from_secs(60),
17277 "test gate",
17278 &wait,
17279 |_, _| async { panic!("commands must not start without the cache lease") },
17280 )
17281 .await
17282 .unwrap();
17283 assert_eq!(outcomes.len(), 1);
17284 assert!(outcomes[0].resource_blocked);
17285 assert!(st.resource_wait.is_none());
17286 }
17287
17288 #[tokio::test]
17289 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
17290 crate::run::pin_test_home();
17291 let tmp = tempfile::tempdir().unwrap();
17292 let cache = tmp.path().join("target-a");
17293 let guard = hold(&cache);
17294 tokio::spawn(async move {
17296 tokio::time::sleep(Duration::from_millis(1500)).await;
17297 guard.release();
17298 });
17299 let mut st = state(60);
17300 let results = run(
17301 vec![job(
17302 sh("alpha", "echo ok"),
17303 tmp.path(),
17304 Duration::from_secs(1),
17305 )],
17306 Arc::new(Semaphore::new(1)),
17307 Some(&cache),
17308 &mut st,
17309 )
17310 .await;
17311 assert!(
17312 matches!(results[0].2, AgentOutcome::Ok(_)),
17313 "the wait must not be charged to the agent"
17314 );
17315 assert!(st.resource_block.is_none());
17316 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
17317 assert!(st.resource_wait.is_none() && st.active.is_empty());
17318 }
17319
17320 #[tokio::test]
17321 #[cfg(unix)]
17322 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
17323 use std::os::fd::AsRawFd as _;
17324 crate::run::pin_test_home();
17325 let tmp = tempfile::tempdir().unwrap();
17326 crate::git::git(tmp.path(), &["init"]).await.unwrap();
17327 let common = tmp.path().join(".git");
17328 std::fs::write(common.join("kanade-sync.contract"), "shared-lock-v1\n").unwrap();
17329 let external = std::fs::File::create(common.join("kanade-sync.lock")).unwrap();
17330 assert_eq!(
17331 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17332 0
17333 );
17334 let mut st = state(1);
17335 let mut reader = job(
17336 sh("reader", "echo must-not-run"),
17337 tmp.path(),
17338 Duration::from_secs(30),
17339 );
17340 reader.allow_write = false;
17341 let results = run(
17342 vec![reader.clone()],
17343 Arc::new(Semaphore::new(1)),
17344 None,
17345 &mut st,
17346 )
17347 .await;
17348 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
17349 assert_eq!(results[0].1.turns, 0);
17350 assert!(FailClass::of(&results[0].2).is_none());
17351 assert_eq!(
17352 st.resource_block.as_ref().unwrap().kind,
17353 "repository coordination"
17354 );
17355 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
17356 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
17357 drop(external);
17358 let editor = crate::repo_lock::SeatLease::for_checkout(tmp.path(), true)
17359 .await
17360 .unwrap();
17361 let pause = Pause::new();
17362 let interrupt = Pause::new();
17363 let prompts = Prompts::default();
17364 let ctx = WaveCtx {
17365 run: "r",
17366 node: "review",
17367 prompts: &prompts,
17368 cache: None,
17369 round: None,
17370 carry_seats: false,
17371 pauses: Some((&pause, &interrupt)),
17372 };
17373 let (results, ()) = tokio::join!(
17374 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
17375 async {
17376 tokio::time::sleep(Duration::from_millis(50)).await;
17377 pause.park();
17378 }
17379 );
17380 assert!(
17381 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
17382 );
17383 assert_eq!(results[0].1.turns, 0);
17384 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
17385 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
17386 drop(editor);
17387 }
17388
17389 #[tokio::test]
17390 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
17391 crate::run::pin_test_home();
17392 let tmp = tempfile::tempdir().unwrap();
17393 let cache = tmp.path().join("target-b");
17394 let _held = hold(&cache);
17395 let mut st = state(1);
17396 let results = run(
17397 vec![job(
17398 sh("alpha", "echo ok"),
17399 tmp.path(),
17400 Duration::from_secs(1),
17401 )],
17402 Arc::new(Semaphore::new(1)),
17403 Some(&cache),
17404 &mut st,
17405 )
17406 .await;
17407 let (_, seat, out) = &results[0];
17408 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
17409 assert!(wave_blocked(&results));
17410 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
17411 assert_eq!(seat.turns, 0, "nothing was asked");
17412 assert!(
17413 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
17414 "no agent was spawned"
17415 );
17416 assert!(FailClass::of(out).is_none(), "not a failure class");
17417 let block = st.resource_block.as_ref().expect("recorded for the resume");
17418 assert_eq!(block.kind, "shared build cache");
17419 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
17420 assert!(st.active.is_empty());
17421 }
17422
17423 #[tokio::test]
17424 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
17425 crate::run::pin_test_home();
17426 let tmp = tempfile::tempdir().unwrap();
17427 let mut st = state(1);
17428 let results = run(
17429 vec![job(
17430 sh("alpha", "echo ok"),
17431 tmp.path(),
17432 Duration::from_secs(1),
17433 )],
17434 Arc::new(Semaphore::new(0)),
17435 None,
17436 &mut st,
17437 )
17438 .await;
17439 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
17440 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
17441 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
17442 }
17443
17444 #[tokio::test]
17445 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
17446 crate::run::pin_test_home();
17447 let tmp = tempfile::tempdir().unwrap();
17448 let mut st = state(60);
17449 let results = run(
17450 vec![job(
17451 sh("alpha", "sleep 30"),
17452 tmp.path(),
17453 Duration::from_secs(1),
17454 )],
17455 Arc::new(Semaphore::new(1)),
17456 None,
17457 &mut st,
17458 )
17459 .await;
17460 let out = &results[0].2;
17461 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
17462 let class = FailClass::of(out).expect("a failure");
17463 assert_eq!(class, FailClass::Timeout);
17464 assert!(should_hand_over(None, &class), "existing handover policy");
17465 assert!(st.resource_block.is_none());
17466 }
17467}