1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::Deserialize;
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Default, Deserialize)]
102pub struct Decision {
103 pub id: String,
105 #[serde(default)]
108 pub blocked_by: Vec<String>,
109 #[serde(default)]
111 pub reason: Option<String>,
112 #[serde(default)]
120 pub recovery: Option<Recovery>,
121 #[serde(default)]
125 pub question: Option<String>,
126 #[serde(default)]
128 pub choices: Vec<String>,
129 #[serde(default)]
131 pub evidence: Vec<String>,
132}
133
134#[derive(Debug, Clone, Default, Deserialize)]
147pub struct Verdict {
148 pub decisions: Vec<Decision>,
151}
152
153fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
156 prompt::ConductTask {
157 id: t.id.clone(),
158 title: t.title.clone(),
159 instruction: t.instruction.clone(),
160 repo: t.repo.display().to_string(),
161 priority: t.priority,
162 status: t.status.as_str().to_owned(),
163 attempts: t.attempts,
164 max_attempts: t.effective_attempt_limit(max_attempts),
165 last_error: t.last_error.clone(),
166 hold_reason: t.hold_reason.clone(),
167 hold_source: t.hold_source.map(|source| source.label().to_owned()),
168 blocked_by: t.blocked_by.clone(),
169 answers: t
170 .answers
171 .iter()
172 .map(|a| prompt::ConductAnswer {
173 question: a.question.clone(),
174 answer: a.answer.clone(),
175 })
176 .collect(),
177 operator_resume: t.resumption.as_ref().map(|r| format!(
178 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
179 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
180 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
181 wording alone is insufficient.",
182 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
183 )).or_else(|| t.resume_override.as_ref().map(|o| {
184 format!(
185 "the operator explicitly answered \"resume\" at {}; do not hold this \
186 task again for the same reason unless there is new information",
187 o.at
188 )
189 })),
190 }
191}
192
193fn pinned_resume(task: &Task) -> bool {
197 task.resume_override
198 .as_ref()
199 .is_some_and(|o| o.pinned_run.is_some())
200}
201
202fn new_blocking_evidence(
205 queue: &Queue,
206 questions: &Questions,
207 task: &Task,
208 d: &Decision,
209) -> Vec<String> {
210 let Some(r) = &task.resumption else {
211 return Vec::new();
212 };
213 d.evidence
214 .iter()
215 .filter_map(|id| {
216 let version = if let Ok(q) = questions.get(id) {
217 (q.asked_at > r.at
218 && (q.run == task.id || task.runs.contains(&q.run))
219 && q.node != NODE
220 && q.status != crate::ask::QuestionStatus::Abandoned)
221 .then(|| {
222 format!(
223 "question:{id}:{}:{:?}:{}",
224 q.asked_at,
225 q.status,
226 q.resolution().unwrap_or_default()
227 )
228 })
229 } else if d.blocked_by.contains(id) {
230 queue
231 .get(id)
232 .ok()
233 .filter(|dep| {
234 dep.id != task.id && dep.updated_at > r.at && dep.status != TaskStatus::Done
235 })
236 .map(|dep| {
237 format!(
238 "dependency:{id}:{:?}:{:?}:{:?}",
239 dep.status, dep.hold_reason, dep.last_error
240 )
241 })
242 } else if task.runs.iter().any(|run| run == id) {
243 RunState::load(id)
244 .ok()
245 .filter(|run| {
246 run.updated_at > r.at
249 && run
250 .passes
251 .last()
252 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
253 && matches!(
254 run.status,
255 crate::run::RunStatus::Failed
256 | crate::run::RunStatus::Blocked
257 | crate::run::RunStatus::Stalled
258 )
259 })
260 .map(|run| {
261 format!(
262 "run:{id}:{:?}:{}:{}:{:?}",
263 run.status,
264 run.reviews.len(),
265 task.attempts,
266 run.passes.last().map(|p| (p.index, p.started_at))
267 )
268 })
269 } else {
270 None
271 };
272 version.filter(|v| !r.blocking_evidence.contains(v))
273 })
274 .collect()
275}
276
277fn may_hold(task: &mut Task, reason: &str) -> bool {
281 if pinned_resume(task) {
282 return false;
283 }
284 if task.resumption.is_some() {
285 return true; }
287 let Some(o) = task.resume_override.as_mut() else {
288 return true;
289 };
290 if o.forced || o.pinned_run.is_some() {
291 tracing::warn!(
292 "conductor tried to hold task {} after the operator forced a resume: {reason}",
293 task.id
294 );
295 return false;
296 }
297 if o.conductor_rehold.is_some() {
298 return false;
299 }
300 o.conductor_rehold = Some(reason.to_owned());
301 true
302}
303
304fn severity_str(s: crate::verdict::Severity) -> &'static str {
307 match s {
308 crate::verdict::Severity::Nit => "nit",
309 crate::verdict::Severity::Minor => "minor",
310 crate::verdict::Severity::Major => "major",
311 crate::verdict::Severity::Blocker => "blocker",
312 }
313}
314
315fn surviving_branch(task: &Task) -> Option<String> {
320 let last = task.runs.last()?;
321 let state = RunState::load(last).ok()?;
322 if !crate::daemon::compatible_run_instruction(&state, task) {
323 return None;
324 }
325 state.winner().map(|c| c.branch.clone())
326}
327
328fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
355 let note = match &d.reason {
356 Some(reason) => reason.clone(),
357 None => match task.answers.last() {
358 Some(a) => format!(
359 "conduct held this again with no new reason given; last operator \
360 answer on record: {}",
361 a.answer
362 ),
363 None => "conduct held this again with no reason given".to_owned(),
364 },
365 };
366 match task.hold_reason.as_deref() {
367 Some(prior) if !prior.is_empty() => {
368 let outer = outermost_hold_note(prior);
369 if note.trim() == prior.trim() || note.trim() == outer {
370 None
371 } else {
372 Some(format!("{note}\n\n(previously: {outer})"))
373 }
374 }
375 _ => Some(note),
376 }
377}
378
379const PREVIOUSLY: &str = "\n\n(previously: ";
382
383fn outermost_hold_note(reason: &str) -> &str {
386 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
387}
388
389fn hold_note(d: &Decision) -> String {
391 d.reason
392 .clone()
393 .unwrap_or_else(|| "(no reason given)".to_owned())
394}
395
396async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
398 let Some(run_id) = task.runs.last().cloned() else {
399 return prompt::ConductOutcome {
400 run_id: "(none)".to_owned(),
401 unreadable: Some("this task has not produced a run yet".to_owned()),
402 run_status: None,
403 open_findings: Vec::new(),
404 rounds_used: 0,
405 rounds_max: 0,
406 rounds: Vec::new(),
407 branch: None,
408 branch_head: None,
409 references: None,
410 empty_candidate: false,
411 };
412 };
413 let state = match RunState::load(&run_id) {
414 Ok(s) => s,
415 Err(e) => {
416 tracing::warn!(
421 "conductor: could not read run {run_id} for task {}: {e:#}",
422 task.short()
423 );
424 return prompt::ConductOutcome {
425 run_id,
426 unreadable: Some(format!("{e:#}")),
427 run_status: None,
428 open_findings: Vec::new(),
429 rounds_used: 0,
430 rounds_max: 0,
431 rounds: Vec::new(),
432 branch: None,
433 branch_head: None,
434 references: None,
435 empty_candidate: false,
436 };
437 }
438 };
439
440 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
441 id: f.id.clone(),
442 title: f.title.clone(),
443 severity: severity_str(f.severity).to_owned(),
444 };
445 let open_findings = state
446 .open_findings()
447 .into_iter()
448 .map(finding_view)
449 .collect();
450 let rounds = state
451 .reviews
452 .iter()
453 .map(|r| prompt::ConductRound {
454 round: r.round,
455 findings: r
456 .reviews
457 .iter()
458 .flat_map(|rec| rec.findings.iter())
459 .map(finding_view)
460 .collect(),
461 addressed: r
462 .fix
463 .as_ref()
464 .map(|fx| fx.addressed.clone())
465 .unwrap_or_default(),
466 rejected: r
467 .fix
468 .as_ref()
469 .map(|fx| {
470 fx.rejected
471 .iter()
472 .map(|rej| prompt::ConductRejection {
473 id: rej.id.clone(),
474 why: rej.why.clone(),
475 })
476 .collect()
477 })
478 .unwrap_or_default(),
479 })
480 .collect();
481 let branch = state.winner().map(|c| c.branch.clone());
482 let branch_head = match &branch {
483 Some(b) => crate::git::rev_parse(repo, b)
484 .await
485 .ok()
486 .map(|h| h.chars().take(8).collect()),
487 None => None,
488 };
489
490 prompt::ConductOutcome {
491 run_id,
492 unreadable: None,
493 run_status: Some(state.status.as_str().to_owned()),
494 open_findings,
495 rounds_used: state.reviews.len(),
496 rounds_max: state.config.graph.review_rounds,
497 rounds,
498 branch,
499 branch_head,
500 references: crate::refs::describe(&state.seeds),
501 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
502 }
503}
504
505async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
509 let id = task.runs.last()?;
510 let state = match RunState::load(id) {
511 Ok(s) => s,
512 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
513 };
514 let branch = state.winner()?.branch.clone();
515 let base = state.base_branch.clone();
516 match crate::land::find_open_pr(repo, &branch, &base).await {
517 Ok(crate::land::OpenPr::None) => None,
518 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
519 "pull request {url} is already open for {branch} into {base}"
520 )),
521 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
522 "several pull requests are already open for {branch} into {base}: {}",
523 urls.join(" ")
524 )),
525 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
526 }
527}
528
529async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
536 for d in verdict
537 .decisions
538 .iter_mut()
539 .filter(|d| d.question.is_some())
540 {
541 let Ok(task) = queue.get(&d.id) else {
542 continue;
543 };
544 let text = format!("{}\n{}", task.title, task.instruction);
545 let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
546 if crate::refs::scan(&text).is_empty() {
547 if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
548 q.push_str(&format!(
549 "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
550 ));
551 }
552 continue;
553 }
554 let repo = repo_for(&task, repo);
555 let remote = &cfg.merge.remote;
556 let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
557 .await
558 .ok();
559 let facts = match base {
560 Some(base) => {
561 let base_name = base.clone();
562 let tracking = format!("{remote}/{base}");
563 let refreshed = crate::git::fetch(&repo, remote, &base)
566 .await
567 .is_ok_and(|o| o.ok());
568 let against = tracking;
571 match crate::git::rev_parse(&repo, &against).await {
572 Ok(tip) => crate::refs::describe(
573 &crate::refs::resolve(&repo, &tip, remote, &text).await,
574 )
575 .map(|facts| {
576 if refreshed {
577 facts
578 } else {
579 format!(
580 "{facts}\n(could not fetch {remote}/{base_name}: this is \
581 against the last fetched `{against}`, which may be behind \
582 the remote)"
583 )
584 }
585 }),
586 Err(_) => Some(format!(
587 "could NOT be checked against the repository: {against} is not \
588 available here, so whether the work is in the base is unknown"
589 )),
590 }
591 }
592 None => Some("could not check the repository: no base branch known".to_owned()),
593 };
594 let facts = match (facts, pr_fact) {
595 (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
596 (f, p) => f.or(p),
597 };
598 if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
599 q.push_str(&format!(
600 "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
601 ));
602 }
603 }
604}
605
606async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
608 prompt::ConductFinished {
609 task: view(t, max_attempts),
610 outcome: outcome_for(t, &repo_for(t, repo)).await,
611 }
612}
613
614fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
617 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
618 fallback.to_path_buf()
619 } else {
620 task.repo.clone()
621 }
622}
623
624fn apply_one(
641 queue: &Queue,
642 questions: &Questions,
643 d: &Decision,
644 snapshot: Option<&Task>,
645) -> Result<()> {
646 let _claim = queue
647 .claim(&d.id)
648 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
649 let mut task = queue.get(&d.id).context("no such task")?;
650
651 if snapshot
654 .is_some_and(|old| serde_json::to_value(old).ok() != serde_json::to_value(&task).ok())
655 {
656 return Ok(());
657 }
658 let blocking =
659 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
660 let evidence = new_blocking_evidence(queue, questions, &task, d);
661 if blocking && task.resumption.is_some() && evidence.is_empty() {
662 crate::notices::raise(
663 crate::notices::Notice::warn(
664 &format!("conductor-resumption:{}", task.id),
665 format!(
666 "Conductor concern after explicit resumption (no verified new evidence): {}; question: {:?}; dependencies: {:?}; evidence: {:?}",
667 hold_note(d), d.question, d.blocked_by, d.evidence
668 ),
669 )
670 .about([task.id.clone()]),
671 );
672 return Ok(());
673 }
674
675 if blocking {
676 if let Some(r) = task.resumption.as_mut() {
677 r.blocking_evidence.extend(evidence);
678 }
679 }
680
681 if task.operator_held() {
685 return Ok(());
686 }
687
688 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
692 return Ok(());
693 }
694
695 let pinned = pinned_resume(&task);
698 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
699 return Ok(());
700 }
701
702 if let Some(text) = &d.question {
703 if task.status == TaskStatus::Done {
704 return Ok(());
705 }
706 let question_id = match questions
710 .list()
711 .into_iter()
712 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
713 {
714 Some(existing) => existing.id,
715 None => {
723 let mut q = Question::new(
724 task.id.clone(),
725 NODE.to_owned(),
726 SEAT.to_owned(),
727 text.clone(),
728 d.reason.clone().unwrap_or_default(),
729 d.choices.clone(),
730 );
731 q.deputy = Some(Deputy::new(crate::deputy::brief(
736 &task.id,
737 d.reason.as_deref().unwrap_or_default(),
738 &q.choices,
739 &q.actions,
740 )));
741 if task.repo.is_dir() && task.repo != Path::new(".") {
742 q.cwd = Some(task.repo.to_string_lossy().into_owned());
743 }
744 questions.put(&mut q)?;
745 q.id
746 }
747 };
748 task.block(vec![question_id], d.reason.clone());
749 return queue.put(&mut task);
750 }
751
752 match task.status {
753 TaskStatus::Queued if !d.blocked_by.is_empty() => {
754 task.block(d.blocked_by.clone(), d.reason.clone());
755 queue.put(&mut task)?;
756 }
757 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
761 if may_hold(&mut task, &hold_note(d)) {
762 task.hold_machine(d.reason.clone());
763 queue.put(&mut task)?;
764 }
765 }
766 TaskStatus::Running => match d.recovery {
767 Some(Recovery::Requeue) if pinned_resume(&task) => {}
768 Some(Recovery::Requeue) => {
769 task.requeue();
770 queue.put(&mut task)?;
771 }
772 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
773 task.hold_machine(d.reason.clone());
774 queue.put(&mut task)?;
775 }
776 _ => {}
780 },
781 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
782 Some(Recovery::Requeue) if pinned_resume(&task) => {}
785 Some(Recovery::Requeue) => {
786 task.requeue();
787 queue.put(&mut task)?;
788 }
789 Some(Recovery::Hold) => {
790 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
794 if may_hold(&mut task, &hold_note(d)) {
795 task.hold_machine(Some(reason));
796 queue.put(&mut task)?;
797 }
798 }
799 }
800 Some(Recovery::Review) => {
801 if let Some(branch) = surviving_branch(&task) {
802 task.request_review(branch);
803 queue.put(&mut task)?;
804 }
805 }
811 Some(Recovery::Done) => {
812 task.succeed();
813 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
817 queue.put(&mut task)?;
818 }
819 None => {}
820 },
821 _ => {}
824 }
825 Ok(())
826}
827
828pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
832 for d in &verdict.decisions {
833 if let Err(e) = apply_one(queue, questions, d, None) {
834 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
835 }
836 }
837 Ok(())
838}
839
840pub fn seat_path(home: &Path) -> PathBuf {
846 home.join("conduct").join("seat.json")
847}
848
849pub fn load_seat(home: &Path) -> Option<SeatState> {
851 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
852}
853
854fn busy_path(home: &Path) -> PathBuf {
855 home.join("conduct").join("busy")
856}
857
858pub fn busy(home: &Path) -> bool {
864 std::fs::metadata(busy_path(home))
865 .and_then(|m| m.modified())
866 .ok()
867 .and_then(|t| t.elapsed().ok())
868 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
869}
870
871struct Busy(PathBuf);
873
874impl Busy {
875 fn mark(home: &Path) -> Self {
876 let path = busy_path(home);
877 if let Some(dir) = path.parent() {
878 let _ = std::fs::create_dir_all(dir);
879 }
880 let _ = std::fs::write(&path, std::process::id().to_string());
881 Self(path)
882 }
883}
884
885impl Drop for Busy {
886 fn drop(&mut self) {
887 let _ = std::fs::remove_file(&self.0);
888 }
889}
890
891#[derive(Debug, Default)]
894pub struct Conductor {
895 seat: Option<SeatState>,
896 last_seen: Option<(u64, BTreeSet<String>)>,
897 considered: BTreeMap<String, String>,
901}
902
903fn input_fingerprint(task: &Task) -> String {
908 let mut v = serde_json::to_value(task).unwrap_or_default();
909 if let Some(o) = v.as_object_mut() {
910 o.remove("updated_at");
911 o.remove("hold_reason");
912 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
913 ro.remove("conductor_rehold");
914 }
915 }
916 v.to_string()
917}
918
919impl Conductor {
920 #[must_use]
922 pub fn new() -> Self {
923 Self::default()
924 }
925
926 fn settled(
931 considered: &BTreeMap<String, String>,
932 stalled: &[Task],
933 finished: &[Task],
934 ) -> BTreeSet<String> {
935 stalled
936 .iter()
937 .chain(finished)
938 .filter(|t| t.status == TaskStatus::Held)
939 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
940 .map(|t| t.id.clone())
941 .collect()
942 }
943
944 fn snapshot_with(
945 considered: &BTreeMap<String, String>,
946 queue: &Queue,
947 stalled: &[Task],
948 finished: &[Task],
949 ) -> (u64, BTreeSet<String>) {
950 let ids = stalled
951 .iter()
952 .chain(finished)
953 .map(|t| t.id.clone())
954 .collect();
955 let skip = Self::settled(considered, stalled, finished);
956 (queue.revision_excluding(&skip), ids)
957 }
958
959 fn snapshot(
960 &self,
961 queue: &Queue,
962 stalled: &[Task],
963 finished: &[Task],
964 ) -> (u64, BTreeSet<String>) {
965 Self::snapshot_with(&self.considered, queue, stalled, finished)
966 }
967
968 #[must_use]
985 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
986 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
987 }
988
989 #[allow(clippy::too_many_arguments)]
993 pub async fn maybe_run(
994 &mut self,
995 cfg: &Config,
996 repo: &Path,
997 queue: &Queue,
998 questions: &Questions,
999 home: &Path,
1000 queued: &[Task],
1001 stalled: &[Task],
1002 finished: &[Task],
1003 max_attempts: usize,
1004 ) {
1005 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1006 return;
1007 }
1008 self.considered = stalled
1011 .iter()
1012 .chain(finished)
1013 .filter(|t| t.status == TaskStatus::Held)
1014 .map(|t| (t.id.clone(), input_fingerprint(t)))
1015 .collect();
1016 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1017 if let Err(e) = self
1018 .run_once(
1019 cfg,
1020 repo,
1021 queue,
1022 questions,
1023 home,
1024 queued,
1025 stalled,
1026 finished,
1027 max_attempts,
1028 )
1029 .await
1030 {
1031 tracing::warn!("conductor: {e:#}");
1037 }
1038 }
1039
1040 #[allow(clippy::too_many_arguments)]
1041 async fn run_once(
1042 &mut self,
1043 cfg: &Config,
1044 repo: &Path,
1045 queue: &Queue,
1046 questions: &Questions,
1047 home: &Path,
1048 queued: &[Task],
1049 stalled: &[Task],
1050 finished: &[Task],
1051 max_attempts: usize,
1052 ) -> Result<()> {
1053 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1054 return Ok(());
1055 }
1056
1057 let primary = cfg
1058 .resolve_roles()
1059 .context("resolving the conductor seat")?
1060 .conductor;
1061 let chain = match cfg.roles.conductor.as_ref() {
1065 Some(c) if c.ids().len() > 1 => {
1066 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1067 }
1068 _ => vec![primary],
1069 };
1070
1071 let runnable_views: Vec<prompt::ConductTask> =
1072 queued.iter().map(|t| view(t, max_attempts)).collect();
1073 let stalled_views: Vec<prompt::ConductTask> =
1074 stalled.iter().map(|t| view(t, max_attempts)).collect();
1075 let mut finished_views = Vec::with_capacity(finished.len());
1076 for t in finished {
1077 finished_views.push(finished_view(t, repo, max_attempts).await);
1078 }
1079
1080 let body = prompt::with_overlay(
1081 prompt::conduct(
1082 &runnable_views,
1083 &stalled_views,
1084 &finished_views,
1085 &cfg.graph.language,
1086 ),
1087 cfg.prompts.overlay(NODE),
1088 );
1089
1090 let artifacts = home.join("conduct").join("artifacts");
1091 let cache_dir = cfg.cache_dir();
1094
1095 let busy = Busy::mark(home);
1097 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1098 for spec in &chain {
1099 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1102 if needs_new_seat {
1103 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1104 }
1105 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1106 let stem = if std::ptr::eq(spec, &chain[0]) {
1109 format!("turn-{}", seat.turns + 1)
1110 } else {
1111 format!("turn-{}-{}", seat.turns + 1, spec.id)
1112 };
1113 let inv = Invocation {
1114 cwd: repo,
1115 prompt: &body,
1116 timeout: TURN_TIMEOUT,
1117 allow_write: false,
1120 unsandboxed: false,
1121 sessions: cfg.graph.sessions,
1122 artifacts: &artifacts,
1123 stem: &stem,
1124 run: NODE,
1125 node: NODE,
1126 cache_dir: cache_dir.as_deref(),
1127 attachments: &[],
1128 writable: &[],
1129 };
1130 let out = agent::invoke(spec, seat, &inv).await;
1131 if let Ok(body) = serde_json::to_string(&*seat) {
1134 let path = seat_path(home);
1135 if let Some(dir) = path.parent() {
1136 let _ = std::fs::create_dir_all(dir);
1137 }
1138 let _ = std::fs::write(path, body);
1139 }
1140 let advance = agent::chain_advances(&out);
1141 result = match out {
1142 Err(e) => Err(e.context("invoking the conductor")),
1143 Ok(out) if advance => Err(anyhow::anyhow!(
1144 "no usable reply (exit {:?}, timed out {})",
1145 out.exit_code,
1146 out.timed_out
1147 )),
1148 Ok(out) => verdict::extract_json(&out.text)
1149 .context("the conductor's reply could not be parsed"),
1150 };
1151 if result.is_ok() {
1152 break;
1153 }
1154 if chain.len() > 1 {
1155 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1156 }
1157 }
1158 drop(busy);
1159 let mut verdict = result?;
1160 attach_facts(cfg, repo, queue, &mut verdict).await;
1161 for d in &verdict.decisions {
1162 let snapshot = queued
1163 .iter()
1164 .chain(stalled)
1165 .chain(finished)
1166 .find(|t| t.id == d.id);
1167 if let Some(snapshot) = snapshot {
1168 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1169 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1170 }
1171 }
1172 }
1173 Ok(())
1174 }
1175}
1176
1177#[cfg(test)]
1178mod tests {
1179 use std::collections::BTreeMap;
1180
1181 use tempfile::tempdir;
1182
1183 use super::*;
1184 use crate::ask::{Answer, QuestionStatus};
1185 use crate::config::{AgentKind, AgentSpec, Graph};
1186 use crate::queue::Source;
1187
1188 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1189 let path = dir.join("mock-conduct-agent.sh");
1190 std::fs::write(&path, script).expect("write mock");
1191 AgentSpec {
1192 id: "mock".to_owned(),
1193 kind: AgentKind::Command,
1194 model: None,
1195 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1196 extra_args: Vec::new(),
1197 env,
1198 prompt_delivery: None,
1199 }
1200 }
1201
1202 fn config(spec: AgentSpec) -> Config {
1203 Config {
1204 agents: vec![spec],
1205 graph: Graph {
1206 language: "en".to_owned(),
1207 ..Graph::default()
1208 },
1209 ..Config::default()
1210 }
1211 }
1212
1213 fn task(title: &str) -> Task {
1214 Task::new(
1215 title.to_owned(),
1216 format!("do {title}"),
1217 std::path::PathBuf::from("."),
1218 Source::Human,
1219 )
1220 }
1221
1222 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1223 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1224
1225 fn env(reply: &str) -> BTreeMap<String, String> {
1226 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1227 }
1228
1229 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1230
1231 fn init_repo_with_branch(dir: &Path, branch: &str) {
1235 use crate::proc::Quiet as _;
1236 let run = |args: &[&str]| {
1237 let out = std::process::Command::new("git")
1238 .args(args)
1239 .current_dir(dir)
1240 .quiet()
1241 .output()
1242 .expect("spawn git");
1243 assert!(
1244 out.status.success(),
1245 "git {args:?} failed: {}",
1246 String::from_utf8_lossy(&out.stderr)
1247 );
1248 };
1249 run(&["init", "-b", "main"]);
1250 run(&["config", "user.name", "magi test"]);
1251 run(&["config", "user.email", "magi@example.com"]);
1252 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1253 run(&["add", "-A"]);
1254 run(&["commit", "-m", "init"]);
1255 run(&["checkout", "-b", branch]);
1256 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1257 run(&["add", "-A"]);
1258 run(&["commit", "-m", "candidate work"]);
1259 }
1260
1261 fn review_round_with_finding(
1262 round: usize,
1263 finding_id: &str,
1264 title: &str,
1265 addressed: &[&str],
1266 rejected: &[(&str, &str)],
1267 ) -> crate::run::ReviewRound {
1268 crate::run::ReviewRound {
1269 round,
1270 head: "deadbeef".to_owned(),
1271 verified_head: None,
1272 verified_at: None,
1273 reviews: vec![crate::run::ReviewRecord {
1274 attempts: 0,
1275 reviewer: 1,
1276 agent: "mock".to_owned(),
1277 summary: String::new(),
1278 findings: vec![crate::verdict::Finding {
1279 id: finding_id.to_owned(),
1280 severity: crate::verdict::Severity::Major,
1281 file: None,
1282 line: None,
1283 title: title.to_owned(),
1284 detail: String::new(),
1285 }],
1286 vote: None,
1287 failed: None,
1288 duration_ms: 0,
1289 }],
1290 e2e: Vec::new(),
1291 verify_retried: false,
1292 e2e_deferred: false,
1293 e2e_defer_reason: None,
1294 fix: Some(crate::run::FixRecord {
1295 agent: "mock".to_owned(),
1296 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1297 rejected: rejected
1298 .iter()
1299 .map(|(id, why)| crate::verdict::Rejection {
1300 id: (*id).to_owned(),
1301 why: (*why).to_owned(),
1302 })
1303 .collect(),
1304 notes: String::new(),
1305 committed: false,
1306 failed: None,
1307 duration_ms: 0,
1308 continuation: None,
1309 }),
1310 blocking: 1,
1311 answered: 1,
1312 expected: 1,
1313 clean: false,
1314 progressed: true,
1315 vote_split: false,
1316 reconsideration: Vec::new(),
1317 verdict: None,
1318 }
1319 }
1320
1321 #[test]
1322 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1323 crate::run::pin_test_home();
1324 let dir = tempdir().unwrap();
1325 let queue = Queue::at(dir.path().join("queue"));
1326 let questions = Questions::at(dir.path().join("questions"));
1327 let mut t = task("recover");
1328 t.answers.push(crate::queue::AnsweredQuestion {
1329 question: "Inspect remote first?".into(),
1330 answer: "Inspect, then await resumption instructions".into(),
1331 });
1332 t.hold_machine(Some("waiting for remote inspection".into()));
1333 queue.put(&mut t).unwrap();
1334 let snapshot = queue.get(&t.id).unwrap();
1335 t.edit(
1336 "recover".into(),
1337 "Reuse the measured PR head; fix Clippy; review".into(),
1338 )
1339 .unwrap();
1340 t.release_operator("cli", None);
1341 queue.put(&mut t).unwrap();
1342 let approval = t.resumption.clone();
1343 let d = Decision {
1344 id: t.id.clone(),
1345 recovery: Some(Recovery::Hold),
1346 reason: Some("remote inspection still required".into()),
1347 ..Decision::default()
1348 };
1349 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1350 let stale_question = Decision {
1351 question: Some("Inspect remote?".into()),
1352 ..d.clone()
1353 };
1354 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1355 assert!(questions.list().is_empty());
1356 for reason in [
1357 "waiting for remote inspection",
1358 "inspection has not been confirmed",
1359 ] {
1360 let repeated = Decision {
1361 reason: Some(reason.into()),
1362 ..d.clone()
1363 };
1364 apply(
1365 &queue,
1366 &questions,
1367 &Verdict {
1368 decisions: vec![repeated],
1369 },
1370 )
1371 .unwrap();
1372 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1373 }
1374 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1375 assert_eq!(reloaded.resumption, approval);
1376 assert_eq!(reloaded.answers, t.answers);
1377 let v = view(&reloaded, 3);
1378 assert!(v.operator_resume.unwrap().contains("superseded"));
1379 let mut dep = task("new prerequisite");
1380 dep.hold_machine(Some("new authentication failure".into()));
1381 queue.put(&mut dep).unwrap();
1382 let new_blocker = Decision {
1383 blocked_by: vec![dep.id.clone()],
1384 evidence: vec![dep.id.clone()],
1385 reason: Some("new prerequisite failed".into()),
1386 ..d.clone()
1387 };
1388 apply(
1389 &queue,
1390 &questions,
1391 &Verdict {
1392 decisions: vec![new_blocker.clone()],
1393 },
1394 )
1395 .unwrap();
1396 let mut after = queue.get(&t.id).unwrap();
1397 assert_eq!(after.status, TaskStatus::Blocked);
1398 after.release();
1401 queue.put(&mut after).unwrap();
1402 apply(
1403 &queue,
1404 &questions,
1405 &Verdict {
1406 decisions: vec![new_blocker],
1407 },
1408 )
1409 .unwrap();
1410 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1411 after.start("new-run".into());
1412 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1413 after.hold_manual(Some("operator says wait".into()));
1414 queue.put(&mut after).unwrap();
1415 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1416 assert!(queue.get(&t.id).unwrap().operator_held());
1417 }
1418
1419 #[test]
1420 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1421 crate::run::pin_test_home();
1422 let dir = tempdir().unwrap();
1423 let queue = Queue::at(dir.path().join("queue"));
1424 let questions = Questions::at(dir.path().join("questions"));
1425 let mut t = task("resume evidence");
1426 let mut run = RunState::new(
1427 t.repo.clone(),
1428 "main".into(),
1429 "base".into(),
1430 "work".into(),
1431 crate::config::Config::default(),
1432 );
1433 run.begin_pass();
1434 run.status = crate::run::RunStatus::Failed;
1435 run.save().unwrap();
1436 t.start(run.id.clone());
1437 t.runs.push(run.id.clone());
1439 t.release_operator("cli", None);
1440 let approval = t.resumption.as_ref().unwrap().at;
1441 let decision = Decision {
1442 evidence: vec![run.id.clone()],
1443 ..Decision::default()
1444 };
1445 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1446 t.start(run.id.clone());
1447 assert_eq!(t.runs.len(), 1);
1448 run.begin_pass();
1449 run.save().unwrap();
1450 assert!(run.passes.last().unwrap().started_at > approval);
1451 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1452 assert_eq!(evidence.len(), 1);
1453 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1454 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1455 run.begin_pass();
1456 run.save().unwrap();
1457 assert_eq!(
1458 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1459 1
1460 );
1461
1462 let mut q = Question::new(
1463 run.id.clone(),
1464 "review".into(),
1465 "check".into(),
1466 "New failure".into(),
1467 String::new(),
1468 Vec::new(),
1469 );
1470 questions.put(&mut q).unwrap();
1471 let decision = Decision {
1472 evidence: vec![q.id.clone()],
1473 ..Decision::default()
1474 };
1475 assert_eq!(
1476 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1477 1
1478 );
1479 q.asked_at = approval;
1480 questions.put(&mut q).unwrap();
1481 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1482 q.asked_at = jiff::Timestamp::now();
1483 q.run = "unrelated".into();
1484 questions.put(&mut q).unwrap();
1485 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1486
1487 let mut fresh = RunState::new(
1488 t.repo.clone(),
1489 "main".into(),
1490 "base".into(),
1491 "fresh work".into(),
1492 crate::config::Config::default(),
1493 );
1494 fresh.status = crate::run::RunStatus::Failed;
1495 fresh.save().unwrap();
1496 t.start(fresh.id.clone());
1497 let decision = Decision {
1498 evidence: vec![fresh.id.clone()],
1499 ..Decision::default()
1500 };
1501 assert_eq!(
1502 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1503 1
1504 );
1505 }
1506
1507 #[test]
1508 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1509 crate::run::pin_test_home();
1510 let dir = tempdir().unwrap();
1511 let default_repo = dir.path().join("default");
1512 let task_repo = dir.path().join("task");
1513 std::fs::create_dir_all(&default_repo).unwrap();
1514 std::fs::create_dir_all(&task_repo).unwrap();
1515 init_repo_with_branch(&default_repo, "other-branch");
1516 init_repo_with_branch(&task_repo, "magi/f00d/A");
1517
1518 let mut config = Config::default();
1519 config.graph.review_rounds = 6;
1520 let mut state = crate::run::RunState::new(
1521 task_repo.clone(),
1522 "main".to_owned(),
1523 "deadbeef".to_owned(),
1524 "task".to_owned(),
1525 config,
1526 );
1527 state.status = crate::run::RunStatus::Blocked;
1528 state.candidates.push(crate::run::Candidate {
1529 index: 0,
1530 label: 'A',
1531 agent: "mock".to_owned(),
1532 branch: "magi/f00d/A".to_owned(),
1533 worktree: task_repo.clone(),
1534 summary: String::new(),
1535 stat: String::new(),
1536 files: 1,
1537 commits: 1,
1538 empty: false,
1539 failed: None,
1540 verified_noop: None,
1541 duration_ms: 0,
1542 folded: false,
1543 });
1544 state.tally = Some(crate::run::Tally {
1545 first_choice: std::collections::BTreeMap::new(),
1546 borda: std::collections::BTreeMap::new(),
1547 winner: 'A',
1548 rankings: 0,
1549 unanimous_initial: false,
1550 deliberated: false,
1551 changed_votes: 0,
1552 unanimous_final: false,
1553 tie_break: None,
1554 judges: 0,
1555 present: 0,
1556 quorum: 0,
1557 met_quorum: true,
1558 uncontested: Some("solo".to_owned()),
1559 });
1560 state.reviews = vec![
1561 review_round_with_finding(
1562 1,
1563 "R1-1-2",
1564 "answer content is dropped",
1565 &[],
1566 &[("R1-1-2", "the id leaving blocked_by is enough")],
1567 ),
1568 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1569 ];
1570 state.save().unwrap();
1571
1572 let mut t = task("outcome test");
1573 t.repo = task_repo;
1574 t.runs.push(state.id.clone());
1575
1576 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1577 let outcome = finished.outcome;
1578
1579 assert!(outcome.unreadable.is_none());
1580 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1581 assert_eq!(outcome.rounds_used, 2);
1582 assert_eq!(outcome.rounds_max, 6);
1583 assert_eq!(outcome.rounds.len(), 2);
1584 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1585 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1586 assert!(outcome.rounds[1].addressed.is_empty());
1587 assert!(outcome.rounds[1].rejected.is_empty());
1588 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1589 assert!(
1590 outcome.branch_head.is_some(),
1591 "a real branch must resolve a head commit: {outcome:?}"
1592 );
1593 }
1594
1595 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1599 tokio::runtime::Builder::new_current_thread()
1600 .enable_all()
1601 .build()
1602 .unwrap()
1603 .block_on(f)
1604 }
1605
1606 #[test]
1607 fn view_resolves_task_attempt_limit_before_daemon_default() {
1608 let mut t = task("budget");
1609 assert_eq!(view(&t, 7).max_attempts, 7);
1610 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1611 .unwrap();
1612 assert_eq!(view(&t, 7).max_attempts, 3);
1613 t.change_attempt_limit(None).unwrap();
1614 assert_eq!(view(&t, 7).max_attempts, 7);
1615 }
1616
1617 #[test]
1618 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1619 let mut t = task("answered");
1620 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1621 let v = view(&t, 2);
1622 assert_eq!(v.answers.len(), 1);
1623 assert_eq!(v.answers[0].question, "Which backend?");
1624 assert_eq!(v.answers[0].answer, "SQLite");
1625 }
1626
1627 #[test]
1628 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
1629 let dir = tempdir().unwrap();
1630 let queue = Queue::at(dir.path().join("queue"));
1631 let questions = Questions::at(dir.path().join("questions"));
1632 let mut a = task("a");
1633 a.priority = 9;
1634 queue.put(&mut a).unwrap();
1635
1636 let verdict = Verdict {
1637 decisions: vec![Decision {
1638 id: a.id.clone(),
1639 blocked_by: vec!["20260101-000000-dead".to_owned()],
1640 reason: Some("waits on the other task".to_owned()),
1641 recovery: None,
1642 question: None,
1643 choices: Vec::new(),
1644 evidence: Vec::new(),
1645 }],
1646 };
1647 apply(&queue, &questions, &verdict).unwrap();
1648
1649 let back = queue.get(&a.id).unwrap();
1650 assert_eq!(back.status, TaskStatus::Blocked);
1651 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
1652 assert_eq!(
1653 back.priority, 9,
1654 "the conductor's reply cannot carry priority"
1655 );
1656 }
1657
1658 #[test]
1659 fn a_question_decision_files_one_and_blocks_on_its_id() {
1660 let dir = tempdir().unwrap();
1661 let queue = Queue::at(dir.path().join("queue"));
1662 let questions = Questions::at(dir.path().join("questions"));
1663 let mut t = task("ambiguous");
1664 queue.put(&mut t).unwrap();
1665
1666 let verdict = Verdict {
1667 decisions: vec![Decision {
1668 id: t.id.clone(),
1669 blocked_by: Vec::new(),
1670 reason: Some("which backend?".to_owned()),
1671 recovery: None,
1672 question: Some("Which storage backend?".to_owned()),
1673 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
1674 evidence: Vec::new(),
1675 }],
1676 };
1677 apply(&queue, &questions, &verdict).unwrap();
1678
1679 let back = queue.get(&t.id).unwrap();
1680 assert_eq!(back.status, TaskStatus::Blocked);
1681 assert_eq!(back.blocked_by.len(), 1);
1682 let q = questions.get(&back.blocked_by[0]).unwrap();
1683 assert_eq!(q.summary, "Which storage backend?");
1684 assert_eq!(q.node, NODE);
1685 assert!(q.status.open());
1686 }
1687
1688 #[test]
1689 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
1690 let dir = tempdir().unwrap();
1691 let queue = Queue::at(dir.path().join("queue"));
1692 let questions = Questions::at(dir.path().join("questions"));
1693 let mut t = task("asked once");
1694 queue.put(&mut t).unwrap();
1695
1696 let decision = Decision {
1697 id: t.id.clone(),
1698 reason: Some("still deciding".to_owned()),
1699 question: Some("Which backend?".to_owned()),
1700 ..Decision::default()
1701 };
1702 apply(
1703 &queue,
1704 &questions,
1705 &Verdict {
1706 decisions: vec![decision.clone()],
1707 },
1708 )
1709 .unwrap();
1710 assert_eq!(questions.list().len(), 1);
1711 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
1712
1713 let mut released = queue.get(&t.id).unwrap();
1718 released.release();
1719 queue.put(&mut released).unwrap();
1720
1721 apply(
1722 &queue,
1723 &questions,
1724 &Verdict {
1725 decisions: vec![decision],
1726 },
1727 )
1728 .unwrap();
1729 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
1730 let after = queue.get(&t.id).unwrap();
1731 assert_eq!(
1732 after.blocked_by,
1733 [first_question_id],
1734 "the existing open question is reused, not replaced"
1735 );
1736 }
1737
1738 #[test]
1739 fn a_same_id_question_from_another_node_is_not_reused() {
1740 let dir = tempdir().unwrap();
1741 let queue = Queue::at(dir.path().join("queue"));
1742 let questions = Questions::at(dir.path().join("questions"));
1743 let mut t = task("must ask the conductor");
1744 queue.put(&mut t).unwrap();
1745
1746 let mut unrelated = Question::new(
1747 t.id.clone(),
1748 "review".to_owned(),
1749 "reviewer-1".to_owned(),
1750 "An unrelated review question".to_owned(),
1751 String::new(),
1752 Vec::new(),
1753 );
1754 questions.put(&mut unrelated).unwrap();
1755
1756 apply(
1757 &queue,
1758 &questions,
1759 &Verdict {
1760 decisions: vec![Decision {
1761 id: t.id.clone(),
1762 question: Some("Which backend?".to_owned()),
1763 ..Decision::default()
1764 }],
1765 },
1766 )
1767 .unwrap();
1768
1769 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
1770 assert_eq!(blocked_by.len(), 1);
1771 assert_ne!(blocked_by[0], unrelated.id);
1772 assert!(questions.get(&unrelated.id).unwrap().status.open());
1773 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
1774 }
1775
1776 #[test]
1777 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
1778 let dir = tempdir().unwrap();
1779 let queue = Queue::at(dir.path().join("queue"));
1780 let questions = Questions::at(dir.path().join("questions"));
1781 let mut t = task("waits on an answer");
1782 queue.put(&mut t).unwrap();
1783
1784 apply(
1785 &queue,
1786 &questions,
1787 &Verdict {
1788 decisions: vec![Decision {
1789 id: t.id.clone(),
1790 blocked_by: Vec::new(),
1791 reason: None,
1792 recovery: None,
1793 question: Some("Which backend?".to_owned()),
1794 choices: Vec::new(),
1795 evidence: Vec::new(),
1796 }],
1797 },
1798 )
1799 .unwrap();
1800 let blocked = queue.get(&t.id).unwrap();
1801 let question_id = blocked.blocked_by[0].clone();
1802
1803 let mut q = questions.get(&question_id).unwrap();
1804 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
1805 questions.put(&mut q).unwrap();
1806 assert_eq!(q.status, QuestionStatus::Answered);
1807
1808 let mut task_after = queue.get(&t.id).unwrap();
1812 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
1813 task_after.unblock(&question_id);
1814 assert_eq!(task_after.status, TaskStatus::Queued);
1815 assert_eq!(task_after.answers[0].answer, "SQLite");
1816 }
1817
1818 #[test]
1819 fn a_stalled_task_can_be_requeued_or_held() {
1820 let dir = tempdir().unwrap();
1821 let queue = Queue::at(dir.path().join("queue"));
1822 let questions = Questions::at(dir.path().join("questions"));
1823
1824 let mut requeue_me = task("stuck a");
1825 requeue_me.start("run-1".to_owned());
1826 queue.put(&mut requeue_me).unwrap();
1827
1828 let mut hold_me = task("stuck b");
1829 hold_me.start("run-2".to_owned());
1830 queue.put(&mut hold_me).unwrap();
1831
1832 apply(
1833 &queue,
1834 &questions,
1835 &Verdict {
1836 decisions: vec![
1837 Decision {
1838 id: requeue_me.id.clone(),
1839 recovery: Some(Recovery::Requeue),
1840 ..Decision::default()
1841 },
1842 Decision {
1843 id: hold_me.id.clone(),
1844 recovery: Some(Recovery::Hold),
1845 reason: Some("looks broken".to_owned()),
1846 ..Decision::default()
1847 },
1848 ],
1849 },
1850 )
1851 .unwrap();
1852
1853 let requeued = queue.get(&requeue_me.id).unwrap();
1854 assert_eq!(requeued.status, TaskStatus::Queued);
1855 assert_eq!(requeued.attempts, 0);
1856
1857 let held = queue.get(&hold_me.id).unwrap();
1858 assert_eq!(held.status, TaskStatus::Held);
1859 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
1860 }
1861
1862 #[test]
1863 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
1864 let dir = tempdir().unwrap();
1869 let queue = Queue::at(dir.path().join("queue"));
1870 let questions = Questions::at(dir.path().join("questions"));
1871 let mut t = task("held out of attempts");
1872 t.hold_machine(Some("out of attempts".to_owned()));
1873 queue.put(&mut t).unwrap();
1874
1875 apply(
1876 &queue,
1877 &questions,
1878 &Verdict {
1879 decisions: vec![Decision {
1880 id: t.id.clone(),
1881 reason: Some("what should happen to this one?".to_owned()),
1882 question: Some("Hold it, or try again?".to_owned()),
1883 ..Decision::default()
1884 }],
1885 },
1886 )
1887 .unwrap();
1888 let blocked = queue.get(&t.id).unwrap();
1889 assert_eq!(blocked.status, TaskStatus::Blocked);
1890 let question_id = blocked.blocked_by[0].clone();
1891
1892 let mut q = questions.get(&question_id).unwrap();
1893 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
1894 questions.put(&mut q).unwrap();
1895
1896 let mut after = queue.get(&t.id).unwrap();
1898 after.record_answer(q.summary.clone(), "leave it held".to_owned());
1899 after.unblock(&question_id);
1900 assert_eq!(
1901 after.status,
1902 TaskStatus::Held,
1903 "must not fall back to queued"
1904 );
1905 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
1906 }
1907
1908 #[test]
1909 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
1910 let dir = tempdir().unwrap();
1920 let queue = Queue::at(dir.path().join("queue"));
1921 let questions = Questions::at(dir.path().join("questions"));
1922 let mut t = task("disk pressure, then reconsidered");
1923 t.hold_machine(Some(
1924 "not enough free space to start a run: 10 bytes free, 100 required by \
1925 `[disk] min_free_bytes`"
1926 .to_owned(),
1927 ));
1928 t.record_answer(
1929 "How should this be handled?".to_owned(),
1930 "keep it held, a human will look at it later".to_owned(),
1931 );
1932 queue.put(&mut t).unwrap();
1933
1934 apply(
1935 &queue,
1936 &questions,
1937 &Verdict {
1938 decisions: vec![Decision {
1939 id: t.id.clone(),
1940 recovery: Some(Recovery::Hold),
1941 ..Decision::default()
1942 }],
1943 },
1944 )
1945 .unwrap();
1946
1947 let after = queue.get(&t.id).unwrap();
1948 assert_eq!(after.status, TaskStatus::Held);
1949 assert!(
1950 !after
1951 .hold_reason
1952 .as_deref()
1953 .unwrap_or_default()
1954 .starts_with("not enough free space"),
1955 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
1956 after.hold_reason
1957 );
1958
1959 let cfg_dir = tempdir().unwrap();
1962 let config = cfg_dir.path().join("magi.toml");
1963 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1964 let report =
1965 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
1966 assert!(report.resumed.is_empty(), "must not be auto-released");
1967 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
1968 }
1969
1970 #[test]
1971 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
1972 crate::run::pin_test_home();
1973 let dir = tempdir().unwrap();
1974 let queue = Queue::at(dir.path().join("queue"));
1975 let questions = Questions::at(dir.path().join("questions"));
1976 let config = dir.path().join("magi.toml");
1977 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1978 let now = jiff::Timestamp::now();
1979 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
1980 let open = || {
1981 questions
1982 .list()
1983 .into_iter()
1984 .filter(|q| q.node == "triage" && q.status.open())
1985 .collect::<Vec<_>>()
1986 };
1987 let hold = Decision {
1988 recovery: Some(Recovery::Hold),
1989 reason: Some("waiting on manual worktree cleanup".to_owned()),
1990 ..Decision::default()
1991 };
1992
1993 let mut t = task("looping hold");
1994 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
1995 queue.put(&mut t).unwrap();
1996 let hold = Decision {
1997 id: t.id.clone(),
1998 ..hold
1999 };
2000
2001 assert_eq!(triage().asked.len(), 1);
2003 let first = open().remove(0);
2004 let mut q = questions.get(&first.id).unwrap();
2005 let resume = q.choices[0].clone();
2006 q.answer(Answer::Choice(resume)).unwrap();
2007 questions.put(&mut q).unwrap();
2008 assert_eq!(triage().answered.len(), 1);
2009 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2010
2011 apply(
2013 &queue,
2014 &questions,
2015 &Verdict {
2016 decisions: vec![hold.clone()],
2017 },
2018 )
2019 .unwrap();
2020 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2021 assert!(triage().asked.is_empty());
2022
2023 let mut evidence = Question::new(
2025 t.id.clone(),
2026 "preflight".into(),
2027 "check".into(),
2028 "New permission failure".into(),
2029 String::new(),
2030 Vec::new(),
2031 );
2032 questions.put(&mut evidence).unwrap();
2033 let hold = Decision {
2034 evidence: vec![evidence.id.clone()],
2035 reason: Some("new permission check failed".into()),
2036 ..hold
2037 };
2038 apply(
2039 &queue,
2040 &questions,
2041 &Verdict {
2042 decisions: vec![hold.clone()],
2043 },
2044 )
2045 .unwrap();
2046 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2047
2048 assert_eq!(triage().asked.len(), 1);
2050 let second = open().remove(0);
2051 assert!(second.detail.contains("new permission check failed"));
2052 assert!(triage().asked.is_empty(), "no duplicate question");
2053 assert_eq!(open().len(), 1);
2054
2055 let mut q = questions.get(&second.id).unwrap();
2057 let force = q.choices[0].clone();
2058 q.answer(Answer::Choice(force)).unwrap();
2059 questions.put(&mut q).unwrap();
2060 assert_eq!(triage().answered.len(), 1);
2061 apply(
2062 &queue,
2063 &questions,
2064 &Verdict {
2065 decisions: vec![hold],
2066 },
2067 )
2068 .unwrap();
2069 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2070 }
2071
2072 #[test]
2073 fn a_runnable_task_can_be_held_directly_without_a_question() {
2074 let dir = tempdir().unwrap();
2075 let queue = Queue::at(dir.path().join("queue"));
2076 let questions = Questions::at(dir.path().join("questions"));
2077 let mut t = task("already answered, should stay put");
2078 queue.put(&mut t).unwrap();
2079
2080 apply(
2081 &queue,
2082 &questions,
2083 &Verdict {
2084 decisions: vec![Decision {
2085 id: t.id.clone(),
2086 recovery: Some(Recovery::Hold),
2087 reason: Some("operator already said keep this held".to_owned()),
2088 ..Decision::default()
2089 }],
2090 },
2091 )
2092 .unwrap();
2093
2094 let after = queue.get(&t.id).unwrap();
2095 assert_eq!(after.status, TaskStatus::Held);
2096 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2097 }
2098
2099 #[test]
2100 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2101 crate::run::pin_test_home();
2102 let dir = tempdir().unwrap();
2108 let queue = Queue::at(dir.path().join("queue"));
2109 let questions = Questions::at(dir.path().join("questions"));
2110 let mut t = task("already merged by hand");
2111 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2112 t.record_answer(
2113 "Handle this one?".to_owned(),
2114 "already merged and cleaned up, close it".to_owned(),
2115 );
2116 queue.put(&mut t).unwrap();
2117
2118 apply(
2119 &queue,
2120 &questions,
2121 &Verdict {
2122 decisions: vec![Decision {
2123 id: t.id.clone(),
2124 recovery: Some(Recovery::Done),
2125 reason: Some("operator confirmed this already landed".to_owned()),
2126 ..Decision::default()
2127 }],
2128 },
2129 )
2130 .unwrap();
2131
2132 let after = queue.get(&t.id).unwrap();
2133 assert_eq!(after.status, TaskStatus::Done);
2134 assert!(after.hold_reason.is_none());
2135 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2136 }
2137
2138 #[test]
2139 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2140 let dir = tempdir().unwrap();
2141 let queue = Queue::at(dir.path().join("queue"));
2142 let questions = Questions::at(dir.path().join("questions"));
2143
2144 let mut queued = task("never ran yet");
2145 queue.put(&mut queued).unwrap();
2146
2147 let mut running = task("mid-run");
2148 running.start("run-1".to_owned());
2149 queue.put(&mut running).unwrap();
2150
2151 for id in [queued.id.clone(), running.id.clone()] {
2152 apply(
2153 &queue,
2154 &questions,
2155 &Verdict {
2156 decisions: vec![Decision {
2157 id,
2158 recovery: Some(Recovery::Done),
2159 ..Decision::default()
2160 }],
2161 },
2162 )
2163 .unwrap();
2164 }
2165
2166 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2167 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2168 }
2169
2170 #[test]
2171 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2172 let dir = tempdir().unwrap();
2175 let queue = Queue::at(dir.path().join("queue"));
2176 let questions = Questions::at(dir.path().join("questions"));
2177 let mut t = task("asked about repeatedly");
2178 t.hold_machine(Some("out of attempts".to_owned()));
2179 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2180 t.record_answer(
2181 "Handle this one? (2)".to_owned(),
2182 "still not yet".to_owned(),
2183 );
2184 queue.put(&mut t).unwrap();
2185 assert_eq!(questions.list().len(), 0);
2186
2187 let verdict = Verdict {
2188 decisions: vec![Decision {
2189 id: t.id.clone(),
2190 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2191 ..Decision::default()
2192 }],
2193 };
2194 apply(&queue, &questions, &verdict).unwrap();
2195
2196 let filed = questions.list();
2197 assert_eq!(filed.len(), 1, "the question was filed");
2198 assert_eq!(
2199 filed[0].summary,
2200 "Branch conflicts with origin/main, how do we proceed?"
2201 );
2202 let deputy = filed[0]
2203 .deputy
2204 .as_ref()
2205 .expect("the wait is handed to a deputy");
2206 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2207 assert_eq!(
2208 deputy.starts, 0,
2209 "filing never starts anything: the loop does not wait"
2210 );
2211 assert!(filed[0].status.open());
2212 assert_eq!(filed[0].node, NODE);
2213 let after = queue.get(&t.id).unwrap();
2214 assert_eq!(after.status, TaskStatus::Blocked);
2215 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2216 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2217 assert!(
2218 !after
2219 .hold_reason
2220 .clone()
2221 .unwrap_or_default()
2222 .contains("conduct tried to ask"),
2223 "no hold was applied"
2224 );
2225
2226 apply(&queue, &questions, &verdict).unwrap();
2227 let again = questions.list();
2228 assert_eq!(again.len(), 1, "the open question is reused");
2229 assert_eq!(again[0].id, filed[0].id);
2230 }
2231
2232 #[test]
2233 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
2234 let dir = tempdir().unwrap();
2235 let queue = Queue::at(dir.path().join("queue"));
2236 let questions = Questions::at(dir.path().join("questions"));
2237 let mut t = task("asked about once already");
2238 t.hold_machine(Some("out of attempts".to_owned()));
2239 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
2240 queue.put(&mut t).unwrap();
2241
2242 apply(
2243 &queue,
2244 &questions,
2245 &Verdict {
2246 decisions: vec![Decision {
2247 id: t.id.clone(),
2248 question: Some("Still not sure - now what?".to_owned()),
2249 ..Decision::default()
2250 }],
2251 },
2252 )
2253 .unwrap();
2254
2255 assert_eq!(questions.list().len(), 1, "the second question was filed");
2256 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2257 }
2258
2259 #[test]
2260 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
2261 let dir = tempdir().unwrap();
2267 let queue = Queue::at(dir.path().join("queue"));
2268 let questions = Questions::at(dir.path().join("questions"));
2269 let mut t = task("held, triage already asking about it");
2270 t.hold_machine(Some("cause unclear".to_owned()));
2271 queue.put(&mut t).unwrap();
2272
2273 let mut triage_q = Question::new(
2274 t.id.clone(),
2275 crate::triage::NODE.to_owned(),
2276 "triage".to_owned(),
2277 "Still needed?".to_owned(),
2278 String::new(),
2279 vec![
2280 "resume".to_owned(),
2281 "not yet".to_owned(),
2282 "discard".to_owned(),
2283 ],
2284 );
2285 questions.put(&mut triage_q).unwrap();
2286
2287 for decision in [
2288 Decision {
2289 id: t.id.clone(),
2290 question: Some("what now?".to_owned()),
2291 ..Decision::default()
2292 },
2293 Decision {
2294 id: t.id.clone(),
2295 recovery: Some(Recovery::Requeue),
2296 ..Decision::default()
2297 },
2298 ] {
2299 apply(
2300 &queue,
2301 &questions,
2302 &Verdict {
2303 decisions: vec![decision],
2304 },
2305 )
2306 .unwrap();
2307 }
2308
2309 let after = queue.get(&t.id).unwrap();
2310 assert_eq!(
2311 after.status,
2312 TaskStatus::Held,
2313 "triage still owns this hold"
2314 );
2315 assert!(after.blocked_by.is_empty());
2316 assert_eq!(
2317 questions.list().len(),
2318 1,
2319 "no second, conductor-owned question was filed"
2320 );
2321 }
2322
2323 #[test]
2324 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
2325 let dir = tempdir().unwrap();
2333 let queue = Queue::at(dir.path().join("queue"));
2334 let questions = Questions::at(dir.path().join("questions"));
2335 let mut t = task("held, triage question answered but not yet applied");
2336 t.hold_machine(Some("cause unclear".to_owned()));
2337 queue.put(&mut t).unwrap();
2338
2339 let mut triage_q = Question::new(
2340 t.id.clone(),
2341 crate::triage::NODE.to_owned(),
2342 "triage".to_owned(),
2343 "Still needed?".to_owned(),
2344 String::new(),
2345 vec![
2346 "resume".to_owned(),
2347 "not yet".to_owned(),
2348 "discard".to_owned(),
2349 ],
2350 );
2351 questions.put(&mut triage_q).unwrap();
2352 triage_q
2353 .answer(Answer::Choice("not yet".to_owned()))
2354 .unwrap();
2355 questions.put(&mut triage_q).unwrap();
2356 assert!(!triage_q.status.open());
2357
2358 apply(
2359 &queue,
2360 &questions,
2361 &Verdict {
2362 decisions: vec![Decision {
2363 id: t.id.clone(),
2364 question: Some("what now?".to_owned()),
2365 ..Decision::default()
2366 }],
2367 },
2368 )
2369 .unwrap();
2370
2371 let after = queue.get(&t.id).unwrap();
2372 assert_eq!(
2373 after.status,
2374 TaskStatus::Held,
2375 "triage's own answer is not yet applied - conduct must wait"
2376 );
2377 assert_eq!(
2378 questions.list().len(),
2379 1,
2380 "no conductor question was filed over the pending triage answer"
2381 );
2382 }
2383
2384 #[test]
2385 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
2386 let dir = tempdir().unwrap();
2387 let queue = Queue::at(dir.path().join("queue"));
2388 let questions = Questions::at(dir.path().join("questions"));
2389 let mut held = task("manual recovery");
2390 held.priority = 300;
2391 held.runs.push("run20260912-224242-daf5".to_owned());
2392 held.hold_manual(Some(
2393 "active manual recovery run20260912-224242-daf5".to_owned(),
2394 ));
2395 queue.put(&mut held).unwrap();
2396
2397 for decision in [
2401 Decision {
2402 id: held.id.clone(),
2403 recovery: Some(Recovery::Requeue),
2404 ..Decision::default()
2405 },
2406 Decision {
2407 id: held.id.clone(),
2408 recovery: Some(Recovery::Hold),
2409 reason: Some("stale replacement reason".to_owned()),
2410 ..Decision::default()
2411 },
2412 Decision {
2413 id: held.id.clone(),
2414 recovery: Some(Recovery::Review),
2415 ..Decision::default()
2416 },
2417 Decision {
2418 id: held.id.clone(),
2419 blocked_by: vec!["other-task".to_owned()],
2420 question: Some("retry now?".to_owned()),
2421 ..Decision::default()
2422 },
2423 ] {
2424 apply(
2425 &queue,
2426 &questions,
2427 &Verdict {
2428 decisions: vec![decision],
2429 },
2430 )
2431 .unwrap();
2432 }
2433
2434 let after = queue.get(&held.id).unwrap();
2435 assert_eq!(after.status, TaskStatus::Held);
2436 assert!(after.operator_held());
2437 assert_eq!(after.priority, 300);
2438 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
2439 assert_eq!(
2440 after.hold_reason.as_deref(),
2441 Some("active manual recovery run20260912-224242-daf5")
2442 );
2443 assert!(after.blocked_by.is_empty());
2444 assert!(questions.list().is_empty());
2445 assert!(
2446 queue.next_runnable().is_none(),
2447 "must not dispatch a duplicate"
2448 );
2449 }
2450
2451 #[test]
2452 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
2453 let dir = tempdir().unwrap();
2454 let queue = Queue::at(dir.path().join("queue"));
2455 let questions = Questions::at(dir.path().join("questions"));
2456
2457 let mut automatic = task("disk gate");
2458 automatic.hold_machine(Some("disk full".to_owned()));
2459 queue.put(&mut automatic).unwrap();
2460 let requeue = || Verdict {
2461 decisions: vec![Decision {
2462 id: automatic.id.clone(),
2463 recovery: Some(Recovery::Requeue),
2464 ..Decision::default()
2465 }],
2466 };
2467 apply(&queue, &questions, &requeue()).unwrap();
2468 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
2469
2470 let mut manual = task("operator gate");
2471 manual.hold_manual(Some("wait for operator".to_owned()));
2472 queue.put(&mut manual).unwrap();
2473 apply(
2474 &queue,
2475 &questions,
2476 &Verdict {
2477 decisions: vec![Decision {
2478 id: manual.id.clone(),
2479 recovery: Some(Recovery::Requeue),
2480 ..Decision::default()
2481 }],
2482 },
2483 )
2484 .unwrap();
2485 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
2486
2487 let mut released = queue.get(&manual.id).unwrap();
2490 released.release();
2491 queue.put(&mut released).unwrap();
2492 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
2493 }
2494
2495 #[test]
2496 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
2497 let dir = tempdir().unwrap();
2498 let queue = Queue::at(dir.path().join("queue"));
2499 let questions = Questions::at(dir.path().join("questions"));
2500 let mut legacy = task("old explicit hold");
2501 legacy.status = TaskStatus::Held;
2502 legacy.hold_reason = Some("manual recovery already active".to_owned());
2503 legacy.hold_source = None;
2504 legacy.blocked_by = vec!["dependency".to_owned()];
2505 queue.put(&mut legacy).unwrap();
2506
2507 apply(
2508 &queue,
2509 &questions,
2510 &Verdict {
2511 decisions: vec![Decision {
2512 id: legacy.id.clone(),
2513 recovery: Some(Recovery::Requeue),
2514 ..Decision::default()
2515 }],
2516 },
2517 )
2518 .unwrap();
2519
2520 let after = queue.get(&legacy.id).unwrap();
2521 assert_eq!(after.status, TaskStatus::Held);
2522 assert_eq!(after.hold_source, None);
2523 assert_eq!(after.hold_reason, legacy.hold_reason);
2524 assert_eq!(after.blocked_by, legacy.blocked_by);
2525 }
2526
2527 #[test]
2528 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
2529 crate::run::pin_test_home();
2535 let dir = tempdir().unwrap();
2536 let queue = Queue::at(dir.path().join("queue"));
2537 let questions = Questions::at(dir.path().join("questions"));
2538 let mut t = task("blocked with no readable run");
2539 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
2541 queue.put(&mut t).unwrap();
2542
2543 apply(
2544 &queue,
2545 &questions,
2546 &Verdict {
2547 decisions: vec![Decision {
2548 id: t.id.clone(),
2549 recovery: Some(Recovery::Review),
2550 ..Decision::default()
2551 }],
2552 },
2553 )
2554 .unwrap();
2555
2556 let after = queue.get(&t.id).unwrap();
2557 assert_eq!(
2558 after.status,
2559 TaskStatus::Failed,
2560 "with nothing to reopen, the decision is dropped rather than guessed at"
2561 );
2562 assert!(after.review_branch.is_none());
2563 }
2564
2565 #[test]
2566 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
2567 let dir = tempdir().unwrap();
2571 let queue = Queue::at(dir.path().join("queue"));
2572 let questions = Questions::at(dir.path().join("questions"));
2573
2574 for recovery in [Recovery::Requeue, Recovery::Review] {
2575 let mut t = task("ordinary");
2576 queue.put(&mut t).unwrap();
2577
2578 apply(
2579 &queue,
2580 &questions,
2581 &Verdict {
2582 decisions: vec![Decision {
2583 id: t.id.clone(),
2584 recovery: Some(recovery),
2585 ..Decision::default()
2586 }],
2587 },
2588 )
2589 .unwrap();
2590
2591 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2592 }
2593 }
2594
2595 #[tokio::test]
2596 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
2597 let dir = tempdir().unwrap();
2598 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
2599 let queue = Queue::at(dir.path().join("queue"));
2600 let questions = Questions::at(dir.path().join("questions"));
2601 let mut t = task("normal");
2602 queue.put(&mut t).unwrap();
2603
2604 let mut conductor = Conductor::new();
2605 conductor
2606 .maybe_run(
2607 &cfg,
2608 dir.path(),
2609 &queue,
2610 &questions,
2611 dir.path(),
2612 &[t.clone()],
2613 &[],
2614 &[],
2615 2,
2616 )
2617 .await;
2618
2619 assert_eq!(
2620 queue.get(&t.id).unwrap().status,
2621 TaskStatus::Queued,
2622 "a failed invocation must change nothing"
2623 );
2624 assert!(
2625 queue.next_runnable().is_some(),
2626 "the loop must still be able to take the next task"
2627 );
2628 }
2629
2630 #[tokio::test]
2631 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
2632 let dir = tempdir().unwrap();
2633 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
2634 let queue = Queue::at(dir.path().join("queue"));
2635 let questions = Questions::at(dir.path().join("questions"));
2636 let mut t = task("normal");
2637 queue.put(&mut t).unwrap();
2638
2639 let mut conductor = Conductor::new();
2640 conductor
2641 .maybe_run(
2642 &cfg,
2643 dir.path(),
2644 &queue,
2645 &questions,
2646 dir.path(),
2647 &[t.clone()],
2648 &[],
2649 &[],
2650 2,
2651 )
2652 .await;
2653
2654 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2655 }
2656
2657 #[tokio::test]
2658 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
2659 let dir = tempdir().unwrap();
2660 let mut t = task("chained");
2661 let reply = format!(
2662 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
2663 t.id
2664 );
2665 let counted = |id: &str, tail: &str| {
2666 let calls = dir.path().join(format!("{id}.calls"));
2667 let path = dir.path().join(format!("{id}.sh"));
2668 std::fs::write(
2669 &path,
2670 format!(
2671 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
2672 calls.display()
2673 ),
2674 )
2675 .unwrap();
2676 AgentSpec {
2677 id: id.to_owned(),
2678 kind: AgentKind::Command,
2679 model: None,
2680 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
2681 extra_args: Vec::new(),
2682 env: BTreeMap::new(),
2683 prompt_delivery: None,
2684 }
2685 };
2686 let n_calls = |id: &str| {
2687 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
2688 .map_or(0, |s| s.lines().count())
2689 };
2690 let mut cfg = config(counted("a", "exit 3"));
2691 cfg.agents = vec![
2692 counted("a", "exit 3"),
2693 counted("b", &format!("printf '%s' '{reply}'")),
2694 ];
2695 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
2696 "a".into(),
2697 "b".into(),
2698 "a".into(),
2699 ]));
2700 let queue = Queue::at(dir.path().join("queue"));
2701 let questions = Questions::at(dir.path().join("questions"));
2702 queue.put(&mut t).unwrap();
2703
2704 let mut conductor = Conductor::new();
2705 conductor
2706 .maybe_run(
2707 &cfg,
2708 dir.path(),
2709 &queue,
2710 &questions,
2711 dir.path(),
2712 &[t.clone()],
2713 &[],
2714 &[],
2715 2,
2716 )
2717 .await;
2718
2719 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2720 assert_eq!(n_calls("a"), 1, "each id is tried once");
2721 assert_eq!(n_calls("b"), 1);
2722 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
2723 assert!(
2724 saved.contains("\"b\""),
2725 "the seat on disk is the one that ran: {saved}"
2726 );
2727 }
2728
2729 #[tokio::test]
2730 async fn json_survives_code_fences_and_a_preamble() {
2731 let dir = tempdir().unwrap();
2732 let mut t = task("fenced");
2733 let reply = format!(
2734 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
2735 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
2736 t.id
2737 );
2738 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2739 let queue = Queue::at(dir.path().join("queue"));
2740 let questions = Questions::at(dir.path().join("questions"));
2741 queue.put(&mut t).unwrap();
2742
2743 let mut conductor = Conductor::new();
2744 conductor
2745 .maybe_run(
2746 &cfg,
2747 dir.path(),
2748 &queue,
2749 &questions,
2750 dir.path(),
2751 &[t.clone()],
2752 &[],
2753 &[],
2754 2,
2755 )
2756 .await;
2757
2758 let back = queue.get(&t.id).unwrap();
2759 assert_eq!(back.status, TaskStatus::Blocked);
2760 assert_eq!(back.blocked_by, ["x"]);
2761 }
2762
2763 #[tokio::test]
2764 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
2765 let dir = tempdir().unwrap();
2768 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2769 let queue = Queue::at(dir.path().join("queue"));
2770 let questions = Questions::at(dir.path().join("questions"));
2771 let mut t = task("stable");
2772 queue.put(&mut t).unwrap();
2773 let artifacts = dir.path().join("conduct").join("artifacts");
2774 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2775
2776 let mut conductor = Conductor::new();
2777 conductor
2778 .maybe_run(
2779 &cfg,
2780 dir.path(),
2781 &queue,
2782 &questions,
2783 dir.path(),
2784 &[t.clone()],
2785 &[],
2786 &[],
2787 2,
2788 )
2789 .await;
2790 assert!(turn(1).is_file(), "the first cycle must call the conductor");
2791
2792 conductor
2793 .maybe_run(
2794 &cfg,
2795 dir.path(),
2796 &queue,
2797 &questions,
2798 dir.path(),
2799 &[t.clone()],
2800 &[],
2801 &[],
2802 2,
2803 )
2804 .await;
2805 assert!(
2806 !turn(2).is_file(),
2807 "an unchanged revision and an unchanged stalled/finished set must not call the \
2808 conductor twice"
2809 );
2810
2811 t.priority = 1;
2813 queue.put(&mut t).unwrap();
2814 conductor
2815 .maybe_run(
2816 &cfg,
2817 dir.path(),
2818 &queue,
2819 &questions,
2820 dir.path(),
2821 &[t.clone()],
2822 &[],
2823 &[],
2824 2,
2825 )
2826 .await;
2827 assert!(turn(2).is_file(), "a moved revision calls it again");
2828 }
2829
2830 #[tokio::test]
2831 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
2832 let dir = tempdir().unwrap();
2838 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2839 let queue = Queue::at(dir.path().join("queue"));
2840 let questions = Questions::at(dir.path().join("questions"));
2841 let mut t = task("quiet");
2842 queue.put(&mut t).unwrap();
2843 let artifacts = dir.path().join("conduct").join("artifacts");
2844 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2845
2846 let mut conductor = Conductor::new();
2847 conductor
2848 .maybe_run(
2849 &cfg,
2850 dir.path(),
2851 &queue,
2852 &questions,
2853 dir.path(),
2854 &[t.clone()],
2855 &[],
2856 &[],
2857 2,
2858 )
2859 .await;
2860 assert!(turn(1).is_file());
2861
2862 conductor
2863 .maybe_run(
2864 &cfg,
2865 dir.path(),
2866 &queue,
2867 &questions,
2868 dir.path(),
2869 &[],
2870 &[t.clone()],
2871 &[],
2872 2,
2873 )
2874 .await;
2875 assert!(
2876 turn(2).is_file(),
2877 "a task turning stalled must call the conductor again"
2878 );
2879
2880 conductor
2883 .maybe_run(
2884 &cfg,
2885 dir.path(),
2886 &queue,
2887 &questions,
2888 dir.path(),
2889 &[],
2890 &[t.clone()],
2891 &[],
2892 2,
2893 )
2894 .await;
2895 assert!(
2896 !turn(3).is_file(),
2897 "the same stalled task lingering must not call the conductor every cycle"
2898 );
2899 }
2900
2901 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
2902 apply(
2903 queue,
2904 questions,
2905 &Verdict {
2906 decisions: vec![Decision {
2907 id: id.to_owned(),
2908 reason: Some(note.to_owned()),
2909 recovery: Some(Recovery::Hold),
2910 ..Decision::default()
2911 }],
2912 },
2913 )
2914 .unwrap();
2915 }
2916
2917 #[test]
2918 fn reholding_with_an_identical_note_changes_nothing() {
2919 let dir = tempdir().unwrap();
2920 let queue = Queue::at(dir.path().join("queue"));
2921 let questions = Questions::at(dir.path().join("questions"));
2922 let mut t = task("held");
2923 t.hold_machine(Some("waiting on a human".to_owned()));
2924 queue.put(&mut t).unwrap();
2925
2926 rehold(&queue, &questions, &t.id, "owner said keep it held");
2927 let first = queue.get(&t.id).unwrap();
2928 let rev = queue.revision();
2929 for _ in 0..5 {
2930 rehold(&queue, &questions, &t.id, "owner said keep it held");
2931 }
2932 let after = queue.get(&t.id).unwrap();
2933 assert_eq!(after.hold_reason, first.hold_reason);
2934 assert_eq!(after.updated_at, first.updated_at);
2935 assert_eq!(queue.revision(), rev, "nothing was written");
2936 }
2937
2938 #[test]
2939 fn reholding_with_different_notes_keeps_one_previously_level() {
2940 let dir = tempdir().unwrap();
2941 let queue = Queue::at(dir.path().join("queue"));
2942 let questions = Questions::at(dir.path().join("questions"));
2943 let mut t = task("held");
2944 t.hold_machine(Some("original".to_owned()));
2945 queue.put(&mut t).unwrap();
2946
2947 for i in 0..20 {
2948 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
2949 }
2950 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
2951 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
2952 assert!(reason.starts_with("note "), "the new note leads: {reason}");
2953 }
2954
2955 #[test]
2956 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
2957 let mut t = task("deep");
2958 let nested = "same\n\n(previously: same\n\n(previously: same))";
2959 t.hold_machine(Some(nested.to_owned()));
2960 let d = Decision {
2961 id: t.id.clone(),
2962 reason: Some("same".to_owned()),
2963 ..Decision::default()
2964 };
2965 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
2966 let d2 = Decision {
2967 reason: Some("other".to_owned()),
2968 ..d
2969 };
2970 assert_eq!(
2971 reaffirmed_hold_reason(&t, &d2).as_deref(),
2972 Some("other\n\n(previously: same)")
2973 );
2974 }
2975
2976 #[tokio::test]
2977 async fn a_repeated_hold_decision_settles_instead_of_looping() {
2978 let dir = tempdir().unwrap();
2979 let queue = Queue::at(dir.path().join("queue"));
2980 let questions = Questions::at(dir.path().join("questions"));
2981 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
2982 let mut t = task("held");
2983 t.hold_machine(Some("first".to_owned()));
2984 queue.put(&mut t).unwrap();
2985 let cfg = config(mock_agent(
2986 dir.path(),
2987 REPLY,
2988 env(&reply.replace("ID", &t.id)),
2989 ));
2990 let mut conductor = Conductor::new();
2991 for _ in 0..2 {
2993 let held = queue.get(&t.id).unwrap();
2994 conductor
2995 .maybe_run(
2996 &cfg,
2997 dir.path(),
2998 &queue,
2999 &questions,
3000 dir.path(),
3001 &[],
3002 &[],
3003 &[held],
3004 2,
3005 )
3006 .await;
3007 }
3008 let held = queue.get(&t.id).unwrap();
3009 assert!(
3010 held.hold_reason
3011 .as_deref()
3012 .unwrap()
3013 .starts_with("keep held")
3014 );
3015 assert!(
3016 !conductor.worth_a_look(&queue, &[], &[held]),
3017 "an unchanged held task must stop being reconsidered"
3018 );
3019 }
3020
3021 #[tokio::test]
3022 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3023 let dir = tempdir().unwrap();
3024 let queue = Queue::at(dir.path().join("queue"));
3025 let questions = Questions::at(dir.path().join("questions"));
3026 let mut t = task("held");
3027 t.hold_machine(Some("first".to_owned()));
3028 queue.put(&mut t).unwrap();
3029 let reply = |note: &str| {
3030 format!(
3031 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3032 t.id
3033 )
3034 };
3035 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3036 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3037 let mut conductor = Conductor::new();
3038 let mut writes = 0;
3039 for i in 0..6 {
3040 let held = queue.get(&t.id).unwrap();
3041 let before = queue.revision();
3042 conductor
3043 .maybe_run(
3044 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3045 dir.path(),
3046 &queue,
3047 &questions,
3048 dir.path(),
3049 &[],
3050 &[],
3051 &[held],
3052 2,
3053 )
3054 .await;
3055 if queue.revision() != before {
3056 writes += 1;
3057 }
3058 }
3059 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3060 let mut held = queue.get(&t.id).unwrap();
3061 held.instruction.push_str(" (edited)");
3062 queue.put(&mut held).unwrap();
3063 let held = queue.get(&t.id).unwrap();
3064 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3065 }
3066
3067 #[test]
3068 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3069 let dir = tempdir().unwrap();
3070 let queue = Queue::at(dir.path().join("queue"));
3071 let mut t = task("t");
3072 queue.put(&mut t).unwrap();
3073
3074 let mut conductor = Conductor::new();
3075 assert!(
3076 conductor.worth_a_look(&queue, &[], &[]),
3077 "a conductor that has never run has something to look at"
3078 );
3079
3080 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3081 assert!(
3082 !conductor.worth_a_look(&queue, &[], &[]),
3083 "nothing changed and nothing is stalled or finished"
3084 );
3085 assert!(
3086 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3087 "a stalled task is worth a look even at the same revision"
3088 );
3089 assert!(
3090 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3091 "a finished task is worth a look even at the same revision"
3092 );
3093 }
3094
3095 #[tokio::test]
3096 async fn the_conduct_path_never_calls_ask_and_wait() {
3097 let dir = tempdir().unwrap();
3103 let queue = Queue::at(dir.path().join("queue"));
3104 let questions = Questions::at(dir.path().join("questions"));
3105 let mut t = task("asks without blocking");
3106 queue.put(&mut t).unwrap();
3107
3108 apply(
3109 &queue,
3110 &questions,
3111 &Verdict {
3112 decisions: vec![Decision {
3113 id: t.id.clone(),
3114 question: Some("ok?".to_owned()),
3115 ..Decision::default()
3116 }],
3117 },
3118 )
3119 .unwrap();
3120 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3122 }
3123
3124 #[test]
3125 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3126 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3127 t.resume_override = Some(crate::queue::OperatorResume {
3128 question_id: "q".into(),
3129 at: jiff::Timestamp::now(),
3130 conductor_rehold: None,
3131 forced: false,
3132 pinned_run: Some("run-1".into()),
3133 });
3134 assert!(pinned_resume(&t));
3135 assert!(!may_hold(&mut t, "waiting for magi resume"));
3136 assert!(
3137 t.resume_override
3138 .as_ref()
3139 .unwrap()
3140 .conductor_rehold
3141 .is_none(),
3142 "a refused hold is not recorded as an override"
3143 );
3144 t.resume_override = None;
3145 assert!(!pinned_resume(&t));
3146 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
3147 }
3148
3149 #[test]
3150 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
3151 let dir = tempdir().unwrap();
3152 let queue = Queue::at(dir.path().join("queue"));
3153 let questions = Questions::at(dir.path().join("questions"));
3154 let mut t = task("resume me");
3155 t.resume_override = Some(crate::queue::OperatorResume {
3156 question_id: "q".into(),
3157 at: jiff::Timestamp::now(),
3158 conductor_rehold: None,
3159 forced: true,
3160 pinned_run: Some("run-1".into()),
3161 });
3162 queue.put(&mut t).unwrap();
3163
3164 apply(
3165 &queue,
3166 &questions,
3167 &Verdict {
3168 decisions: vec![
3169 Decision {
3170 id: t.id.clone(),
3171 question: Some("really?".to_owned()),
3172 ..Decision::default()
3173 },
3174 Decision {
3175 id: t.id.clone(),
3176 blocked_by: vec!["other".to_owned()],
3177 ..Decision::default()
3178 },
3179 ],
3180 },
3181 )
3182 .unwrap();
3183
3184 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3185 assert!(questions.list().is_empty());
3186 }
3187}