1use std::collections::BTreeSet;
42use std::path::{Path, PathBuf};
43use std::time::Duration;
44
45use anyhow::{Context as _, Result, bail};
46use serde::Deserialize;
47
48use crate::agent::{self, Invocation, SeatState};
49use crate::ask::{Question, Questions};
50use crate::config::Config;
51use crate::prompt;
52use crate::queue::{Queue, Task, TaskStatus};
53use crate::run::RunState;
54use crate::verdict;
55
56const SEAT: &str = "conduct";
59
60pub const NODE: &str = "conduct";
64
65const TURN_TIMEOUT: Duration = Duration::from_secs(300);
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
74#[serde(rename_all = "lowercase")]
75pub enum Recovery {
76 Requeue,
79 Hold,
82 Review,
87}
88
89#[derive(Debug, Clone, Default, Deserialize)]
92pub struct Decision {
93 pub id: String,
95 #[serde(default)]
98 pub blocked_by: Vec<String>,
99 #[serde(default)]
101 pub reason: Option<String>,
102 #[serde(default)]
104 pub recovery: Option<Recovery>,
105 #[serde(default)]
109 pub question: Option<String>,
110 #[serde(default)]
112 pub choices: Vec<String>,
113}
114
115#[derive(Debug, Clone, Default, Deserialize)]
128pub struct Verdict {
129 pub decisions: Vec<Decision>,
132}
133
134fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
137 prompt::ConductTask {
138 id: t.id.clone(),
139 title: t.title.clone(),
140 instruction: t.instruction.clone(),
141 repo: t.repo.display().to_string(),
142 priority: t.priority,
143 status: t.status.as_str().to_owned(),
144 attempts: t.attempts,
145 max_attempts,
146 last_error: t.last_error.clone(),
147 hold_reason: t.hold_reason.clone(),
148 hold_source: t.hold_source.map(|source| source.label().to_owned()),
149 blocked_by: t.blocked_by.clone(),
150 answers: t
151 .answers
152 .iter()
153 .map(|a| prompt::ConductAnswer {
154 question: a.question.clone(),
155 answer: a.answer.clone(),
156 })
157 .collect(),
158 }
159}
160
161fn severity_str(s: crate::verdict::Severity) -> &'static str {
164 match s {
165 crate::verdict::Severity::Nit => "nit",
166 crate::verdict::Severity::Minor => "minor",
167 crate::verdict::Severity::Major => "major",
168 crate::verdict::Severity::Blocker => "blocker",
169 }
170}
171
172fn surviving_branch(task: &Task) -> Option<String> {
177 let last = task.runs.last()?;
178 let state = RunState::load(last).ok()?;
179 state.winner().map(|c| c.branch.clone())
180}
181
182async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
184 let Some(run_id) = task.runs.last().cloned() else {
185 return prompt::ConductOutcome {
186 run_id: "(none)".to_owned(),
187 unreadable: Some("this task has not produced a run yet".to_owned()),
188 run_status: None,
189 open_findings: Vec::new(),
190 rounds_used: 0,
191 rounds_max: 0,
192 rounds: Vec::new(),
193 branch: None,
194 branch_head: None,
195 };
196 };
197 let state = match RunState::load(&run_id) {
198 Ok(s) => s,
199 Err(e) => {
200 tracing::warn!(
205 "conductor: could not read run {run_id} for task {}: {e:#}",
206 task.short()
207 );
208 return prompt::ConductOutcome {
209 run_id,
210 unreadable: Some(format!("{e:#}")),
211 run_status: None,
212 open_findings: Vec::new(),
213 rounds_used: 0,
214 rounds_max: 0,
215 rounds: Vec::new(),
216 branch: None,
217 branch_head: None,
218 };
219 }
220 };
221
222 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
223 id: f.id.clone(),
224 title: f.title.clone(),
225 severity: severity_str(f.severity).to_owned(),
226 };
227 let open_findings = state
228 .open_findings()
229 .into_iter()
230 .map(finding_view)
231 .collect();
232 let rounds = state
233 .reviews
234 .iter()
235 .map(|r| prompt::ConductRound {
236 round: r.round,
237 findings: r
238 .reviews
239 .iter()
240 .flat_map(|rec| rec.findings.iter())
241 .map(finding_view)
242 .collect(),
243 addressed: r
244 .fix
245 .as_ref()
246 .map(|fx| fx.addressed.clone())
247 .unwrap_or_default(),
248 rejected: r
249 .fix
250 .as_ref()
251 .map(|fx| {
252 fx.rejected
253 .iter()
254 .map(|rej| prompt::ConductRejection {
255 id: rej.id.clone(),
256 why: rej.why.clone(),
257 })
258 .collect()
259 })
260 .unwrap_or_default(),
261 })
262 .collect();
263 let branch = state.winner().map(|c| c.branch.clone());
264 let branch_head = match &branch {
265 Some(b) => crate::git::rev_parse(repo, b)
266 .await
267 .ok()
268 .map(|h| h.chars().take(8).collect()),
269 None => None,
270 };
271
272 prompt::ConductOutcome {
273 run_id,
274 unreadable: None,
275 run_status: Some(state.status.as_str().to_owned()),
276 open_findings,
277 rounds_used: state.reviews.len(),
278 rounds_max: state.config.graph.review_rounds,
279 rounds,
280 branch,
281 branch_head,
282 }
283}
284
285async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
287 prompt::ConductFinished {
288 task: view(t, max_attempts),
289 outcome: outcome_for(t, &repo_for(t, repo)).await,
290 }
291}
292
293fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
296 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
297 fallback.to_path_buf()
298 } else {
299 task.repo.clone()
300 }
301}
302
303fn apply_one(queue: &Queue, questions: &Questions, d: &Decision) -> Result<()> {
320 let _claim = queue
321 .claim(&d.id)
322 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
323 let mut task = queue.get(&d.id).context("no such task")?;
324
325 if task.operator_held() {
329 return Ok(());
330 }
331
332 if let Some(text) = &d.question {
333 if task.status == TaskStatus::Done {
334 return Ok(());
335 }
336 let question_id = match questions
340 .list()
341 .into_iter()
342 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
343 {
344 Some(existing) => existing.id,
345 None => {
346 let mut q = Question::new(
347 task.id.clone(),
348 NODE.to_owned(),
349 SEAT.to_owned(),
350 text.clone(),
351 d.reason.clone().unwrap_or_default(),
352 d.choices.clone(),
353 );
354 questions.put(&mut q)?;
355 q.id
356 }
357 };
358 task.block(vec![question_id], d.reason.clone());
359 return queue.put(&mut task);
360 }
361
362 match task.status {
363 TaskStatus::Queued if !d.blocked_by.is_empty() => {
364 task.block(d.blocked_by.clone(), d.reason.clone());
365 queue.put(&mut task)?;
366 }
367 TaskStatus::Running => match d.recovery {
368 Some(Recovery::Requeue) => {
369 task.requeue();
370 queue.put(&mut task)?;
371 }
372 Some(Recovery::Hold) => {
373 task.hold_machine(d.reason.clone());
374 queue.put(&mut task)?;
375 }
376 _ => {}
380 },
381 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
382 Some(Recovery::Requeue) => {
383 task.requeue();
384 queue.put(&mut task)?;
385 }
386 Some(Recovery::Hold) => {
387 task.hold_machine(d.reason.clone());
388 queue.put(&mut task)?;
389 }
390 Some(Recovery::Review) => {
391 if let Some(branch) = surviving_branch(&task) {
392 task.request_review(branch);
393 queue.put(&mut task)?;
394 }
395 }
401 None => {}
402 },
403 _ => {}
406 }
407 Ok(())
408}
409
410pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
414 for d in &verdict.decisions {
415 if let Err(e) = apply_one(queue, questions, d) {
416 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
417 }
418 }
419 Ok(())
420}
421
422#[derive(Debug, Default)]
425pub struct Conductor {
426 seat: Option<SeatState>,
427 last_seen: Option<(u64, BTreeSet<String>)>,
428}
429
430impl Conductor {
431 #[must_use]
433 pub fn new() -> Self {
434 Self::default()
435 }
436
437 fn snapshot(queue: &Queue, stalled: &[Task], finished: &[Task]) -> (u64, BTreeSet<String>) {
438 let ids = stalled
439 .iter()
440 .chain(finished)
441 .map(|t| t.id.clone())
442 .collect();
443 (queue.revision(), ids)
444 }
445
446 #[must_use]
463 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
464 self.last_seen.as_ref() != Some(&Self::snapshot(queue, stalled, finished))
465 }
466
467 #[allow(clippy::too_many_arguments)]
471 pub async fn maybe_run(
472 &mut self,
473 cfg: &Config,
474 repo: &Path,
475 queue: &Queue,
476 questions: &Questions,
477 home: &Path,
478 queued: &[Task],
479 stalled: &[Task],
480 finished: &[Task],
481 max_attempts: usize,
482 ) {
483 let snapshot = Self::snapshot(queue, stalled, finished);
484 if self.last_seen.as_ref() == Some(&snapshot) {
485 return;
486 }
487 self.last_seen = Some(snapshot);
488 if let Err(e) = self
489 .run_once(
490 cfg,
491 repo,
492 queue,
493 questions,
494 home,
495 queued,
496 stalled,
497 finished,
498 max_attempts,
499 )
500 .await
501 {
502 tracing::warn!("conductor: {e:#}");
503 }
504 }
505
506 #[allow(clippy::too_many_arguments)]
507 async fn run_once(
508 &mut self,
509 cfg: &Config,
510 repo: &Path,
511 queue: &Queue,
512 questions: &Questions,
513 home: &Path,
514 queued: &[Task],
515 stalled: &[Task],
516 finished: &[Task],
517 max_attempts: usize,
518 ) -> Result<()> {
519 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
520 return Ok(());
521 }
522
523 let spec = cfg
524 .resolve_roles()
525 .context("resolving the conductor seat")?
526 .conductor;
527 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
528 if needs_new_seat {
529 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
530 }
531 let seat = self.seat.as_mut().expect("just ensured a seat exists");
532
533 let runnable_views: Vec<prompt::ConductTask> =
534 queued.iter().map(|t| view(t, max_attempts)).collect();
535 let stalled_views: Vec<prompt::ConductTask> =
536 stalled.iter().map(|t| view(t, max_attempts)).collect();
537 let mut finished_views = Vec::with_capacity(finished.len());
538 for t in finished {
539 finished_views.push(finished_view(t, repo, max_attempts).await);
540 }
541
542 let body = prompt::with_overlay(
543 prompt::conduct(
544 &runnable_views,
545 &stalled_views,
546 &finished_views,
547 &cfg.graph.language,
548 ),
549 cfg.prompts.overlay(NODE),
550 );
551
552 let artifacts = home.join("conduct").join("artifacts");
553 let stem = format!("turn-{}", seat.turns + 1);
554 let cache_dir = cfg.cache_dir();
557 let inv = Invocation {
558 cwd: repo,
559 prompt: &body,
560 timeout: TURN_TIMEOUT,
561 allow_write: false,
564 sessions: cfg.graph.sessions,
565 artifacts: &artifacts,
566 stem: &stem,
567 run: NODE,
568 node: NODE,
569 cache_dir: cache_dir.as_deref(),
570 attachments: &[],
571 };
572
573 let out = agent::invoke(&spec, seat, &inv)
574 .await
575 .context("invoking the conductor")?;
576 if !out.usable() {
577 bail!(
578 "no usable reply (exit {:?}, timed out {})",
579 out.exit_code,
580 out.timed_out
581 );
582 }
583 let verdict: Verdict = verdict::extract_json(&out.text)
584 .context("the conductor's reply could not be parsed")?;
585 apply(queue, questions, &verdict)
586 }
587}
588
589#[cfg(test)]
590mod tests {
591 use std::collections::BTreeMap;
592
593 use tempfile::tempdir;
594
595 use super::*;
596 use crate::ask::{Answer, QuestionStatus};
597 use crate::config::{AgentKind, AgentSpec, Graph};
598 use crate::queue::Source;
599
600 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
601 let path = dir.join("mock-conduct-agent.sh");
602 std::fs::write(&path, script).expect("write mock");
603 AgentSpec {
604 id: "mock".to_owned(),
605 kind: AgentKind::Command,
606 model: None,
607 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
608 extra_args: Vec::new(),
609 env,
610 prompt_delivery: None,
611 }
612 }
613
614 fn config(spec: AgentSpec) -> Config {
615 Config {
616 agents: vec![spec],
617 graph: Graph {
618 language: "en".to_owned(),
619 ..Graph::default()
620 },
621 ..Config::default()
622 }
623 }
624
625 fn task(title: &str) -> Task {
626 Task::new(
627 title.to_owned(),
628 format!("do {title}"),
629 std::path::PathBuf::from("."),
630 Source::Human,
631 )
632 }
633
634 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
635 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
636
637 fn env(reply: &str) -> BTreeMap<String, String> {
638 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
639 }
640
641 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
642
643 fn init_repo_with_branch(dir: &Path, branch: &str) {
647 use crate::proc::Quiet as _;
648 let run = |args: &[&str]| {
649 let out = std::process::Command::new("git")
650 .args(args)
651 .current_dir(dir)
652 .quiet()
653 .output()
654 .expect("spawn git");
655 assert!(
656 out.status.success(),
657 "git {args:?} failed: {}",
658 String::from_utf8_lossy(&out.stderr)
659 );
660 };
661 run(&["init", "-b", "main"]);
662 run(&["config", "user.name", "magi test"]);
663 run(&["config", "user.email", "magi@example.com"]);
664 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
665 run(&["add", "-A"]);
666 run(&["commit", "-m", "init"]);
667 run(&["checkout", "-b", branch]);
668 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
669 run(&["add", "-A"]);
670 run(&["commit", "-m", "candidate work"]);
671 }
672
673 fn review_round_with_finding(
674 round: usize,
675 finding_id: &str,
676 title: &str,
677 addressed: &[&str],
678 rejected: &[(&str, &str)],
679 ) -> crate::run::ReviewRound {
680 crate::run::ReviewRound {
681 round,
682 head: "deadbeef".to_owned(),
683 verified_head: None,
684 reviews: vec![crate::run::ReviewRecord {
685 reviewer: 1,
686 agent: "mock".to_owned(),
687 summary: String::new(),
688 findings: vec![crate::verdict::Finding {
689 id: finding_id.to_owned(),
690 severity: crate::verdict::Severity::Major,
691 file: None,
692 line: None,
693 title: title.to_owned(),
694 detail: String::new(),
695 }],
696 vote: None,
697 failed: None,
698 duration_ms: 0,
699 }],
700 e2e: Vec::new(),
701 verify_retried: false,
702 e2e_deferred: false,
703 e2e_defer_reason: None,
704 fix: Some(crate::run::FixRecord {
705 agent: "mock".to_owned(),
706 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
707 rejected: rejected
708 .iter()
709 .map(|(id, why)| crate::verdict::Rejection {
710 id: (*id).to_owned(),
711 why: (*why).to_owned(),
712 })
713 .collect(),
714 notes: String::new(),
715 committed: false,
716 failed: None,
717 duration_ms: 0,
718 }),
719 blocking: 1,
720 answered: 1,
721 expected: 1,
722 clean: false,
723 progressed: true,
724 vote_split: false,
725 reconsideration: Vec::new(),
726 verdict: None,
727 }
728 }
729
730 #[test]
731 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
732 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
733 let dir = tempdir().unwrap();
734 let default_repo = dir.path().join("default");
735 let task_repo = dir.path().join("task");
736 std::fs::create_dir_all(&default_repo).unwrap();
737 std::fs::create_dir_all(&task_repo).unwrap();
738 init_repo_with_branch(&default_repo, "other-branch");
739 init_repo_with_branch(&task_repo, "magi/f00d/A");
740
741 let mut config = Config::default();
742 config.graph.review_rounds = 6;
743 let mut state = crate::run::RunState::new(
744 task_repo.clone(),
745 "main".to_owned(),
746 "deadbeef".to_owned(),
747 "task".to_owned(),
748 config,
749 );
750 state.status = crate::run::RunStatus::Blocked;
751 state.candidates.push(crate::run::Candidate {
752 index: 0,
753 label: 'A',
754 agent: "mock".to_owned(),
755 branch: "magi/f00d/A".to_owned(),
756 worktree: task_repo.clone(),
757 summary: String::new(),
758 stat: String::new(),
759 files: 1,
760 commits: 1,
761 empty: false,
762 failed: None,
763 duration_ms: 0,
764 folded: false,
765 });
766 state.tally = Some(crate::run::Tally {
767 first_choice: std::collections::BTreeMap::new(),
768 borda: std::collections::BTreeMap::new(),
769 winner: 'A',
770 rankings: 0,
771 unanimous_initial: false,
772 deliberated: false,
773 changed_votes: 0,
774 unanimous_final: false,
775 tie_break: None,
776 judges: 0,
777 present: 0,
778 quorum: 0,
779 met_quorum: true,
780 uncontested: Some("solo".to_owned()),
781 });
782 state.reviews = vec![
783 review_round_with_finding(
784 1,
785 "R1-1-2",
786 "answer content is dropped",
787 &[],
788 &[("R1-1-2", "the id leaving blocked_by is enough")],
789 ),
790 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
791 ];
792 state.save().unwrap();
793
794 let mut t = task("outcome test");
795 t.repo = task_repo;
796 t.runs.push(state.id.clone());
797
798 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
799 let outcome = finished.outcome;
800
801 assert!(outcome.unreadable.is_none());
802 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
803 assert_eq!(outcome.rounds_used, 2);
804 assert_eq!(outcome.rounds_max, 6);
805 assert_eq!(outcome.rounds.len(), 2);
806 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
807 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
808 assert!(outcome.rounds[1].addressed.is_empty());
809 assert!(outcome.rounds[1].rejected.is_empty());
810 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
811 assert!(
812 outcome.branch_head.is_some(),
813 "a real branch must resolve a head commit: {outcome:?}"
814 );
815 }
816
817 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
821 tokio::runtime::Builder::new_current_thread()
822 .enable_all()
823 .build()
824 .unwrap()
825 .block_on(f)
826 }
827
828 #[test]
829 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
830 let mut t = task("answered");
831 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
832 let v = view(&t, 2);
833 assert_eq!(v.answers.len(), 1);
834 assert_eq!(v.answers[0].question, "Which backend?");
835 assert_eq!(v.answers[0].answer, "SQLite");
836 }
837
838 #[test]
839 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
840 let dir = tempdir().unwrap();
841 let queue = Queue::at(dir.path().join("queue"));
842 let questions = Questions::at(dir.path().join("questions"));
843 let mut a = task("a");
844 a.priority = 9;
845 queue.put(&mut a).unwrap();
846
847 let verdict = Verdict {
848 decisions: vec![Decision {
849 id: a.id.clone(),
850 blocked_by: vec!["20260101-000000-dead".to_owned()],
851 reason: Some("waits on the other task".to_owned()),
852 recovery: None,
853 question: None,
854 choices: Vec::new(),
855 }],
856 };
857 apply(&queue, &questions, &verdict).unwrap();
858
859 let back = queue.get(&a.id).unwrap();
860 assert_eq!(back.status, TaskStatus::Blocked);
861 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
862 assert_eq!(
863 back.priority, 9,
864 "the conductor's reply cannot carry priority"
865 );
866 }
867
868 #[test]
869 fn a_question_decision_files_one_and_blocks_on_its_id() {
870 let dir = tempdir().unwrap();
871 let queue = Queue::at(dir.path().join("queue"));
872 let questions = Questions::at(dir.path().join("questions"));
873 let mut t = task("ambiguous");
874 queue.put(&mut t).unwrap();
875
876 let verdict = Verdict {
877 decisions: vec![Decision {
878 id: t.id.clone(),
879 blocked_by: Vec::new(),
880 reason: Some("which backend?".to_owned()),
881 recovery: None,
882 question: Some("Which storage backend?".to_owned()),
883 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
884 }],
885 };
886 apply(&queue, &questions, &verdict).unwrap();
887
888 let back = queue.get(&t.id).unwrap();
889 assert_eq!(back.status, TaskStatus::Blocked);
890 assert_eq!(back.blocked_by.len(), 1);
891 let q = questions.get(&back.blocked_by[0]).unwrap();
892 assert_eq!(q.summary, "Which storage backend?");
893 assert_eq!(q.node, NODE);
894 assert!(q.status.open());
895 }
896
897 #[test]
898 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
899 let dir = tempdir().unwrap();
900 let queue = Queue::at(dir.path().join("queue"));
901 let questions = Questions::at(dir.path().join("questions"));
902 let mut t = task("asked once");
903 queue.put(&mut t).unwrap();
904
905 let decision = Decision {
906 id: t.id.clone(),
907 reason: Some("still deciding".to_owned()),
908 question: Some("Which backend?".to_owned()),
909 ..Decision::default()
910 };
911 apply(
912 &queue,
913 &questions,
914 &Verdict {
915 decisions: vec![decision.clone()],
916 },
917 )
918 .unwrap();
919 assert_eq!(questions.list().len(), 1);
920 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
921
922 let mut released = queue.get(&t.id).unwrap();
927 released.release();
928 queue.put(&mut released).unwrap();
929
930 apply(
931 &queue,
932 &questions,
933 &Verdict {
934 decisions: vec![decision],
935 },
936 )
937 .unwrap();
938 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
939 let after = queue.get(&t.id).unwrap();
940 assert_eq!(
941 after.blocked_by,
942 [first_question_id],
943 "the existing open question is reused, not replaced"
944 );
945 }
946
947 #[test]
948 fn a_same_id_question_from_another_node_is_not_reused() {
949 let dir = tempdir().unwrap();
950 let queue = Queue::at(dir.path().join("queue"));
951 let questions = Questions::at(dir.path().join("questions"));
952 let mut t = task("must ask the conductor");
953 queue.put(&mut t).unwrap();
954
955 let mut unrelated = Question::new(
956 t.id.clone(),
957 "review".to_owned(),
958 "reviewer-1".to_owned(),
959 "An unrelated review question".to_owned(),
960 String::new(),
961 Vec::new(),
962 );
963 questions.put(&mut unrelated).unwrap();
964
965 apply(
966 &queue,
967 &questions,
968 &Verdict {
969 decisions: vec![Decision {
970 id: t.id.clone(),
971 question: Some("Which backend?".to_owned()),
972 ..Decision::default()
973 }],
974 },
975 )
976 .unwrap();
977
978 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
979 assert_eq!(blocked_by.len(), 1);
980 assert_ne!(blocked_by[0], unrelated.id);
981 assert!(questions.get(&unrelated.id).unwrap().status.open());
982 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
983 }
984
985 #[test]
986 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
987 let dir = tempdir().unwrap();
988 let queue = Queue::at(dir.path().join("queue"));
989 let questions = Questions::at(dir.path().join("questions"));
990 let mut t = task("waits on an answer");
991 queue.put(&mut t).unwrap();
992
993 apply(
994 &queue,
995 &questions,
996 &Verdict {
997 decisions: vec![Decision {
998 id: t.id.clone(),
999 blocked_by: Vec::new(),
1000 reason: None,
1001 recovery: None,
1002 question: Some("Which backend?".to_owned()),
1003 choices: Vec::new(),
1004 }],
1005 },
1006 )
1007 .unwrap();
1008 let blocked = queue.get(&t.id).unwrap();
1009 let question_id = blocked.blocked_by[0].clone();
1010
1011 let mut q = questions.get(&question_id).unwrap();
1012 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
1013 questions.put(&mut q).unwrap();
1014 assert_eq!(q.status, QuestionStatus::Answered);
1015
1016 let mut task_after = queue.get(&t.id).unwrap();
1020 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
1021 task_after.unblock(&question_id);
1022 assert_eq!(task_after.status, TaskStatus::Queued);
1023 assert_eq!(task_after.answers[0].answer, "SQLite");
1024 }
1025
1026 #[test]
1027 fn a_stalled_task_can_be_requeued_or_held() {
1028 let dir = tempdir().unwrap();
1029 let queue = Queue::at(dir.path().join("queue"));
1030 let questions = Questions::at(dir.path().join("questions"));
1031
1032 let mut requeue_me = task("stuck a");
1033 requeue_me.start("run-1".to_owned());
1034 queue.put(&mut requeue_me).unwrap();
1035
1036 let mut hold_me = task("stuck b");
1037 hold_me.start("run-2".to_owned());
1038 queue.put(&mut hold_me).unwrap();
1039
1040 apply(
1041 &queue,
1042 &questions,
1043 &Verdict {
1044 decisions: vec![
1045 Decision {
1046 id: requeue_me.id.clone(),
1047 recovery: Some(Recovery::Requeue),
1048 ..Decision::default()
1049 },
1050 Decision {
1051 id: hold_me.id.clone(),
1052 recovery: Some(Recovery::Hold),
1053 reason: Some("looks broken".to_owned()),
1054 ..Decision::default()
1055 },
1056 ],
1057 },
1058 )
1059 .unwrap();
1060
1061 let requeued = queue.get(&requeue_me.id).unwrap();
1062 assert_eq!(requeued.status, TaskStatus::Queued);
1063 assert_eq!(requeued.attempts, 0);
1064
1065 let held = queue.get(&hold_me.id).unwrap();
1066 assert_eq!(held.status, TaskStatus::Held);
1067 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
1068 }
1069
1070 #[test]
1071 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
1072 let dir = tempdir().unwrap();
1073 let queue = Queue::at(dir.path().join("queue"));
1074 let questions = Questions::at(dir.path().join("questions"));
1075 let mut held = task("manual recovery");
1076 held.priority = 300;
1077 held.runs.push("run20260912-224242-daf5".to_owned());
1078 held.hold_manual(Some(
1079 "active manual recovery run20260912-224242-daf5".to_owned(),
1080 ));
1081 queue.put(&mut held).unwrap();
1082
1083 for decision in [
1087 Decision {
1088 id: held.id.clone(),
1089 recovery: Some(Recovery::Requeue),
1090 ..Decision::default()
1091 },
1092 Decision {
1093 id: held.id.clone(),
1094 recovery: Some(Recovery::Hold),
1095 reason: Some("stale replacement reason".to_owned()),
1096 ..Decision::default()
1097 },
1098 Decision {
1099 id: held.id.clone(),
1100 recovery: Some(Recovery::Review),
1101 ..Decision::default()
1102 },
1103 Decision {
1104 id: held.id.clone(),
1105 blocked_by: vec!["other-task".to_owned()],
1106 question: Some("retry now?".to_owned()),
1107 ..Decision::default()
1108 },
1109 ] {
1110 apply(
1111 &queue,
1112 &questions,
1113 &Verdict {
1114 decisions: vec![decision],
1115 },
1116 )
1117 .unwrap();
1118 }
1119
1120 let after = queue.get(&held.id).unwrap();
1121 assert_eq!(after.status, TaskStatus::Held);
1122 assert!(after.operator_held());
1123 assert_eq!(after.priority, 300);
1124 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
1125 assert_eq!(
1126 after.hold_reason.as_deref(),
1127 Some("active manual recovery run20260912-224242-daf5")
1128 );
1129 assert!(after.blocked_by.is_empty());
1130 assert!(questions.list().is_empty());
1131 assert!(
1132 queue.next_runnable().is_none(),
1133 "must not dispatch a duplicate"
1134 );
1135 }
1136
1137 #[test]
1138 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
1139 let dir = tempdir().unwrap();
1140 let queue = Queue::at(dir.path().join("queue"));
1141 let questions = Questions::at(dir.path().join("questions"));
1142
1143 let mut automatic = task("disk gate");
1144 automatic.hold_machine(Some("disk full".to_owned()));
1145 queue.put(&mut automatic).unwrap();
1146 let requeue = || Verdict {
1147 decisions: vec![Decision {
1148 id: automatic.id.clone(),
1149 recovery: Some(Recovery::Requeue),
1150 ..Decision::default()
1151 }],
1152 };
1153 apply(&queue, &questions, &requeue()).unwrap();
1154 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
1155
1156 let mut manual = task("operator gate");
1157 manual.hold_manual(Some("wait for operator".to_owned()));
1158 queue.put(&mut manual).unwrap();
1159 apply(
1160 &queue,
1161 &questions,
1162 &Verdict {
1163 decisions: vec![Decision {
1164 id: manual.id.clone(),
1165 recovery: Some(Recovery::Requeue),
1166 ..Decision::default()
1167 }],
1168 },
1169 )
1170 .unwrap();
1171 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
1172
1173 let mut released = queue.get(&manual.id).unwrap();
1176 released.release();
1177 queue.put(&mut released).unwrap();
1178 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
1179 }
1180
1181 #[test]
1182 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
1183 let dir = tempdir().unwrap();
1184 let queue = Queue::at(dir.path().join("queue"));
1185 let questions = Questions::at(dir.path().join("questions"));
1186 let mut legacy = task("old explicit hold");
1187 legacy.status = TaskStatus::Held;
1188 legacy.hold_reason = Some("manual recovery already active".to_owned());
1189 legacy.hold_source = None;
1190 legacy.blocked_by = vec!["dependency".to_owned()];
1191 queue.put(&mut legacy).unwrap();
1192
1193 apply(
1194 &queue,
1195 &questions,
1196 &Verdict {
1197 decisions: vec![Decision {
1198 id: legacy.id.clone(),
1199 recovery: Some(Recovery::Requeue),
1200 ..Decision::default()
1201 }],
1202 },
1203 )
1204 .unwrap();
1205
1206 let after = queue.get(&legacy.id).unwrap();
1207 assert_eq!(after.status, TaskStatus::Held);
1208 assert_eq!(after.hold_source, None);
1209 assert_eq!(after.hold_reason, legacy.hold_reason);
1210 assert_eq!(after.blocked_by, legacy.blocked_by);
1211 }
1212
1213 #[test]
1214 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
1215 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
1221 let dir = tempdir().unwrap();
1222 let queue = Queue::at(dir.path().join("queue"));
1223 let questions = Questions::at(dir.path().join("questions"));
1224 let mut t = task("blocked with no readable run");
1225 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
1227 queue.put(&mut t).unwrap();
1228
1229 apply(
1230 &queue,
1231 &questions,
1232 &Verdict {
1233 decisions: vec![Decision {
1234 id: t.id.clone(),
1235 recovery: Some(Recovery::Review),
1236 ..Decision::default()
1237 }],
1238 },
1239 )
1240 .unwrap();
1241
1242 let after = queue.get(&t.id).unwrap();
1243 assert_eq!(
1244 after.status,
1245 TaskStatus::Failed,
1246 "with nothing to reopen, the decision is dropped rather than guessed at"
1247 );
1248 assert!(after.review_branch.is_none());
1249 }
1250
1251 #[test]
1252 fn recovery_is_ignored_for_a_task_that_is_not_actually_stalled_or_finished() {
1253 let dir = tempdir().unwrap();
1254 let queue = Queue::at(dir.path().join("queue"));
1255 let questions = Questions::at(dir.path().join("questions"));
1256 let mut t = task("ordinary");
1257 queue.put(&mut t).unwrap();
1258
1259 apply(
1260 &queue,
1261 &questions,
1262 &Verdict {
1263 decisions: vec![Decision {
1264 id: t.id.clone(),
1265 recovery: Some(Recovery::Hold),
1266 ..Decision::default()
1267 }],
1268 },
1269 )
1270 .unwrap();
1271
1272 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1273 }
1274
1275 #[tokio::test]
1276 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
1277 let dir = tempdir().unwrap();
1278 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
1279 let queue = Queue::at(dir.path().join("queue"));
1280 let questions = Questions::at(dir.path().join("questions"));
1281 let mut t = task("normal");
1282 queue.put(&mut t).unwrap();
1283
1284 let mut conductor = Conductor::new();
1285 conductor
1286 .maybe_run(
1287 &cfg,
1288 dir.path(),
1289 &queue,
1290 &questions,
1291 dir.path(),
1292 &[t.clone()],
1293 &[],
1294 &[],
1295 2,
1296 )
1297 .await;
1298
1299 assert_eq!(
1300 queue.get(&t.id).unwrap().status,
1301 TaskStatus::Queued,
1302 "a failed invocation must change nothing"
1303 );
1304 assert!(
1305 queue.next_runnable().is_some(),
1306 "the loop must still be able to take the next task"
1307 );
1308 }
1309
1310 #[tokio::test]
1311 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
1312 let dir = tempdir().unwrap();
1313 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
1314 let queue = Queue::at(dir.path().join("queue"));
1315 let questions = Questions::at(dir.path().join("questions"));
1316 let mut t = task("normal");
1317 queue.put(&mut t).unwrap();
1318
1319 let mut conductor = Conductor::new();
1320 conductor
1321 .maybe_run(
1322 &cfg,
1323 dir.path(),
1324 &queue,
1325 &questions,
1326 dir.path(),
1327 &[t.clone()],
1328 &[],
1329 &[],
1330 2,
1331 )
1332 .await;
1333
1334 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1335 }
1336
1337 #[tokio::test]
1338 async fn json_survives_code_fences_and_a_preamble() {
1339 let dir = tempdir().unwrap();
1340 let mut t = task("fenced");
1341 let reply = format!(
1342 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
1343 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
1344 t.id
1345 );
1346 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1347 let queue = Queue::at(dir.path().join("queue"));
1348 let questions = Questions::at(dir.path().join("questions"));
1349 queue.put(&mut t).unwrap();
1350
1351 let mut conductor = Conductor::new();
1352 conductor
1353 .maybe_run(
1354 &cfg,
1355 dir.path(),
1356 &queue,
1357 &questions,
1358 dir.path(),
1359 &[t.clone()],
1360 &[],
1361 &[],
1362 2,
1363 )
1364 .await;
1365
1366 let back = queue.get(&t.id).unwrap();
1367 assert_eq!(back.status, TaskStatus::Blocked);
1368 assert_eq!(back.blocked_by, ["x"]);
1369 }
1370
1371 #[tokio::test]
1372 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
1373 let dir = tempdir().unwrap();
1376 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
1377 let queue = Queue::at(dir.path().join("queue"));
1378 let questions = Questions::at(dir.path().join("questions"));
1379 let mut t = task("stable");
1380 queue.put(&mut t).unwrap();
1381 let artifacts = dir.path().join("conduct").join("artifacts");
1382 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
1383
1384 let mut conductor = Conductor::new();
1385 conductor
1386 .maybe_run(
1387 &cfg,
1388 dir.path(),
1389 &queue,
1390 &questions,
1391 dir.path(),
1392 &[t.clone()],
1393 &[],
1394 &[],
1395 2,
1396 )
1397 .await;
1398 assert!(turn(1).is_file(), "the first cycle must call the conductor");
1399
1400 conductor
1401 .maybe_run(
1402 &cfg,
1403 dir.path(),
1404 &queue,
1405 &questions,
1406 dir.path(),
1407 &[t.clone()],
1408 &[],
1409 &[],
1410 2,
1411 )
1412 .await;
1413 assert!(
1414 !turn(2).is_file(),
1415 "an unchanged revision and an unchanged stalled/finished set must not call the \
1416 conductor twice"
1417 );
1418
1419 t.priority = 1;
1421 queue.put(&mut t).unwrap();
1422 conductor
1423 .maybe_run(
1424 &cfg,
1425 dir.path(),
1426 &queue,
1427 &questions,
1428 dir.path(),
1429 &[t.clone()],
1430 &[],
1431 &[],
1432 2,
1433 )
1434 .await;
1435 assert!(turn(2).is_file(), "a moved revision calls it again");
1436 }
1437
1438 #[tokio::test]
1439 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
1440 let dir = tempdir().unwrap();
1446 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
1447 let queue = Queue::at(dir.path().join("queue"));
1448 let questions = Questions::at(dir.path().join("questions"));
1449 let mut t = task("quiet");
1450 queue.put(&mut t).unwrap();
1451 let artifacts = dir.path().join("conduct").join("artifacts");
1452 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
1453
1454 let mut conductor = Conductor::new();
1455 conductor
1456 .maybe_run(
1457 &cfg,
1458 dir.path(),
1459 &queue,
1460 &questions,
1461 dir.path(),
1462 &[t.clone()],
1463 &[],
1464 &[],
1465 2,
1466 )
1467 .await;
1468 assert!(turn(1).is_file());
1469
1470 conductor
1471 .maybe_run(
1472 &cfg,
1473 dir.path(),
1474 &queue,
1475 &questions,
1476 dir.path(),
1477 &[],
1478 &[t.clone()],
1479 &[],
1480 2,
1481 )
1482 .await;
1483 assert!(
1484 turn(2).is_file(),
1485 "a task turning stalled must call the conductor again"
1486 );
1487
1488 conductor
1491 .maybe_run(
1492 &cfg,
1493 dir.path(),
1494 &queue,
1495 &questions,
1496 dir.path(),
1497 &[],
1498 &[t.clone()],
1499 &[],
1500 2,
1501 )
1502 .await;
1503 assert!(
1504 !turn(3).is_file(),
1505 "the same stalled task lingering must not call the conductor every cycle"
1506 );
1507 }
1508
1509 #[test]
1510 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
1511 let dir = tempdir().unwrap();
1512 let queue = Queue::at(dir.path().join("queue"));
1513 let mut t = task("t");
1514 queue.put(&mut t).unwrap();
1515
1516 let mut conductor = Conductor::new();
1517 assert!(
1518 conductor.worth_a_look(&queue, &[], &[]),
1519 "a conductor that has never run has something to look at"
1520 );
1521
1522 conductor.last_seen = Some(Conductor::snapshot(&queue, &[], &[]));
1523 assert!(
1524 !conductor.worth_a_look(&queue, &[], &[]),
1525 "nothing changed and nothing is stalled or finished"
1526 );
1527 assert!(
1528 conductor.worth_a_look(&queue, &[t.clone()], &[]),
1529 "a stalled task is worth a look even at the same revision"
1530 );
1531 assert!(
1532 conductor.worth_a_look(&queue, &[], &[t.clone()]),
1533 "a finished task is worth a look even at the same revision"
1534 );
1535 }
1536
1537 #[tokio::test]
1538 async fn the_conduct_path_never_calls_ask_and_wait() {
1539 let dir = tempdir().unwrap();
1545 let queue = Queue::at(dir.path().join("queue"));
1546 let questions = Questions::at(dir.path().join("questions"));
1547 let mut t = task("asks without blocking");
1548 queue.put(&mut t).unwrap();
1549
1550 apply(
1551 &queue,
1552 &questions,
1553 &Verdict {
1554 decisions: vec![Decision {
1555 id: t.id.clone(),
1556 question: Some("ok?".to_owned()),
1557 ..Decision::default()
1558 }],
1559 },
1560 )
1561 .unwrap();
1562 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1564 }
1565}