1use std::collections::BTreeSet;
42use std::path::{Path, PathBuf};
43use std::time::Duration;
44
45use anyhow::{Context as _, Result, bail};
46use serde::Deserialize;
47
48use crate::agent::{self, Invocation, SeatState};
49use crate::ask::{Question, Questions};
50use crate::config::Config;
51use crate::prompt;
52use crate::queue::{Queue, Task, TaskStatus};
53use crate::run::RunState;
54use crate::verdict;
55
56const SEAT: &str = "conduct";
59
60pub const NODE: &str = "conduct";
64
65const TURN_TIMEOUT: Duration = Duration::from_secs(300);
70
71const MAX_SETTLED_CONDUCT_ANSWERS: usize = 2;
89
90#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
93#[serde(rename_all = "lowercase")]
94pub enum Recovery {
95 Requeue,
98 Hold,
101 Review,
106 Done,
114}
115
116#[derive(Debug, Clone, Default, Deserialize)]
119pub struct Decision {
120 pub id: String,
122 #[serde(default)]
125 pub blocked_by: Vec<String>,
126 #[serde(default)]
128 pub reason: Option<String>,
129 #[serde(default)]
137 pub recovery: Option<Recovery>,
138 #[serde(default)]
142 pub question: Option<String>,
143 #[serde(default)]
145 pub choices: Vec<String>,
146}
147
148#[derive(Debug, Clone, Default, Deserialize)]
161pub struct Verdict {
162 pub decisions: Vec<Decision>,
165}
166
167fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
170 prompt::ConductTask {
171 id: t.id.clone(),
172 title: t.title.clone(),
173 instruction: t.instruction.clone(),
174 repo: t.repo.display().to_string(),
175 priority: t.priority,
176 status: t.status.as_str().to_owned(),
177 attempts: t.attempts,
178 max_attempts,
179 last_error: t.last_error.clone(),
180 hold_reason: t.hold_reason.clone(),
181 hold_source: t.hold_source.map(|source| source.label().to_owned()),
182 blocked_by: t.blocked_by.clone(),
183 answers: t
184 .answers
185 .iter()
186 .map(|a| prompt::ConductAnswer {
187 question: a.question.clone(),
188 answer: a.answer.clone(),
189 })
190 .collect(),
191 operator_resume: t.resume_override.as_ref().map(|o| {
192 format!(
193 "the operator explicitly answered \"resume\" at {}; do not hold this \
194 task again for the same reason unless there is new information",
195 o.at
196 )
197 }),
198 }
199}
200
201fn may_hold(task: &mut Task, reason: &str) -> bool {
210 let Some(o) = task.resume_override.as_mut() else {
211 return true;
212 };
213 if o.forced {
214 tracing::warn!(
215 "conductor tried to hold task {} after the operator forced a resume: {reason}",
216 task.id
217 );
218 return false;
219 }
220 if o.conductor_rehold.is_some() {
221 return false;
222 }
223 o.conductor_rehold = Some(reason.to_owned());
224 true
225}
226
227fn severity_str(s: crate::verdict::Severity) -> &'static str {
230 match s {
231 crate::verdict::Severity::Nit => "nit",
232 crate::verdict::Severity::Minor => "minor",
233 crate::verdict::Severity::Major => "major",
234 crate::verdict::Severity::Blocker => "blocker",
235 }
236}
237
238fn surviving_branch(task: &Task) -> Option<String> {
243 let last = task.runs.last()?;
244 let state = RunState::load(last).ok()?;
245 state.winner().map(|c| c.branch.clone())
246}
247
248fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> String {
268 let note = match &d.reason {
269 Some(reason) => reason.clone(),
270 None => match task.answers.last() {
271 Some(a) => format!(
272 "conduct held this again with no new reason given; last operator \
273 answer on record: {}",
274 a.answer
275 ),
276 None => "conduct held this again with no reason given".to_owned(),
277 },
278 };
279 match task.hold_reason.as_deref() {
280 Some(prior) if !prior.is_empty() => format!("{note}\n\n(previously: {prior})"),
281 _ => note,
282 }
283}
284
285fn hold_note(d: &Decision) -> String {
287 d.reason
288 .clone()
289 .unwrap_or_else(|| "(no reason given)".to_owned())
290}
291
292async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
294 let Some(run_id) = task.runs.last().cloned() else {
295 return prompt::ConductOutcome {
296 run_id: "(none)".to_owned(),
297 unreadable: Some("this task has not produced a run yet".to_owned()),
298 run_status: None,
299 open_findings: Vec::new(),
300 rounds_used: 0,
301 rounds_max: 0,
302 rounds: Vec::new(),
303 branch: None,
304 branch_head: None,
305 };
306 };
307 let state = match RunState::load(&run_id) {
308 Ok(s) => s,
309 Err(e) => {
310 tracing::warn!(
315 "conductor: could not read run {run_id} for task {}: {e:#}",
316 task.short()
317 );
318 return prompt::ConductOutcome {
319 run_id,
320 unreadable: Some(format!("{e:#}")),
321 run_status: None,
322 open_findings: Vec::new(),
323 rounds_used: 0,
324 rounds_max: 0,
325 rounds: Vec::new(),
326 branch: None,
327 branch_head: None,
328 };
329 }
330 };
331
332 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
333 id: f.id.clone(),
334 title: f.title.clone(),
335 severity: severity_str(f.severity).to_owned(),
336 };
337 let open_findings = state
338 .open_findings()
339 .into_iter()
340 .map(finding_view)
341 .collect();
342 let rounds = state
343 .reviews
344 .iter()
345 .map(|r| prompt::ConductRound {
346 round: r.round,
347 findings: r
348 .reviews
349 .iter()
350 .flat_map(|rec| rec.findings.iter())
351 .map(finding_view)
352 .collect(),
353 addressed: r
354 .fix
355 .as_ref()
356 .map(|fx| fx.addressed.clone())
357 .unwrap_or_default(),
358 rejected: r
359 .fix
360 .as_ref()
361 .map(|fx| {
362 fx.rejected
363 .iter()
364 .map(|rej| prompt::ConductRejection {
365 id: rej.id.clone(),
366 why: rej.why.clone(),
367 })
368 .collect()
369 })
370 .unwrap_or_default(),
371 })
372 .collect();
373 let branch = state.winner().map(|c| c.branch.clone());
374 let branch_head = match &branch {
375 Some(b) => crate::git::rev_parse(repo, b)
376 .await
377 .ok()
378 .map(|h| h.chars().take(8).collect()),
379 None => None,
380 };
381
382 prompt::ConductOutcome {
383 run_id,
384 unreadable: None,
385 run_status: Some(state.status.as_str().to_owned()),
386 open_findings,
387 rounds_used: state.reviews.len(),
388 rounds_max: state.config.graph.review_rounds,
389 rounds,
390 branch,
391 branch_head,
392 }
393}
394
395async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
397 prompt::ConductFinished {
398 task: view(t, max_attempts),
399 outcome: outcome_for(t, &repo_for(t, repo)).await,
400 }
401}
402
403fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
406 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
407 fallback.to_path_buf()
408 } else {
409 task.repo.clone()
410 }
411}
412
413fn apply_one(queue: &Queue, questions: &Questions, d: &Decision) -> Result<()> {
430 let _claim = queue
431 .claim(&d.id)
432 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
433 let mut task = queue.get(&d.id).context("no such task")?;
434
435 if task.operator_held() {
439 return Ok(());
440 }
441
442 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
452 return Ok(());
453 }
454
455 if let Some(text) = &d.question {
456 if task.status == TaskStatus::Done {
457 return Ok(());
458 }
459 let question_id = match questions
463 .list()
464 .into_iter()
465 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
466 {
467 Some(existing) => existing.id,
468 None if task.answers.len() >= MAX_SETTLED_CONDUCT_ANSWERS => {
473 task.hold_machine(Some(format!(
474 "conduct tried to ask another question after {} were \
475 already answered about this task: {text}",
476 task.answers.len()
477 )));
478 return queue.put(&mut task);
479 }
480 None => {
481 let mut q = Question::new(
482 task.id.clone(),
483 NODE.to_owned(),
484 SEAT.to_owned(),
485 text.clone(),
486 d.reason.clone().unwrap_or_default(),
487 d.choices.clone(),
488 );
489 questions.put(&mut q)?;
490 q.id
491 }
492 };
493 task.block(vec![question_id], d.reason.clone());
494 return queue.put(&mut task);
495 }
496
497 match task.status {
498 TaskStatus::Queued if !d.blocked_by.is_empty() => {
499 task.block(d.blocked_by.clone(), d.reason.clone());
500 queue.put(&mut task)?;
501 }
502 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
506 if may_hold(&mut task, &hold_note(d)) {
507 task.hold_machine(d.reason.clone());
508 queue.put(&mut task)?;
509 }
510 }
511 TaskStatus::Running => match d.recovery {
512 Some(Recovery::Requeue) => {
513 task.requeue();
514 queue.put(&mut task)?;
515 }
516 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
517 task.hold_machine(d.reason.clone());
518 queue.put(&mut task)?;
519 }
520 _ => {}
524 },
525 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
526 Some(Recovery::Requeue) => {
527 task.requeue();
528 queue.put(&mut task)?;
529 }
530 Some(Recovery::Hold) => {
531 if may_hold(&mut task, &hold_note(d)) {
532 task.hold_machine(Some(reaffirmed_hold_reason(&task, d)));
533 queue.put(&mut task)?;
534 }
535 }
536 Some(Recovery::Review) => {
537 if let Some(branch) = surviving_branch(&task) {
538 task.request_review(branch);
539 queue.put(&mut task)?;
540 }
541 }
547 Some(Recovery::Done) => {
548 task.succeed();
549 queue.put(&mut task)?;
550 }
551 None => {}
552 },
553 _ => {}
556 }
557 Ok(())
558}
559
560pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
564 for d in &verdict.decisions {
565 if let Err(e) = apply_one(queue, questions, d) {
566 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
567 }
568 }
569 Ok(())
570}
571
572#[derive(Debug, Default)]
575pub struct Conductor {
576 seat: Option<SeatState>,
577 last_seen: Option<(u64, BTreeSet<String>)>,
578}
579
580impl Conductor {
581 #[must_use]
583 pub fn new() -> Self {
584 Self::default()
585 }
586
587 fn snapshot(queue: &Queue, stalled: &[Task], finished: &[Task]) -> (u64, BTreeSet<String>) {
588 let ids = stalled
589 .iter()
590 .chain(finished)
591 .map(|t| t.id.clone())
592 .collect();
593 (queue.revision(), ids)
594 }
595
596 #[must_use]
613 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
614 self.last_seen.as_ref() != Some(&Self::snapshot(queue, stalled, finished))
615 }
616
617 #[allow(clippy::too_many_arguments)]
621 pub async fn maybe_run(
622 &mut self,
623 cfg: &Config,
624 repo: &Path,
625 queue: &Queue,
626 questions: &Questions,
627 home: &Path,
628 queued: &[Task],
629 stalled: &[Task],
630 finished: &[Task],
631 max_attempts: usize,
632 ) {
633 let snapshot = Self::snapshot(queue, stalled, finished);
634 if self.last_seen.as_ref() == Some(&snapshot) {
635 return;
636 }
637 self.last_seen = Some(snapshot);
638 if let Err(e) = self
639 .run_once(
640 cfg,
641 repo,
642 queue,
643 questions,
644 home,
645 queued,
646 stalled,
647 finished,
648 max_attempts,
649 )
650 .await
651 {
652 tracing::warn!("conductor: {e:#}");
653 }
654 }
655
656 #[allow(clippy::too_many_arguments)]
657 async fn run_once(
658 &mut self,
659 cfg: &Config,
660 repo: &Path,
661 queue: &Queue,
662 questions: &Questions,
663 home: &Path,
664 queued: &[Task],
665 stalled: &[Task],
666 finished: &[Task],
667 max_attempts: usize,
668 ) -> Result<()> {
669 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
670 return Ok(());
671 }
672
673 let spec = cfg
674 .resolve_roles()
675 .context("resolving the conductor seat")?
676 .conductor;
677 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
678 if needs_new_seat {
679 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
680 }
681 let seat = self.seat.as_mut().expect("just ensured a seat exists");
682
683 let runnable_views: Vec<prompt::ConductTask> =
684 queued.iter().map(|t| view(t, max_attempts)).collect();
685 let stalled_views: Vec<prompt::ConductTask> =
686 stalled.iter().map(|t| view(t, max_attempts)).collect();
687 let mut finished_views = Vec::with_capacity(finished.len());
688 for t in finished {
689 finished_views.push(finished_view(t, repo, max_attempts).await);
690 }
691
692 let body = prompt::with_overlay(
693 prompt::conduct(
694 &runnable_views,
695 &stalled_views,
696 &finished_views,
697 &cfg.graph.language,
698 ),
699 cfg.prompts.overlay(NODE),
700 );
701
702 let artifacts = home.join("conduct").join("artifacts");
703 let stem = format!("turn-{}", seat.turns + 1);
704 let cache_dir = cfg.cache_dir();
707 let inv = Invocation {
708 cwd: repo,
709 prompt: &body,
710 timeout: TURN_TIMEOUT,
711 allow_write: false,
714 sessions: cfg.graph.sessions,
715 artifacts: &artifacts,
716 stem: &stem,
717 run: NODE,
718 node: NODE,
719 cache_dir: cache_dir.as_deref(),
720 attachments: &[],
721 };
722
723 let out = agent::invoke(&spec, seat, &inv)
724 .await
725 .context("invoking the conductor")?;
726 if !out.usable() {
727 bail!(
728 "no usable reply (exit {:?}, timed out {})",
729 out.exit_code,
730 out.timed_out
731 );
732 }
733 let verdict: Verdict = verdict::extract_json(&out.text)
734 .context("the conductor's reply could not be parsed")?;
735 apply(queue, questions, &verdict)
736 }
737}
738
739#[cfg(test)]
740mod tests {
741 use std::collections::BTreeMap;
742
743 use tempfile::tempdir;
744
745 use super::*;
746 use crate::ask::{Answer, QuestionStatus};
747 use crate::config::{AgentKind, AgentSpec, Graph};
748 use crate::queue::Source;
749
750 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
751 let path = dir.join("mock-conduct-agent.sh");
752 std::fs::write(&path, script).expect("write mock");
753 AgentSpec {
754 id: "mock".to_owned(),
755 kind: AgentKind::Command,
756 model: None,
757 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
758 extra_args: Vec::new(),
759 env,
760 prompt_delivery: None,
761 }
762 }
763
764 fn config(spec: AgentSpec) -> Config {
765 Config {
766 agents: vec![spec],
767 graph: Graph {
768 language: "en".to_owned(),
769 ..Graph::default()
770 },
771 ..Config::default()
772 }
773 }
774
775 fn task(title: &str) -> Task {
776 Task::new(
777 title.to_owned(),
778 format!("do {title}"),
779 std::path::PathBuf::from("."),
780 Source::Human,
781 )
782 }
783
784 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
785 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
786
787 fn env(reply: &str) -> BTreeMap<String, String> {
788 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
789 }
790
791 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
792
793 fn init_repo_with_branch(dir: &Path, branch: &str) {
797 use crate::proc::Quiet as _;
798 let run = |args: &[&str]| {
799 let out = std::process::Command::new("git")
800 .args(args)
801 .current_dir(dir)
802 .quiet()
803 .output()
804 .expect("spawn git");
805 assert!(
806 out.status.success(),
807 "git {args:?} failed: {}",
808 String::from_utf8_lossy(&out.stderr)
809 );
810 };
811 run(&["init", "-b", "main"]);
812 run(&["config", "user.name", "magi test"]);
813 run(&["config", "user.email", "magi@example.com"]);
814 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
815 run(&["add", "-A"]);
816 run(&["commit", "-m", "init"]);
817 run(&["checkout", "-b", branch]);
818 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
819 run(&["add", "-A"]);
820 run(&["commit", "-m", "candidate work"]);
821 }
822
823 fn review_round_with_finding(
824 round: usize,
825 finding_id: &str,
826 title: &str,
827 addressed: &[&str],
828 rejected: &[(&str, &str)],
829 ) -> crate::run::ReviewRound {
830 crate::run::ReviewRound {
831 round,
832 head: "deadbeef".to_owned(),
833 verified_head: None,
834 verified_at: None,
835 reviews: vec![crate::run::ReviewRecord {
836 attempts: 0,
837 reviewer: 1,
838 agent: "mock".to_owned(),
839 summary: String::new(),
840 findings: vec![crate::verdict::Finding {
841 id: finding_id.to_owned(),
842 severity: crate::verdict::Severity::Major,
843 file: None,
844 line: None,
845 title: title.to_owned(),
846 detail: String::new(),
847 }],
848 vote: None,
849 failed: None,
850 duration_ms: 0,
851 }],
852 e2e: Vec::new(),
853 verify_retried: false,
854 e2e_deferred: false,
855 e2e_defer_reason: None,
856 fix: Some(crate::run::FixRecord {
857 agent: "mock".to_owned(),
858 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
859 rejected: rejected
860 .iter()
861 .map(|(id, why)| crate::verdict::Rejection {
862 id: (*id).to_owned(),
863 why: (*why).to_owned(),
864 })
865 .collect(),
866 notes: String::new(),
867 committed: false,
868 failed: None,
869 duration_ms: 0,
870 continuation: None,
871 }),
872 blocking: 1,
873 answered: 1,
874 expected: 1,
875 clean: false,
876 progressed: true,
877 vote_split: false,
878 reconsideration: Vec::new(),
879 verdict: None,
880 }
881 }
882
883 #[test]
884 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
885 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
886 let dir = tempdir().unwrap();
887 let default_repo = dir.path().join("default");
888 let task_repo = dir.path().join("task");
889 std::fs::create_dir_all(&default_repo).unwrap();
890 std::fs::create_dir_all(&task_repo).unwrap();
891 init_repo_with_branch(&default_repo, "other-branch");
892 init_repo_with_branch(&task_repo, "magi/f00d/A");
893
894 let mut config = Config::default();
895 config.graph.review_rounds = 6;
896 let mut state = crate::run::RunState::new(
897 task_repo.clone(),
898 "main".to_owned(),
899 "deadbeef".to_owned(),
900 "task".to_owned(),
901 config,
902 );
903 state.status = crate::run::RunStatus::Blocked;
904 state.candidates.push(crate::run::Candidate {
905 index: 0,
906 label: 'A',
907 agent: "mock".to_owned(),
908 branch: "magi/f00d/A".to_owned(),
909 worktree: task_repo.clone(),
910 summary: String::new(),
911 stat: String::new(),
912 files: 1,
913 commits: 1,
914 empty: false,
915 failed: None,
916 verified_noop: None,
917 duration_ms: 0,
918 folded: false,
919 });
920 state.tally = Some(crate::run::Tally {
921 first_choice: std::collections::BTreeMap::new(),
922 borda: std::collections::BTreeMap::new(),
923 winner: 'A',
924 rankings: 0,
925 unanimous_initial: false,
926 deliberated: false,
927 changed_votes: 0,
928 unanimous_final: false,
929 tie_break: None,
930 judges: 0,
931 present: 0,
932 quorum: 0,
933 met_quorum: true,
934 uncontested: Some("solo".to_owned()),
935 });
936 state.reviews = vec![
937 review_round_with_finding(
938 1,
939 "R1-1-2",
940 "answer content is dropped",
941 &[],
942 &[("R1-1-2", "the id leaving blocked_by is enough")],
943 ),
944 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
945 ];
946 state.save().unwrap();
947
948 let mut t = task("outcome test");
949 t.repo = task_repo;
950 t.runs.push(state.id.clone());
951
952 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
953 let outcome = finished.outcome;
954
955 assert!(outcome.unreadable.is_none());
956 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
957 assert_eq!(outcome.rounds_used, 2);
958 assert_eq!(outcome.rounds_max, 6);
959 assert_eq!(outcome.rounds.len(), 2);
960 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
961 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
962 assert!(outcome.rounds[1].addressed.is_empty());
963 assert!(outcome.rounds[1].rejected.is_empty());
964 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
965 assert!(
966 outcome.branch_head.is_some(),
967 "a real branch must resolve a head commit: {outcome:?}"
968 );
969 }
970
971 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
975 tokio::runtime::Builder::new_current_thread()
976 .enable_all()
977 .build()
978 .unwrap()
979 .block_on(f)
980 }
981
982 #[test]
983 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
984 let mut t = task("answered");
985 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
986 let v = view(&t, 2);
987 assert_eq!(v.answers.len(), 1);
988 assert_eq!(v.answers[0].question, "Which backend?");
989 assert_eq!(v.answers[0].answer, "SQLite");
990 }
991
992 #[test]
993 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
994 let dir = tempdir().unwrap();
995 let queue = Queue::at(dir.path().join("queue"));
996 let questions = Questions::at(dir.path().join("questions"));
997 let mut a = task("a");
998 a.priority = 9;
999 queue.put(&mut a).unwrap();
1000
1001 let verdict = Verdict {
1002 decisions: vec![Decision {
1003 id: a.id.clone(),
1004 blocked_by: vec!["20260101-000000-dead".to_owned()],
1005 reason: Some("waits on the other task".to_owned()),
1006 recovery: None,
1007 question: None,
1008 choices: Vec::new(),
1009 }],
1010 };
1011 apply(&queue, &questions, &verdict).unwrap();
1012
1013 let back = queue.get(&a.id).unwrap();
1014 assert_eq!(back.status, TaskStatus::Blocked);
1015 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
1016 assert_eq!(
1017 back.priority, 9,
1018 "the conductor's reply cannot carry priority"
1019 );
1020 }
1021
1022 #[test]
1023 fn a_question_decision_files_one_and_blocks_on_its_id() {
1024 let dir = tempdir().unwrap();
1025 let queue = Queue::at(dir.path().join("queue"));
1026 let questions = Questions::at(dir.path().join("questions"));
1027 let mut t = task("ambiguous");
1028 queue.put(&mut t).unwrap();
1029
1030 let verdict = Verdict {
1031 decisions: vec![Decision {
1032 id: t.id.clone(),
1033 blocked_by: Vec::new(),
1034 reason: Some("which backend?".to_owned()),
1035 recovery: None,
1036 question: Some("Which storage backend?".to_owned()),
1037 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
1038 }],
1039 };
1040 apply(&queue, &questions, &verdict).unwrap();
1041
1042 let back = queue.get(&t.id).unwrap();
1043 assert_eq!(back.status, TaskStatus::Blocked);
1044 assert_eq!(back.blocked_by.len(), 1);
1045 let q = questions.get(&back.blocked_by[0]).unwrap();
1046 assert_eq!(q.summary, "Which storage backend?");
1047 assert_eq!(q.node, NODE);
1048 assert!(q.status.open());
1049 }
1050
1051 #[test]
1052 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
1053 let dir = tempdir().unwrap();
1054 let queue = Queue::at(dir.path().join("queue"));
1055 let questions = Questions::at(dir.path().join("questions"));
1056 let mut t = task("asked once");
1057 queue.put(&mut t).unwrap();
1058
1059 let decision = Decision {
1060 id: t.id.clone(),
1061 reason: Some("still deciding".to_owned()),
1062 question: Some("Which backend?".to_owned()),
1063 ..Decision::default()
1064 };
1065 apply(
1066 &queue,
1067 &questions,
1068 &Verdict {
1069 decisions: vec![decision.clone()],
1070 },
1071 )
1072 .unwrap();
1073 assert_eq!(questions.list().len(), 1);
1074 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
1075
1076 let mut released = queue.get(&t.id).unwrap();
1081 released.release();
1082 queue.put(&mut released).unwrap();
1083
1084 apply(
1085 &queue,
1086 &questions,
1087 &Verdict {
1088 decisions: vec![decision],
1089 },
1090 )
1091 .unwrap();
1092 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
1093 let after = queue.get(&t.id).unwrap();
1094 assert_eq!(
1095 after.blocked_by,
1096 [first_question_id],
1097 "the existing open question is reused, not replaced"
1098 );
1099 }
1100
1101 #[test]
1102 fn a_same_id_question_from_another_node_is_not_reused() {
1103 let dir = tempdir().unwrap();
1104 let queue = Queue::at(dir.path().join("queue"));
1105 let questions = Questions::at(dir.path().join("questions"));
1106 let mut t = task("must ask the conductor");
1107 queue.put(&mut t).unwrap();
1108
1109 let mut unrelated = Question::new(
1110 t.id.clone(),
1111 "review".to_owned(),
1112 "reviewer-1".to_owned(),
1113 "An unrelated review question".to_owned(),
1114 String::new(),
1115 Vec::new(),
1116 );
1117 questions.put(&mut unrelated).unwrap();
1118
1119 apply(
1120 &queue,
1121 &questions,
1122 &Verdict {
1123 decisions: vec![Decision {
1124 id: t.id.clone(),
1125 question: Some("Which backend?".to_owned()),
1126 ..Decision::default()
1127 }],
1128 },
1129 )
1130 .unwrap();
1131
1132 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
1133 assert_eq!(blocked_by.len(), 1);
1134 assert_ne!(blocked_by[0], unrelated.id);
1135 assert!(questions.get(&unrelated.id).unwrap().status.open());
1136 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
1137 }
1138
1139 #[test]
1140 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
1141 let dir = tempdir().unwrap();
1142 let queue = Queue::at(dir.path().join("queue"));
1143 let questions = Questions::at(dir.path().join("questions"));
1144 let mut t = task("waits on an answer");
1145 queue.put(&mut t).unwrap();
1146
1147 apply(
1148 &queue,
1149 &questions,
1150 &Verdict {
1151 decisions: vec![Decision {
1152 id: t.id.clone(),
1153 blocked_by: Vec::new(),
1154 reason: None,
1155 recovery: None,
1156 question: Some("Which backend?".to_owned()),
1157 choices: Vec::new(),
1158 }],
1159 },
1160 )
1161 .unwrap();
1162 let blocked = queue.get(&t.id).unwrap();
1163 let question_id = blocked.blocked_by[0].clone();
1164
1165 let mut q = questions.get(&question_id).unwrap();
1166 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
1167 questions.put(&mut q).unwrap();
1168 assert_eq!(q.status, QuestionStatus::Answered);
1169
1170 let mut task_after = queue.get(&t.id).unwrap();
1174 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
1175 task_after.unblock(&question_id);
1176 assert_eq!(task_after.status, TaskStatus::Queued);
1177 assert_eq!(task_after.answers[0].answer, "SQLite");
1178 }
1179
1180 #[test]
1181 fn a_stalled_task_can_be_requeued_or_held() {
1182 let dir = tempdir().unwrap();
1183 let queue = Queue::at(dir.path().join("queue"));
1184 let questions = Questions::at(dir.path().join("questions"));
1185
1186 let mut requeue_me = task("stuck a");
1187 requeue_me.start("run-1".to_owned());
1188 queue.put(&mut requeue_me).unwrap();
1189
1190 let mut hold_me = task("stuck b");
1191 hold_me.start("run-2".to_owned());
1192 queue.put(&mut hold_me).unwrap();
1193
1194 apply(
1195 &queue,
1196 &questions,
1197 &Verdict {
1198 decisions: vec![
1199 Decision {
1200 id: requeue_me.id.clone(),
1201 recovery: Some(Recovery::Requeue),
1202 ..Decision::default()
1203 },
1204 Decision {
1205 id: hold_me.id.clone(),
1206 recovery: Some(Recovery::Hold),
1207 reason: Some("looks broken".to_owned()),
1208 ..Decision::default()
1209 },
1210 ],
1211 },
1212 )
1213 .unwrap();
1214
1215 let requeued = queue.get(&requeue_me.id).unwrap();
1216 assert_eq!(requeued.status, TaskStatus::Queued);
1217 assert_eq!(requeued.attempts, 0);
1218
1219 let held = queue.get(&hold_me.id).unwrap();
1220 assert_eq!(held.status, TaskStatus::Held);
1221 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
1222 }
1223
1224 #[test]
1225 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
1226 let dir = tempdir().unwrap();
1231 let queue = Queue::at(dir.path().join("queue"));
1232 let questions = Questions::at(dir.path().join("questions"));
1233 let mut t = task("held out of attempts");
1234 t.hold_machine(Some("out of attempts".to_owned()));
1235 queue.put(&mut t).unwrap();
1236
1237 apply(
1238 &queue,
1239 &questions,
1240 &Verdict {
1241 decisions: vec![Decision {
1242 id: t.id.clone(),
1243 reason: Some("what should happen to this one?".to_owned()),
1244 question: Some("Hold it, or try again?".to_owned()),
1245 ..Decision::default()
1246 }],
1247 },
1248 )
1249 .unwrap();
1250 let blocked = queue.get(&t.id).unwrap();
1251 assert_eq!(blocked.status, TaskStatus::Blocked);
1252 let question_id = blocked.blocked_by[0].clone();
1253
1254 let mut q = questions.get(&question_id).unwrap();
1255 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
1256 questions.put(&mut q).unwrap();
1257
1258 let mut after = queue.get(&t.id).unwrap();
1260 after.record_answer(q.summary.clone(), "leave it held".to_owned());
1261 after.unblock(&question_id);
1262 assert_eq!(
1263 after.status,
1264 TaskStatus::Held,
1265 "must not fall back to queued"
1266 );
1267 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
1268 }
1269
1270 #[test]
1271 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
1272 let dir = tempdir().unwrap();
1282 let queue = Queue::at(dir.path().join("queue"));
1283 let questions = Questions::at(dir.path().join("questions"));
1284 let mut t = task("disk pressure, then reconsidered");
1285 t.hold_machine(Some(
1286 "not enough free space to start a run: 10 bytes free, 100 required by \
1287 `[disk] min_free_bytes`"
1288 .to_owned(),
1289 ));
1290 t.record_answer(
1291 "How should this be handled?".to_owned(),
1292 "keep it held, a human will look at it later".to_owned(),
1293 );
1294 queue.put(&mut t).unwrap();
1295
1296 apply(
1297 &queue,
1298 &questions,
1299 &Verdict {
1300 decisions: vec![Decision {
1301 id: t.id.clone(),
1302 recovery: Some(Recovery::Hold),
1303 ..Decision::default()
1304 }],
1305 },
1306 )
1307 .unwrap();
1308
1309 let after = queue.get(&t.id).unwrap();
1310 assert_eq!(after.status, TaskStatus::Held);
1311 assert!(
1312 !after
1313 .hold_reason
1314 .as_deref()
1315 .unwrap_or_default()
1316 .starts_with("not enough free space"),
1317 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
1318 after.hold_reason
1319 );
1320
1321 let cfg_dir = tempdir().unwrap();
1324 let config = cfg_dir.path().join("magi.toml");
1325 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1326 let report =
1327 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
1328 assert!(report.resumed.is_empty(), "must not be auto-released");
1329 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
1330 }
1331
1332 #[test]
1333 fn a_conductor_rehold_after_a_resume_answer_is_not_asked_again_identically() {
1334 let dir = tempdir().unwrap();
1335 let queue = Queue::at(dir.path().join("queue"));
1336 let questions = Questions::at(dir.path().join("questions"));
1337 let config = dir.path().join("magi.toml");
1338 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1339 let now = jiff::Timestamp::now();
1340 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
1341 let open = || {
1342 questions
1343 .list()
1344 .into_iter()
1345 .filter(|q| q.node == "triage" && q.status.open())
1346 .collect::<Vec<_>>()
1347 };
1348 let hold = Decision {
1349 recovery: Some(Recovery::Hold),
1350 reason: Some("waiting on manual worktree cleanup".to_owned()),
1351 ..Decision::default()
1352 };
1353
1354 let mut t = task("looping hold");
1355 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
1356 queue.put(&mut t).unwrap();
1357 let hold = Decision {
1358 id: t.id.clone(),
1359 ..hold
1360 };
1361
1362 assert_eq!(triage().asked.len(), 1);
1364 let first = open().remove(0);
1365 let mut q = questions.get(&first.id).unwrap();
1366 let resume = q.choices[0].clone();
1367 q.answer(Answer::Choice(resume)).unwrap();
1368 questions.put(&mut q).unwrap();
1369 assert_eq!(triage().answered.len(), 1);
1370 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1371
1372 apply(
1374 &queue,
1375 &questions,
1376 &Verdict {
1377 decisions: vec![hold.clone()],
1378 },
1379 )
1380 .unwrap();
1381 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
1382
1383 assert_eq!(triage().asked.len(), 1);
1385 let second = open().remove(0);
1386 assert_ne!(second.summary, first.summary);
1387 assert_ne!(second.choices, first.choices);
1388 assert!(second.detail.contains("waiting on manual worktree cleanup"));
1389 assert!(triage().asked.is_empty(), "no duplicate question");
1390 assert_eq!(open().len(), 1);
1391
1392 let mut q = questions.get(&second.id).unwrap();
1394 let force = q.choices[0].clone();
1395 q.answer(Answer::Choice(force)).unwrap();
1396 questions.put(&mut q).unwrap();
1397 assert_eq!(triage().answered.len(), 1);
1398 apply(
1399 &queue,
1400 &questions,
1401 &Verdict {
1402 decisions: vec![hold],
1403 },
1404 )
1405 .unwrap();
1406 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1407 }
1408
1409 #[test]
1410 fn a_runnable_task_can_be_held_directly_without_a_question() {
1411 let dir = tempdir().unwrap();
1412 let queue = Queue::at(dir.path().join("queue"));
1413 let questions = Questions::at(dir.path().join("questions"));
1414 let mut t = task("already answered, should stay put");
1415 queue.put(&mut t).unwrap();
1416
1417 apply(
1418 &queue,
1419 &questions,
1420 &Verdict {
1421 decisions: vec![Decision {
1422 id: t.id.clone(),
1423 recovery: Some(Recovery::Hold),
1424 reason: Some("operator already said keep this held".to_owned()),
1425 ..Decision::default()
1426 }],
1427 },
1428 )
1429 .unwrap();
1430
1431 let after = queue.get(&t.id).unwrap();
1432 assert_eq!(after.status, TaskStatus::Held);
1433 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
1434 }
1435
1436 #[test]
1437 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
1438 let dir = tempdir().unwrap();
1444 let queue = Queue::at(dir.path().join("queue"));
1445 let questions = Questions::at(dir.path().join("questions"));
1446 let mut t = task("already merged by hand");
1447 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
1448 t.record_answer(
1449 "Handle this one?".to_owned(),
1450 "already merged and cleaned up, close it".to_owned(),
1451 );
1452 queue.put(&mut t).unwrap();
1453
1454 apply(
1455 &queue,
1456 &questions,
1457 &Verdict {
1458 decisions: vec![Decision {
1459 id: t.id.clone(),
1460 recovery: Some(Recovery::Done),
1461 reason: Some("operator confirmed this already landed".to_owned()),
1462 ..Decision::default()
1463 }],
1464 },
1465 )
1466 .unwrap();
1467
1468 let after = queue.get(&t.id).unwrap();
1469 assert_eq!(after.status, TaskStatus::Done);
1470 assert!(after.hold_reason.is_none());
1471 assert_eq!(after.answers.len(), 1, "the record of why is kept");
1472 }
1473
1474 #[test]
1475 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
1476 let dir = tempdir().unwrap();
1477 let queue = Queue::at(dir.path().join("queue"));
1478 let questions = Questions::at(dir.path().join("questions"));
1479
1480 let mut queued = task("never ran yet");
1481 queue.put(&mut queued).unwrap();
1482
1483 let mut running = task("mid-run");
1484 running.start("run-1".to_owned());
1485 queue.put(&mut running).unwrap();
1486
1487 for id in [queued.id.clone(), running.id.clone()] {
1488 apply(
1489 &queue,
1490 &questions,
1491 &Verdict {
1492 decisions: vec![Decision {
1493 id,
1494 recovery: Some(Recovery::Done),
1495 ..Decision::default()
1496 }],
1497 },
1498 )
1499 .unwrap();
1500 }
1501
1502 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
1503 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
1504 }
1505
1506 #[test]
1507 fn a_third_conductor_question_after_two_settled_answers_holds_instead_of_asking_again() {
1508 let dir = tempdir().unwrap();
1514 let queue = Queue::at(dir.path().join("queue"));
1515 let questions = Questions::at(dir.path().join("questions"));
1516 let mut t = task("asked about repeatedly");
1517 t.hold_machine(Some("out of attempts".to_owned()));
1518 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
1519 t.record_answer(
1520 "Handle this one? (2)".to_owned(),
1521 "still not yet".to_owned(),
1522 );
1523 queue.put(&mut t).unwrap();
1524 assert_eq!(questions.list().len(), 0);
1525
1526 apply(
1527 &queue,
1528 &questions,
1529 &Verdict {
1530 decisions: vec![Decision {
1531 id: t.id.clone(),
1532 question: Some("Handle this one? (3)".to_owned()),
1533 ..Decision::default()
1534 }],
1535 },
1536 )
1537 .unwrap();
1538
1539 assert_eq!(questions.list().len(), 0, "no third question was filed");
1540 let after = queue.get(&t.id).unwrap();
1541 assert_eq!(after.status, TaskStatus::Held);
1542 assert!(after.blocked_by.is_empty());
1543 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
1544 }
1545
1546 #[test]
1547 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
1548 let dir = tempdir().unwrap();
1549 let queue = Queue::at(dir.path().join("queue"));
1550 let questions = Questions::at(dir.path().join("questions"));
1551 let mut t = task("asked about once already");
1552 t.hold_machine(Some("out of attempts".to_owned()));
1553 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
1554 queue.put(&mut t).unwrap();
1555
1556 apply(
1557 &queue,
1558 &questions,
1559 &Verdict {
1560 decisions: vec![Decision {
1561 id: t.id.clone(),
1562 question: Some("Still not sure - now what?".to_owned()),
1563 ..Decision::default()
1564 }],
1565 },
1566 )
1567 .unwrap();
1568
1569 assert_eq!(questions.list().len(), 1, "the second question was filed");
1570 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1571 }
1572
1573 #[test]
1574 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
1575 let dir = tempdir().unwrap();
1581 let queue = Queue::at(dir.path().join("queue"));
1582 let questions = Questions::at(dir.path().join("questions"));
1583 let mut t = task("held, triage already asking about it");
1584 t.hold_machine(Some("cause unclear".to_owned()));
1585 queue.put(&mut t).unwrap();
1586
1587 let mut triage_q = Question::new(
1588 t.id.clone(),
1589 crate::triage::NODE.to_owned(),
1590 "triage".to_owned(),
1591 "Still needed?".to_owned(),
1592 String::new(),
1593 vec![
1594 "resume".to_owned(),
1595 "not yet".to_owned(),
1596 "discard".to_owned(),
1597 ],
1598 );
1599 questions.put(&mut triage_q).unwrap();
1600
1601 for decision in [
1602 Decision {
1603 id: t.id.clone(),
1604 question: Some("what now?".to_owned()),
1605 ..Decision::default()
1606 },
1607 Decision {
1608 id: t.id.clone(),
1609 recovery: Some(Recovery::Requeue),
1610 ..Decision::default()
1611 },
1612 ] {
1613 apply(
1614 &queue,
1615 &questions,
1616 &Verdict {
1617 decisions: vec![decision],
1618 },
1619 )
1620 .unwrap();
1621 }
1622
1623 let after = queue.get(&t.id).unwrap();
1624 assert_eq!(
1625 after.status,
1626 TaskStatus::Held,
1627 "triage still owns this hold"
1628 );
1629 assert!(after.blocked_by.is_empty());
1630 assert_eq!(
1631 questions.list().len(),
1632 1,
1633 "no second, conductor-owned question was filed"
1634 );
1635 }
1636
1637 #[test]
1638 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
1639 let dir = tempdir().unwrap();
1647 let queue = Queue::at(dir.path().join("queue"));
1648 let questions = Questions::at(dir.path().join("questions"));
1649 let mut t = task("held, triage question answered but not yet applied");
1650 t.hold_machine(Some("cause unclear".to_owned()));
1651 queue.put(&mut t).unwrap();
1652
1653 let mut triage_q = Question::new(
1654 t.id.clone(),
1655 crate::triage::NODE.to_owned(),
1656 "triage".to_owned(),
1657 "Still needed?".to_owned(),
1658 String::new(),
1659 vec![
1660 "resume".to_owned(),
1661 "not yet".to_owned(),
1662 "discard".to_owned(),
1663 ],
1664 );
1665 questions.put(&mut triage_q).unwrap();
1666 triage_q
1667 .answer(Answer::Choice("not yet".to_owned()))
1668 .unwrap();
1669 questions.put(&mut triage_q).unwrap();
1670 assert!(!triage_q.status.open());
1671
1672 apply(
1673 &queue,
1674 &questions,
1675 &Verdict {
1676 decisions: vec![Decision {
1677 id: t.id.clone(),
1678 question: Some("what now?".to_owned()),
1679 ..Decision::default()
1680 }],
1681 },
1682 )
1683 .unwrap();
1684
1685 let after = queue.get(&t.id).unwrap();
1686 assert_eq!(
1687 after.status,
1688 TaskStatus::Held,
1689 "triage's own answer is not yet applied - conduct must wait"
1690 );
1691 assert_eq!(
1692 questions.list().len(),
1693 1,
1694 "no conductor question was filed over the pending triage answer"
1695 );
1696 }
1697
1698 #[test]
1699 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
1700 let dir = tempdir().unwrap();
1701 let queue = Queue::at(dir.path().join("queue"));
1702 let questions = Questions::at(dir.path().join("questions"));
1703 let mut held = task("manual recovery");
1704 held.priority = 300;
1705 held.runs.push("run20260912-224242-daf5".to_owned());
1706 held.hold_manual(Some(
1707 "active manual recovery run20260912-224242-daf5".to_owned(),
1708 ));
1709 queue.put(&mut held).unwrap();
1710
1711 for decision in [
1715 Decision {
1716 id: held.id.clone(),
1717 recovery: Some(Recovery::Requeue),
1718 ..Decision::default()
1719 },
1720 Decision {
1721 id: held.id.clone(),
1722 recovery: Some(Recovery::Hold),
1723 reason: Some("stale replacement reason".to_owned()),
1724 ..Decision::default()
1725 },
1726 Decision {
1727 id: held.id.clone(),
1728 recovery: Some(Recovery::Review),
1729 ..Decision::default()
1730 },
1731 Decision {
1732 id: held.id.clone(),
1733 blocked_by: vec!["other-task".to_owned()],
1734 question: Some("retry now?".to_owned()),
1735 ..Decision::default()
1736 },
1737 ] {
1738 apply(
1739 &queue,
1740 &questions,
1741 &Verdict {
1742 decisions: vec![decision],
1743 },
1744 )
1745 .unwrap();
1746 }
1747
1748 let after = queue.get(&held.id).unwrap();
1749 assert_eq!(after.status, TaskStatus::Held);
1750 assert!(after.operator_held());
1751 assert_eq!(after.priority, 300);
1752 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
1753 assert_eq!(
1754 after.hold_reason.as_deref(),
1755 Some("active manual recovery run20260912-224242-daf5")
1756 );
1757 assert!(after.blocked_by.is_empty());
1758 assert!(questions.list().is_empty());
1759 assert!(
1760 queue.next_runnable().is_none(),
1761 "must not dispatch a duplicate"
1762 );
1763 }
1764
1765 #[test]
1766 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
1767 let dir = tempdir().unwrap();
1768 let queue = Queue::at(dir.path().join("queue"));
1769 let questions = Questions::at(dir.path().join("questions"));
1770
1771 let mut automatic = task("disk gate");
1772 automatic.hold_machine(Some("disk full".to_owned()));
1773 queue.put(&mut automatic).unwrap();
1774 let requeue = || Verdict {
1775 decisions: vec![Decision {
1776 id: automatic.id.clone(),
1777 recovery: Some(Recovery::Requeue),
1778 ..Decision::default()
1779 }],
1780 };
1781 apply(&queue, &questions, &requeue()).unwrap();
1782 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
1783
1784 let mut manual = task("operator gate");
1785 manual.hold_manual(Some("wait for operator".to_owned()));
1786 queue.put(&mut manual).unwrap();
1787 apply(
1788 &queue,
1789 &questions,
1790 &Verdict {
1791 decisions: vec![Decision {
1792 id: manual.id.clone(),
1793 recovery: Some(Recovery::Requeue),
1794 ..Decision::default()
1795 }],
1796 },
1797 )
1798 .unwrap();
1799 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
1800
1801 let mut released = queue.get(&manual.id).unwrap();
1804 released.release();
1805 queue.put(&mut released).unwrap();
1806 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
1807 }
1808
1809 #[test]
1810 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
1811 let dir = tempdir().unwrap();
1812 let queue = Queue::at(dir.path().join("queue"));
1813 let questions = Questions::at(dir.path().join("questions"));
1814 let mut legacy = task("old explicit hold");
1815 legacy.status = TaskStatus::Held;
1816 legacy.hold_reason = Some("manual recovery already active".to_owned());
1817 legacy.hold_source = None;
1818 legacy.blocked_by = vec!["dependency".to_owned()];
1819 queue.put(&mut legacy).unwrap();
1820
1821 apply(
1822 &queue,
1823 &questions,
1824 &Verdict {
1825 decisions: vec![Decision {
1826 id: legacy.id.clone(),
1827 recovery: Some(Recovery::Requeue),
1828 ..Decision::default()
1829 }],
1830 },
1831 )
1832 .unwrap();
1833
1834 let after = queue.get(&legacy.id).unwrap();
1835 assert_eq!(after.status, TaskStatus::Held);
1836 assert_eq!(after.hold_source, None);
1837 assert_eq!(after.hold_reason, legacy.hold_reason);
1838 assert_eq!(after.blocked_by, legacy.blocked_by);
1839 }
1840
1841 #[test]
1842 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
1843 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
1849 let dir = tempdir().unwrap();
1850 let queue = Queue::at(dir.path().join("queue"));
1851 let questions = Questions::at(dir.path().join("questions"));
1852 let mut t = task("blocked with no readable run");
1853 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
1855 queue.put(&mut t).unwrap();
1856
1857 apply(
1858 &queue,
1859 &questions,
1860 &Verdict {
1861 decisions: vec![Decision {
1862 id: t.id.clone(),
1863 recovery: Some(Recovery::Review),
1864 ..Decision::default()
1865 }],
1866 },
1867 )
1868 .unwrap();
1869
1870 let after = queue.get(&t.id).unwrap();
1871 assert_eq!(
1872 after.status,
1873 TaskStatus::Failed,
1874 "with nothing to reopen, the decision is dropped rather than guessed at"
1875 );
1876 assert!(after.review_branch.is_none());
1877 }
1878
1879 #[test]
1880 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
1881 let dir = tempdir().unwrap();
1885 let queue = Queue::at(dir.path().join("queue"));
1886 let questions = Questions::at(dir.path().join("questions"));
1887
1888 for recovery in [Recovery::Requeue, Recovery::Review] {
1889 let mut t = task("ordinary");
1890 queue.put(&mut t).unwrap();
1891
1892 apply(
1893 &queue,
1894 &questions,
1895 &Verdict {
1896 decisions: vec![Decision {
1897 id: t.id.clone(),
1898 recovery: Some(recovery),
1899 ..Decision::default()
1900 }],
1901 },
1902 )
1903 .unwrap();
1904
1905 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1906 }
1907 }
1908
1909 #[tokio::test]
1910 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
1911 let dir = tempdir().unwrap();
1912 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
1913 let queue = Queue::at(dir.path().join("queue"));
1914 let questions = Questions::at(dir.path().join("questions"));
1915 let mut t = task("normal");
1916 queue.put(&mut t).unwrap();
1917
1918 let mut conductor = Conductor::new();
1919 conductor
1920 .maybe_run(
1921 &cfg,
1922 dir.path(),
1923 &queue,
1924 &questions,
1925 dir.path(),
1926 &[t.clone()],
1927 &[],
1928 &[],
1929 2,
1930 )
1931 .await;
1932
1933 assert_eq!(
1934 queue.get(&t.id).unwrap().status,
1935 TaskStatus::Queued,
1936 "a failed invocation must change nothing"
1937 );
1938 assert!(
1939 queue.next_runnable().is_some(),
1940 "the loop must still be able to take the next task"
1941 );
1942 }
1943
1944 #[tokio::test]
1945 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
1946 let dir = tempdir().unwrap();
1947 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
1948 let queue = Queue::at(dir.path().join("queue"));
1949 let questions = Questions::at(dir.path().join("questions"));
1950 let mut t = task("normal");
1951 queue.put(&mut t).unwrap();
1952
1953 let mut conductor = Conductor::new();
1954 conductor
1955 .maybe_run(
1956 &cfg,
1957 dir.path(),
1958 &queue,
1959 &questions,
1960 dir.path(),
1961 &[t.clone()],
1962 &[],
1963 &[],
1964 2,
1965 )
1966 .await;
1967
1968 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1969 }
1970
1971 #[tokio::test]
1972 async fn json_survives_code_fences_and_a_preamble() {
1973 let dir = tempdir().unwrap();
1974 let mut t = task("fenced");
1975 let reply = format!(
1976 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
1977 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
1978 t.id
1979 );
1980 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1981 let queue = Queue::at(dir.path().join("queue"));
1982 let questions = Questions::at(dir.path().join("questions"));
1983 queue.put(&mut t).unwrap();
1984
1985 let mut conductor = Conductor::new();
1986 conductor
1987 .maybe_run(
1988 &cfg,
1989 dir.path(),
1990 &queue,
1991 &questions,
1992 dir.path(),
1993 &[t.clone()],
1994 &[],
1995 &[],
1996 2,
1997 )
1998 .await;
1999
2000 let back = queue.get(&t.id).unwrap();
2001 assert_eq!(back.status, TaskStatus::Blocked);
2002 assert_eq!(back.blocked_by, ["x"]);
2003 }
2004
2005 #[tokio::test]
2006 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
2007 let dir = tempdir().unwrap();
2010 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2011 let queue = Queue::at(dir.path().join("queue"));
2012 let questions = Questions::at(dir.path().join("questions"));
2013 let mut t = task("stable");
2014 queue.put(&mut t).unwrap();
2015 let artifacts = dir.path().join("conduct").join("artifacts");
2016 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2017
2018 let mut conductor = Conductor::new();
2019 conductor
2020 .maybe_run(
2021 &cfg,
2022 dir.path(),
2023 &queue,
2024 &questions,
2025 dir.path(),
2026 &[t.clone()],
2027 &[],
2028 &[],
2029 2,
2030 )
2031 .await;
2032 assert!(turn(1).is_file(), "the first cycle must call the conductor");
2033
2034 conductor
2035 .maybe_run(
2036 &cfg,
2037 dir.path(),
2038 &queue,
2039 &questions,
2040 dir.path(),
2041 &[t.clone()],
2042 &[],
2043 &[],
2044 2,
2045 )
2046 .await;
2047 assert!(
2048 !turn(2).is_file(),
2049 "an unchanged revision and an unchanged stalled/finished set must not call the \
2050 conductor twice"
2051 );
2052
2053 t.priority = 1;
2055 queue.put(&mut t).unwrap();
2056 conductor
2057 .maybe_run(
2058 &cfg,
2059 dir.path(),
2060 &queue,
2061 &questions,
2062 dir.path(),
2063 &[t.clone()],
2064 &[],
2065 &[],
2066 2,
2067 )
2068 .await;
2069 assert!(turn(2).is_file(), "a moved revision calls it again");
2070 }
2071
2072 #[tokio::test]
2073 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
2074 let dir = tempdir().unwrap();
2080 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2081 let queue = Queue::at(dir.path().join("queue"));
2082 let questions = Questions::at(dir.path().join("questions"));
2083 let mut t = task("quiet");
2084 queue.put(&mut t).unwrap();
2085 let artifacts = dir.path().join("conduct").join("artifacts");
2086 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2087
2088 let mut conductor = Conductor::new();
2089 conductor
2090 .maybe_run(
2091 &cfg,
2092 dir.path(),
2093 &queue,
2094 &questions,
2095 dir.path(),
2096 &[t.clone()],
2097 &[],
2098 &[],
2099 2,
2100 )
2101 .await;
2102 assert!(turn(1).is_file());
2103
2104 conductor
2105 .maybe_run(
2106 &cfg,
2107 dir.path(),
2108 &queue,
2109 &questions,
2110 dir.path(),
2111 &[],
2112 &[t.clone()],
2113 &[],
2114 2,
2115 )
2116 .await;
2117 assert!(
2118 turn(2).is_file(),
2119 "a task turning stalled must call the conductor again"
2120 );
2121
2122 conductor
2125 .maybe_run(
2126 &cfg,
2127 dir.path(),
2128 &queue,
2129 &questions,
2130 dir.path(),
2131 &[],
2132 &[t.clone()],
2133 &[],
2134 2,
2135 )
2136 .await;
2137 assert!(
2138 !turn(3).is_file(),
2139 "the same stalled task lingering must not call the conductor every cycle"
2140 );
2141 }
2142
2143 #[test]
2144 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
2145 let dir = tempdir().unwrap();
2146 let queue = Queue::at(dir.path().join("queue"));
2147 let mut t = task("t");
2148 queue.put(&mut t).unwrap();
2149
2150 let mut conductor = Conductor::new();
2151 assert!(
2152 conductor.worth_a_look(&queue, &[], &[]),
2153 "a conductor that has never run has something to look at"
2154 );
2155
2156 conductor.last_seen = Some(Conductor::snapshot(&queue, &[], &[]));
2157 assert!(
2158 !conductor.worth_a_look(&queue, &[], &[]),
2159 "nothing changed and nothing is stalled or finished"
2160 );
2161 assert!(
2162 conductor.worth_a_look(&queue, &[t.clone()], &[]),
2163 "a stalled task is worth a look even at the same revision"
2164 );
2165 assert!(
2166 conductor.worth_a_look(&queue, &[], &[t.clone()]),
2167 "a finished task is worth a look even at the same revision"
2168 );
2169 }
2170
2171 #[tokio::test]
2172 async fn the_conduct_path_never_calls_ask_and_wait() {
2173 let dir = tempdir().unwrap();
2179 let queue = Queue::at(dir.path().join("queue"));
2180 let questions = Questions::at(dir.path().join("questions"));
2181 let mut t = task("asks without blocking");
2182 queue.put(&mut t).unwrap();
2183
2184 apply(
2185 &queue,
2186 &questions,
2187 &Verdict {
2188 decisions: vec![Decision {
2189 id: t.id.clone(),
2190 question: Some("ok?".to_owned()),
2191 ..Decision::default()
2192 }],
2193 },
2194 )
2195 .unwrap();
2196 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2198 }
2199}