1use std::collections::BTreeSet;
42use std::path::{Path, PathBuf};
43use std::time::Duration;
44
45use anyhow::{Context as _, Result, bail};
46use serde::Deserialize;
47
48use crate::agent::{self, Invocation, SeatState};
49use crate::ask::{Question, Questions};
50use crate::config::Config;
51use crate::prompt;
52use crate::queue::{Queue, Task, TaskStatus};
53use crate::run::RunState;
54use crate::verdict;
55
56const SEAT: &str = "conduct";
59
60pub const NODE: &str = "conduct";
64
65const TURN_TIMEOUT: Duration = Duration::from_secs(300);
70
71const MAX_SETTLED_CONDUCT_ANSWERS: usize = 2;
89
90#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
93#[serde(rename_all = "lowercase")]
94pub enum Recovery {
95 Requeue,
98 Hold,
101 Review,
106 Done,
114}
115
116#[derive(Debug, Clone, Default, Deserialize)]
119pub struct Decision {
120 pub id: String,
122 #[serde(default)]
125 pub blocked_by: Vec<String>,
126 #[serde(default)]
128 pub reason: Option<String>,
129 #[serde(default)]
137 pub recovery: Option<Recovery>,
138 #[serde(default)]
142 pub question: Option<String>,
143 #[serde(default)]
145 pub choices: Vec<String>,
146}
147
148#[derive(Debug, Clone, Default, Deserialize)]
161pub struct Verdict {
162 pub decisions: Vec<Decision>,
165}
166
167fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
170 prompt::ConductTask {
171 id: t.id.clone(),
172 title: t.title.clone(),
173 instruction: t.instruction.clone(),
174 repo: t.repo.display().to_string(),
175 priority: t.priority,
176 status: t.status.as_str().to_owned(),
177 attempts: t.attempts,
178 max_attempts,
179 last_error: t.last_error.clone(),
180 hold_reason: t.hold_reason.clone(),
181 hold_source: t.hold_source.map(|source| source.label().to_owned()),
182 blocked_by: t.blocked_by.clone(),
183 answers: t
184 .answers
185 .iter()
186 .map(|a| prompt::ConductAnswer {
187 question: a.question.clone(),
188 answer: a.answer.clone(),
189 })
190 .collect(),
191 }
192}
193
194fn severity_str(s: crate::verdict::Severity) -> &'static str {
197 match s {
198 crate::verdict::Severity::Nit => "nit",
199 crate::verdict::Severity::Minor => "minor",
200 crate::verdict::Severity::Major => "major",
201 crate::verdict::Severity::Blocker => "blocker",
202 }
203}
204
205fn surviving_branch(task: &Task) -> Option<String> {
210 let last = task.runs.last()?;
211 let state = RunState::load(last).ok()?;
212 state.winner().map(|c| c.branch.clone())
213}
214
215fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> String {
235 let note = match &d.reason {
236 Some(reason) => reason.clone(),
237 None => match task.answers.last() {
238 Some(a) => format!(
239 "conduct held this again with no new reason given; last operator \
240 answer on record: {}",
241 a.answer
242 ),
243 None => "conduct held this again with no reason given".to_owned(),
244 },
245 };
246 match task.hold_reason.as_deref() {
247 Some(prior) if !prior.is_empty() => format!("{note}\n\n(previously: {prior})"),
248 _ => note,
249 }
250}
251
252async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
254 let Some(run_id) = task.runs.last().cloned() else {
255 return prompt::ConductOutcome {
256 run_id: "(none)".to_owned(),
257 unreadable: Some("this task has not produced a run yet".to_owned()),
258 run_status: None,
259 open_findings: Vec::new(),
260 rounds_used: 0,
261 rounds_max: 0,
262 rounds: Vec::new(),
263 branch: None,
264 branch_head: None,
265 };
266 };
267 let state = match RunState::load(&run_id) {
268 Ok(s) => s,
269 Err(e) => {
270 tracing::warn!(
275 "conductor: could not read run {run_id} for task {}: {e:#}",
276 task.short()
277 );
278 return prompt::ConductOutcome {
279 run_id,
280 unreadable: Some(format!("{e:#}")),
281 run_status: None,
282 open_findings: Vec::new(),
283 rounds_used: 0,
284 rounds_max: 0,
285 rounds: Vec::new(),
286 branch: None,
287 branch_head: None,
288 };
289 }
290 };
291
292 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
293 id: f.id.clone(),
294 title: f.title.clone(),
295 severity: severity_str(f.severity).to_owned(),
296 };
297 let open_findings = state
298 .open_findings()
299 .into_iter()
300 .map(finding_view)
301 .collect();
302 let rounds = state
303 .reviews
304 .iter()
305 .map(|r| prompt::ConductRound {
306 round: r.round,
307 findings: r
308 .reviews
309 .iter()
310 .flat_map(|rec| rec.findings.iter())
311 .map(finding_view)
312 .collect(),
313 addressed: r
314 .fix
315 .as_ref()
316 .map(|fx| fx.addressed.clone())
317 .unwrap_or_default(),
318 rejected: r
319 .fix
320 .as_ref()
321 .map(|fx| {
322 fx.rejected
323 .iter()
324 .map(|rej| prompt::ConductRejection {
325 id: rej.id.clone(),
326 why: rej.why.clone(),
327 })
328 .collect()
329 })
330 .unwrap_or_default(),
331 })
332 .collect();
333 let branch = state.winner().map(|c| c.branch.clone());
334 let branch_head = match &branch {
335 Some(b) => crate::git::rev_parse(repo, b)
336 .await
337 .ok()
338 .map(|h| h.chars().take(8).collect()),
339 None => None,
340 };
341
342 prompt::ConductOutcome {
343 run_id,
344 unreadable: None,
345 run_status: Some(state.status.as_str().to_owned()),
346 open_findings,
347 rounds_used: state.reviews.len(),
348 rounds_max: state.config.graph.review_rounds,
349 rounds,
350 branch,
351 branch_head,
352 }
353}
354
355async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
357 prompt::ConductFinished {
358 task: view(t, max_attempts),
359 outcome: outcome_for(t, &repo_for(t, repo)).await,
360 }
361}
362
363fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
366 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
367 fallback.to_path_buf()
368 } else {
369 task.repo.clone()
370 }
371}
372
373fn apply_one(queue: &Queue, questions: &Questions, d: &Decision) -> Result<()> {
390 let _claim = queue
391 .claim(&d.id)
392 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
393 let mut task = queue.get(&d.id).context("no such task")?;
394
395 if task.operator_held() {
399 return Ok(());
400 }
401
402 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
412 return Ok(());
413 }
414
415 if let Some(text) = &d.question {
416 if task.status == TaskStatus::Done {
417 return Ok(());
418 }
419 let question_id = match questions
423 .list()
424 .into_iter()
425 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
426 {
427 Some(existing) => existing.id,
428 None if task.answers.len() >= MAX_SETTLED_CONDUCT_ANSWERS => {
433 task.hold_machine(Some(format!(
434 "conduct tried to ask another question after {} were \
435 already answered about this task: {text}",
436 task.answers.len()
437 )));
438 return queue.put(&mut task);
439 }
440 None => {
441 let mut q = Question::new(
442 task.id.clone(),
443 NODE.to_owned(),
444 SEAT.to_owned(),
445 text.clone(),
446 d.reason.clone().unwrap_or_default(),
447 d.choices.clone(),
448 );
449 questions.put(&mut q)?;
450 q.id
451 }
452 };
453 task.block(vec![question_id], d.reason.clone());
454 return queue.put(&mut task);
455 }
456
457 match task.status {
458 TaskStatus::Queued if !d.blocked_by.is_empty() => {
459 task.block(d.blocked_by.clone(), d.reason.clone());
460 queue.put(&mut task)?;
461 }
462 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
466 task.hold_machine(d.reason.clone());
467 queue.put(&mut task)?;
468 }
469 TaskStatus::Running => match d.recovery {
470 Some(Recovery::Requeue) => {
471 task.requeue();
472 queue.put(&mut task)?;
473 }
474 Some(Recovery::Hold) => {
475 task.hold_machine(d.reason.clone());
476 queue.put(&mut task)?;
477 }
478 _ => {}
482 },
483 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
484 Some(Recovery::Requeue) => {
485 task.requeue();
486 queue.put(&mut task)?;
487 }
488 Some(Recovery::Hold) => {
489 task.hold_machine(Some(reaffirmed_hold_reason(&task, d)));
490 queue.put(&mut task)?;
491 }
492 Some(Recovery::Review) => {
493 if let Some(branch) = surviving_branch(&task) {
494 task.request_review(branch);
495 queue.put(&mut task)?;
496 }
497 }
503 Some(Recovery::Done) => {
504 task.succeed();
505 queue.put(&mut task)?;
506 }
507 None => {}
508 },
509 _ => {}
512 }
513 Ok(())
514}
515
516pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
520 for d in &verdict.decisions {
521 if let Err(e) = apply_one(queue, questions, d) {
522 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
523 }
524 }
525 Ok(())
526}
527
528#[derive(Debug, Default)]
531pub struct Conductor {
532 seat: Option<SeatState>,
533 last_seen: Option<(u64, BTreeSet<String>)>,
534}
535
536impl Conductor {
537 #[must_use]
539 pub fn new() -> Self {
540 Self::default()
541 }
542
543 fn snapshot(queue: &Queue, stalled: &[Task], finished: &[Task]) -> (u64, BTreeSet<String>) {
544 let ids = stalled
545 .iter()
546 .chain(finished)
547 .map(|t| t.id.clone())
548 .collect();
549 (queue.revision(), ids)
550 }
551
552 #[must_use]
569 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
570 self.last_seen.as_ref() != Some(&Self::snapshot(queue, stalled, finished))
571 }
572
573 #[allow(clippy::too_many_arguments)]
577 pub async fn maybe_run(
578 &mut self,
579 cfg: &Config,
580 repo: &Path,
581 queue: &Queue,
582 questions: &Questions,
583 home: &Path,
584 queued: &[Task],
585 stalled: &[Task],
586 finished: &[Task],
587 max_attempts: usize,
588 ) {
589 let snapshot = Self::snapshot(queue, stalled, finished);
590 if self.last_seen.as_ref() == Some(&snapshot) {
591 return;
592 }
593 self.last_seen = Some(snapshot);
594 if let Err(e) = self
595 .run_once(
596 cfg,
597 repo,
598 queue,
599 questions,
600 home,
601 queued,
602 stalled,
603 finished,
604 max_attempts,
605 )
606 .await
607 {
608 tracing::warn!("conductor: {e:#}");
609 }
610 }
611
612 #[allow(clippy::too_many_arguments)]
613 async fn run_once(
614 &mut self,
615 cfg: &Config,
616 repo: &Path,
617 queue: &Queue,
618 questions: &Questions,
619 home: &Path,
620 queued: &[Task],
621 stalled: &[Task],
622 finished: &[Task],
623 max_attempts: usize,
624 ) -> Result<()> {
625 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
626 return Ok(());
627 }
628
629 let spec = cfg
630 .resolve_roles()
631 .context("resolving the conductor seat")?
632 .conductor;
633 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
634 if needs_new_seat {
635 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
636 }
637 let seat = self.seat.as_mut().expect("just ensured a seat exists");
638
639 let runnable_views: Vec<prompt::ConductTask> =
640 queued.iter().map(|t| view(t, max_attempts)).collect();
641 let stalled_views: Vec<prompt::ConductTask> =
642 stalled.iter().map(|t| view(t, max_attempts)).collect();
643 let mut finished_views = Vec::with_capacity(finished.len());
644 for t in finished {
645 finished_views.push(finished_view(t, repo, max_attempts).await);
646 }
647
648 let body = prompt::with_overlay(
649 prompt::conduct(
650 &runnable_views,
651 &stalled_views,
652 &finished_views,
653 &cfg.graph.language,
654 ),
655 cfg.prompts.overlay(NODE),
656 );
657
658 let artifacts = home.join("conduct").join("artifacts");
659 let stem = format!("turn-{}", seat.turns + 1);
660 let cache_dir = cfg.cache_dir();
663 let inv = Invocation {
664 cwd: repo,
665 prompt: &body,
666 timeout: TURN_TIMEOUT,
667 allow_write: false,
670 sessions: cfg.graph.sessions,
671 artifacts: &artifacts,
672 stem: &stem,
673 run: NODE,
674 node: NODE,
675 cache_dir: cache_dir.as_deref(),
676 attachments: &[],
677 };
678
679 let out = agent::invoke(&spec, seat, &inv)
680 .await
681 .context("invoking the conductor")?;
682 if !out.usable() {
683 bail!(
684 "no usable reply (exit {:?}, timed out {})",
685 out.exit_code,
686 out.timed_out
687 );
688 }
689 let verdict: Verdict = verdict::extract_json(&out.text)
690 .context("the conductor's reply could not be parsed")?;
691 apply(queue, questions, &verdict)
692 }
693}
694
695#[cfg(test)]
696mod tests {
697 use std::collections::BTreeMap;
698
699 use tempfile::tempdir;
700
701 use super::*;
702 use crate::ask::{Answer, QuestionStatus};
703 use crate::config::{AgentKind, AgentSpec, Graph};
704 use crate::queue::Source;
705
706 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
707 let path = dir.join("mock-conduct-agent.sh");
708 std::fs::write(&path, script).expect("write mock");
709 AgentSpec {
710 id: "mock".to_owned(),
711 kind: AgentKind::Command,
712 model: None,
713 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
714 extra_args: Vec::new(),
715 env,
716 prompt_delivery: None,
717 }
718 }
719
720 fn config(spec: AgentSpec) -> Config {
721 Config {
722 agents: vec![spec],
723 graph: Graph {
724 language: "en".to_owned(),
725 ..Graph::default()
726 },
727 ..Config::default()
728 }
729 }
730
731 fn task(title: &str) -> Task {
732 Task::new(
733 title.to_owned(),
734 format!("do {title}"),
735 std::path::PathBuf::from("."),
736 Source::Human,
737 )
738 }
739
740 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
741 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
742
743 fn env(reply: &str) -> BTreeMap<String, String> {
744 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
745 }
746
747 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
748
749 fn init_repo_with_branch(dir: &Path, branch: &str) {
753 use crate::proc::Quiet as _;
754 let run = |args: &[&str]| {
755 let out = std::process::Command::new("git")
756 .args(args)
757 .current_dir(dir)
758 .quiet()
759 .output()
760 .expect("spawn git");
761 assert!(
762 out.status.success(),
763 "git {args:?} failed: {}",
764 String::from_utf8_lossy(&out.stderr)
765 );
766 };
767 run(&["init", "-b", "main"]);
768 run(&["config", "user.name", "magi test"]);
769 run(&["config", "user.email", "magi@example.com"]);
770 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
771 run(&["add", "-A"]);
772 run(&["commit", "-m", "init"]);
773 run(&["checkout", "-b", branch]);
774 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
775 run(&["add", "-A"]);
776 run(&["commit", "-m", "candidate work"]);
777 }
778
779 fn review_round_with_finding(
780 round: usize,
781 finding_id: &str,
782 title: &str,
783 addressed: &[&str],
784 rejected: &[(&str, &str)],
785 ) -> crate::run::ReviewRound {
786 crate::run::ReviewRound {
787 round,
788 head: "deadbeef".to_owned(),
789 verified_head: None,
790 verified_at: None,
791 reviews: vec![crate::run::ReviewRecord {
792 attempts: 0,
793 reviewer: 1,
794 agent: "mock".to_owned(),
795 summary: String::new(),
796 findings: vec![crate::verdict::Finding {
797 id: finding_id.to_owned(),
798 severity: crate::verdict::Severity::Major,
799 file: None,
800 line: None,
801 title: title.to_owned(),
802 detail: String::new(),
803 }],
804 vote: None,
805 failed: None,
806 duration_ms: 0,
807 }],
808 e2e: Vec::new(),
809 verify_retried: false,
810 e2e_deferred: false,
811 e2e_defer_reason: None,
812 fix: Some(crate::run::FixRecord {
813 agent: "mock".to_owned(),
814 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
815 rejected: rejected
816 .iter()
817 .map(|(id, why)| crate::verdict::Rejection {
818 id: (*id).to_owned(),
819 why: (*why).to_owned(),
820 })
821 .collect(),
822 notes: String::new(),
823 committed: false,
824 failed: None,
825 duration_ms: 0,
826 continuation: None,
827 }),
828 blocking: 1,
829 answered: 1,
830 expected: 1,
831 clean: false,
832 progressed: true,
833 vote_split: false,
834 reconsideration: Vec::new(),
835 verdict: None,
836 }
837 }
838
839 #[test]
840 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
841 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
842 let dir = tempdir().unwrap();
843 let default_repo = dir.path().join("default");
844 let task_repo = dir.path().join("task");
845 std::fs::create_dir_all(&default_repo).unwrap();
846 std::fs::create_dir_all(&task_repo).unwrap();
847 init_repo_with_branch(&default_repo, "other-branch");
848 init_repo_with_branch(&task_repo, "magi/f00d/A");
849
850 let mut config = Config::default();
851 config.graph.review_rounds = 6;
852 let mut state = crate::run::RunState::new(
853 task_repo.clone(),
854 "main".to_owned(),
855 "deadbeef".to_owned(),
856 "task".to_owned(),
857 config,
858 );
859 state.status = crate::run::RunStatus::Blocked;
860 state.candidates.push(crate::run::Candidate {
861 index: 0,
862 label: 'A',
863 agent: "mock".to_owned(),
864 branch: "magi/f00d/A".to_owned(),
865 worktree: task_repo.clone(),
866 summary: String::new(),
867 stat: String::new(),
868 files: 1,
869 commits: 1,
870 empty: false,
871 failed: None,
872 verified_noop: None,
873 duration_ms: 0,
874 folded: false,
875 });
876 state.tally = Some(crate::run::Tally {
877 first_choice: std::collections::BTreeMap::new(),
878 borda: std::collections::BTreeMap::new(),
879 winner: 'A',
880 rankings: 0,
881 unanimous_initial: false,
882 deliberated: false,
883 changed_votes: 0,
884 unanimous_final: false,
885 tie_break: None,
886 judges: 0,
887 present: 0,
888 quorum: 0,
889 met_quorum: true,
890 uncontested: Some("solo".to_owned()),
891 });
892 state.reviews = vec![
893 review_round_with_finding(
894 1,
895 "R1-1-2",
896 "answer content is dropped",
897 &[],
898 &[("R1-1-2", "the id leaving blocked_by is enough")],
899 ),
900 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
901 ];
902 state.save().unwrap();
903
904 let mut t = task("outcome test");
905 t.repo = task_repo;
906 t.runs.push(state.id.clone());
907
908 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
909 let outcome = finished.outcome;
910
911 assert!(outcome.unreadable.is_none());
912 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
913 assert_eq!(outcome.rounds_used, 2);
914 assert_eq!(outcome.rounds_max, 6);
915 assert_eq!(outcome.rounds.len(), 2);
916 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
917 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
918 assert!(outcome.rounds[1].addressed.is_empty());
919 assert!(outcome.rounds[1].rejected.is_empty());
920 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
921 assert!(
922 outcome.branch_head.is_some(),
923 "a real branch must resolve a head commit: {outcome:?}"
924 );
925 }
926
927 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
931 tokio::runtime::Builder::new_current_thread()
932 .enable_all()
933 .build()
934 .unwrap()
935 .block_on(f)
936 }
937
938 #[test]
939 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
940 let mut t = task("answered");
941 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
942 let v = view(&t, 2);
943 assert_eq!(v.answers.len(), 1);
944 assert_eq!(v.answers[0].question, "Which backend?");
945 assert_eq!(v.answers[0].answer, "SQLite");
946 }
947
948 #[test]
949 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
950 let dir = tempdir().unwrap();
951 let queue = Queue::at(dir.path().join("queue"));
952 let questions = Questions::at(dir.path().join("questions"));
953 let mut a = task("a");
954 a.priority = 9;
955 queue.put(&mut a).unwrap();
956
957 let verdict = Verdict {
958 decisions: vec![Decision {
959 id: a.id.clone(),
960 blocked_by: vec!["20260101-000000-dead".to_owned()],
961 reason: Some("waits on the other task".to_owned()),
962 recovery: None,
963 question: None,
964 choices: Vec::new(),
965 }],
966 };
967 apply(&queue, &questions, &verdict).unwrap();
968
969 let back = queue.get(&a.id).unwrap();
970 assert_eq!(back.status, TaskStatus::Blocked);
971 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
972 assert_eq!(
973 back.priority, 9,
974 "the conductor's reply cannot carry priority"
975 );
976 }
977
978 #[test]
979 fn a_question_decision_files_one_and_blocks_on_its_id() {
980 let dir = tempdir().unwrap();
981 let queue = Queue::at(dir.path().join("queue"));
982 let questions = Questions::at(dir.path().join("questions"));
983 let mut t = task("ambiguous");
984 queue.put(&mut t).unwrap();
985
986 let verdict = Verdict {
987 decisions: vec![Decision {
988 id: t.id.clone(),
989 blocked_by: Vec::new(),
990 reason: Some("which backend?".to_owned()),
991 recovery: None,
992 question: Some("Which storage backend?".to_owned()),
993 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
994 }],
995 };
996 apply(&queue, &questions, &verdict).unwrap();
997
998 let back = queue.get(&t.id).unwrap();
999 assert_eq!(back.status, TaskStatus::Blocked);
1000 assert_eq!(back.blocked_by.len(), 1);
1001 let q = questions.get(&back.blocked_by[0]).unwrap();
1002 assert_eq!(q.summary, "Which storage backend?");
1003 assert_eq!(q.node, NODE);
1004 assert!(q.status.open());
1005 }
1006
1007 #[test]
1008 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
1009 let dir = tempdir().unwrap();
1010 let queue = Queue::at(dir.path().join("queue"));
1011 let questions = Questions::at(dir.path().join("questions"));
1012 let mut t = task("asked once");
1013 queue.put(&mut t).unwrap();
1014
1015 let decision = Decision {
1016 id: t.id.clone(),
1017 reason: Some("still deciding".to_owned()),
1018 question: Some("Which backend?".to_owned()),
1019 ..Decision::default()
1020 };
1021 apply(
1022 &queue,
1023 &questions,
1024 &Verdict {
1025 decisions: vec![decision.clone()],
1026 },
1027 )
1028 .unwrap();
1029 assert_eq!(questions.list().len(), 1);
1030 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
1031
1032 let mut released = queue.get(&t.id).unwrap();
1037 released.release();
1038 queue.put(&mut released).unwrap();
1039
1040 apply(
1041 &queue,
1042 &questions,
1043 &Verdict {
1044 decisions: vec![decision],
1045 },
1046 )
1047 .unwrap();
1048 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
1049 let after = queue.get(&t.id).unwrap();
1050 assert_eq!(
1051 after.blocked_by,
1052 [first_question_id],
1053 "the existing open question is reused, not replaced"
1054 );
1055 }
1056
1057 #[test]
1058 fn a_same_id_question_from_another_node_is_not_reused() {
1059 let dir = tempdir().unwrap();
1060 let queue = Queue::at(dir.path().join("queue"));
1061 let questions = Questions::at(dir.path().join("questions"));
1062 let mut t = task("must ask the conductor");
1063 queue.put(&mut t).unwrap();
1064
1065 let mut unrelated = Question::new(
1066 t.id.clone(),
1067 "review".to_owned(),
1068 "reviewer-1".to_owned(),
1069 "An unrelated review question".to_owned(),
1070 String::new(),
1071 Vec::new(),
1072 );
1073 questions.put(&mut unrelated).unwrap();
1074
1075 apply(
1076 &queue,
1077 &questions,
1078 &Verdict {
1079 decisions: vec![Decision {
1080 id: t.id.clone(),
1081 question: Some("Which backend?".to_owned()),
1082 ..Decision::default()
1083 }],
1084 },
1085 )
1086 .unwrap();
1087
1088 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
1089 assert_eq!(blocked_by.len(), 1);
1090 assert_ne!(blocked_by[0], unrelated.id);
1091 assert!(questions.get(&unrelated.id).unwrap().status.open());
1092 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
1093 }
1094
1095 #[test]
1096 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
1097 let dir = tempdir().unwrap();
1098 let queue = Queue::at(dir.path().join("queue"));
1099 let questions = Questions::at(dir.path().join("questions"));
1100 let mut t = task("waits on an answer");
1101 queue.put(&mut t).unwrap();
1102
1103 apply(
1104 &queue,
1105 &questions,
1106 &Verdict {
1107 decisions: vec![Decision {
1108 id: t.id.clone(),
1109 blocked_by: Vec::new(),
1110 reason: None,
1111 recovery: None,
1112 question: Some("Which backend?".to_owned()),
1113 choices: Vec::new(),
1114 }],
1115 },
1116 )
1117 .unwrap();
1118 let blocked = queue.get(&t.id).unwrap();
1119 let question_id = blocked.blocked_by[0].clone();
1120
1121 let mut q = questions.get(&question_id).unwrap();
1122 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
1123 questions.put(&mut q).unwrap();
1124 assert_eq!(q.status, QuestionStatus::Answered);
1125
1126 let mut task_after = queue.get(&t.id).unwrap();
1130 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
1131 task_after.unblock(&question_id);
1132 assert_eq!(task_after.status, TaskStatus::Queued);
1133 assert_eq!(task_after.answers[0].answer, "SQLite");
1134 }
1135
1136 #[test]
1137 fn a_stalled_task_can_be_requeued_or_held() {
1138 let dir = tempdir().unwrap();
1139 let queue = Queue::at(dir.path().join("queue"));
1140 let questions = Questions::at(dir.path().join("questions"));
1141
1142 let mut requeue_me = task("stuck a");
1143 requeue_me.start("run-1".to_owned());
1144 queue.put(&mut requeue_me).unwrap();
1145
1146 let mut hold_me = task("stuck b");
1147 hold_me.start("run-2".to_owned());
1148 queue.put(&mut hold_me).unwrap();
1149
1150 apply(
1151 &queue,
1152 &questions,
1153 &Verdict {
1154 decisions: vec![
1155 Decision {
1156 id: requeue_me.id.clone(),
1157 recovery: Some(Recovery::Requeue),
1158 ..Decision::default()
1159 },
1160 Decision {
1161 id: hold_me.id.clone(),
1162 recovery: Some(Recovery::Hold),
1163 reason: Some("looks broken".to_owned()),
1164 ..Decision::default()
1165 },
1166 ],
1167 },
1168 )
1169 .unwrap();
1170
1171 let requeued = queue.get(&requeue_me.id).unwrap();
1172 assert_eq!(requeued.status, TaskStatus::Queued);
1173 assert_eq!(requeued.attempts, 0);
1174
1175 let held = queue.get(&hold_me.id).unwrap();
1176 assert_eq!(held.status, TaskStatus::Held);
1177 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
1178 }
1179
1180 #[test]
1181 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
1182 let dir = tempdir().unwrap();
1187 let queue = Queue::at(dir.path().join("queue"));
1188 let questions = Questions::at(dir.path().join("questions"));
1189 let mut t = task("held out of attempts");
1190 t.hold_machine(Some("out of attempts".to_owned()));
1191 queue.put(&mut t).unwrap();
1192
1193 apply(
1194 &queue,
1195 &questions,
1196 &Verdict {
1197 decisions: vec![Decision {
1198 id: t.id.clone(),
1199 reason: Some("what should happen to this one?".to_owned()),
1200 question: Some("Hold it, or try again?".to_owned()),
1201 ..Decision::default()
1202 }],
1203 },
1204 )
1205 .unwrap();
1206 let blocked = queue.get(&t.id).unwrap();
1207 assert_eq!(blocked.status, TaskStatus::Blocked);
1208 let question_id = blocked.blocked_by[0].clone();
1209
1210 let mut q = questions.get(&question_id).unwrap();
1211 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
1212 questions.put(&mut q).unwrap();
1213
1214 let mut after = queue.get(&t.id).unwrap();
1216 after.record_answer(q.summary.clone(), "leave it held".to_owned());
1217 after.unblock(&question_id);
1218 assert_eq!(
1219 after.status,
1220 TaskStatus::Held,
1221 "must not fall back to queued"
1222 );
1223 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
1224 }
1225
1226 #[test]
1227 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
1228 let dir = tempdir().unwrap();
1238 let queue = Queue::at(dir.path().join("queue"));
1239 let questions = Questions::at(dir.path().join("questions"));
1240 let mut t = task("disk pressure, then reconsidered");
1241 t.hold_machine(Some(
1242 "not enough free space to start a run: 10 bytes free, 100 required by \
1243 `[disk] min_free_bytes`"
1244 .to_owned(),
1245 ));
1246 t.record_answer(
1247 "How should this be handled?".to_owned(),
1248 "keep it held, a human will look at it later".to_owned(),
1249 );
1250 queue.put(&mut t).unwrap();
1251
1252 apply(
1253 &queue,
1254 &questions,
1255 &Verdict {
1256 decisions: vec![Decision {
1257 id: t.id.clone(),
1258 recovery: Some(Recovery::Hold),
1259 ..Decision::default()
1260 }],
1261 },
1262 )
1263 .unwrap();
1264
1265 let after = queue.get(&t.id).unwrap();
1266 assert_eq!(after.status, TaskStatus::Held);
1267 assert!(
1268 !after
1269 .hold_reason
1270 .as_deref()
1271 .unwrap_or_default()
1272 .starts_with("not enough free space"),
1273 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
1274 after.hold_reason
1275 );
1276
1277 let cfg_dir = tempdir().unwrap();
1280 let config = cfg_dir.path().join("magi.toml");
1281 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1282 let report =
1283 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
1284 assert!(report.resumed.is_empty(), "must not be auto-released");
1285 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
1286 }
1287
1288 #[test]
1289 fn a_runnable_task_can_be_held_directly_without_a_question() {
1290 let dir = tempdir().unwrap();
1291 let queue = Queue::at(dir.path().join("queue"));
1292 let questions = Questions::at(dir.path().join("questions"));
1293 let mut t = task("already answered, should stay put");
1294 queue.put(&mut t).unwrap();
1295
1296 apply(
1297 &queue,
1298 &questions,
1299 &Verdict {
1300 decisions: vec![Decision {
1301 id: t.id.clone(),
1302 recovery: Some(Recovery::Hold),
1303 reason: Some("operator already said keep this held".to_owned()),
1304 ..Decision::default()
1305 }],
1306 },
1307 )
1308 .unwrap();
1309
1310 let after = queue.get(&t.id).unwrap();
1311 assert_eq!(after.status, TaskStatus::Held);
1312 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
1313 }
1314
1315 #[test]
1316 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
1317 let dir = tempdir().unwrap();
1323 let queue = Queue::at(dir.path().join("queue"));
1324 let questions = Questions::at(dir.path().join("questions"));
1325 let mut t = task("already merged by hand");
1326 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
1327 t.record_answer(
1328 "Handle this one?".to_owned(),
1329 "already merged and cleaned up, close it".to_owned(),
1330 );
1331 queue.put(&mut t).unwrap();
1332
1333 apply(
1334 &queue,
1335 &questions,
1336 &Verdict {
1337 decisions: vec![Decision {
1338 id: t.id.clone(),
1339 recovery: Some(Recovery::Done),
1340 reason: Some("operator confirmed this already landed".to_owned()),
1341 ..Decision::default()
1342 }],
1343 },
1344 )
1345 .unwrap();
1346
1347 let after = queue.get(&t.id).unwrap();
1348 assert_eq!(after.status, TaskStatus::Done);
1349 assert!(after.hold_reason.is_none());
1350 assert_eq!(after.answers.len(), 1, "the record of why is kept");
1351 }
1352
1353 #[test]
1354 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
1355 let dir = tempdir().unwrap();
1356 let queue = Queue::at(dir.path().join("queue"));
1357 let questions = Questions::at(dir.path().join("questions"));
1358
1359 let mut queued = task("never ran yet");
1360 queue.put(&mut queued).unwrap();
1361
1362 let mut running = task("mid-run");
1363 running.start("run-1".to_owned());
1364 queue.put(&mut running).unwrap();
1365
1366 for id in [queued.id.clone(), running.id.clone()] {
1367 apply(
1368 &queue,
1369 &questions,
1370 &Verdict {
1371 decisions: vec![Decision {
1372 id,
1373 recovery: Some(Recovery::Done),
1374 ..Decision::default()
1375 }],
1376 },
1377 )
1378 .unwrap();
1379 }
1380
1381 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
1382 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
1383 }
1384
1385 #[test]
1386 fn a_third_conductor_question_after_two_settled_answers_holds_instead_of_asking_again() {
1387 let dir = tempdir().unwrap();
1393 let queue = Queue::at(dir.path().join("queue"));
1394 let questions = Questions::at(dir.path().join("questions"));
1395 let mut t = task("asked about repeatedly");
1396 t.hold_machine(Some("out of attempts".to_owned()));
1397 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
1398 t.record_answer(
1399 "Handle this one? (2)".to_owned(),
1400 "still not yet".to_owned(),
1401 );
1402 queue.put(&mut t).unwrap();
1403 assert_eq!(questions.list().len(), 0);
1404
1405 apply(
1406 &queue,
1407 &questions,
1408 &Verdict {
1409 decisions: vec![Decision {
1410 id: t.id.clone(),
1411 question: Some("Handle this one? (3)".to_owned()),
1412 ..Decision::default()
1413 }],
1414 },
1415 )
1416 .unwrap();
1417
1418 assert_eq!(questions.list().len(), 0, "no third question was filed");
1419 let after = queue.get(&t.id).unwrap();
1420 assert_eq!(after.status, TaskStatus::Held);
1421 assert!(after.blocked_by.is_empty());
1422 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
1423 }
1424
1425 #[test]
1426 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
1427 let dir = tempdir().unwrap();
1428 let queue = Queue::at(dir.path().join("queue"));
1429 let questions = Questions::at(dir.path().join("questions"));
1430 let mut t = task("asked about once already");
1431 t.hold_machine(Some("out of attempts".to_owned()));
1432 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
1433 queue.put(&mut t).unwrap();
1434
1435 apply(
1436 &queue,
1437 &questions,
1438 &Verdict {
1439 decisions: vec![Decision {
1440 id: t.id.clone(),
1441 question: Some("Still not sure - now what?".to_owned()),
1442 ..Decision::default()
1443 }],
1444 },
1445 )
1446 .unwrap();
1447
1448 assert_eq!(questions.list().len(), 1, "the second question was filed");
1449 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1450 }
1451
1452 #[test]
1453 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
1454 let dir = tempdir().unwrap();
1460 let queue = Queue::at(dir.path().join("queue"));
1461 let questions = Questions::at(dir.path().join("questions"));
1462 let mut t = task("held, triage already asking about it");
1463 t.hold_machine(Some("cause unclear".to_owned()));
1464 queue.put(&mut t).unwrap();
1465
1466 let mut triage_q = Question::new(
1467 t.id.clone(),
1468 crate::triage::NODE.to_owned(),
1469 "triage".to_owned(),
1470 "Still needed?".to_owned(),
1471 String::new(),
1472 vec![
1473 "resume".to_owned(),
1474 "not yet".to_owned(),
1475 "discard".to_owned(),
1476 ],
1477 );
1478 questions.put(&mut triage_q).unwrap();
1479
1480 for decision in [
1481 Decision {
1482 id: t.id.clone(),
1483 question: Some("what now?".to_owned()),
1484 ..Decision::default()
1485 },
1486 Decision {
1487 id: t.id.clone(),
1488 recovery: Some(Recovery::Requeue),
1489 ..Decision::default()
1490 },
1491 ] {
1492 apply(
1493 &queue,
1494 &questions,
1495 &Verdict {
1496 decisions: vec![decision],
1497 },
1498 )
1499 .unwrap();
1500 }
1501
1502 let after = queue.get(&t.id).unwrap();
1503 assert_eq!(
1504 after.status,
1505 TaskStatus::Held,
1506 "triage still owns this hold"
1507 );
1508 assert!(after.blocked_by.is_empty());
1509 assert_eq!(
1510 questions.list().len(),
1511 1,
1512 "no second, conductor-owned question was filed"
1513 );
1514 }
1515
1516 #[test]
1517 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
1518 let dir = tempdir().unwrap();
1526 let queue = Queue::at(dir.path().join("queue"));
1527 let questions = Questions::at(dir.path().join("questions"));
1528 let mut t = task("held, triage question answered but not yet applied");
1529 t.hold_machine(Some("cause unclear".to_owned()));
1530 queue.put(&mut t).unwrap();
1531
1532 let mut triage_q = Question::new(
1533 t.id.clone(),
1534 crate::triage::NODE.to_owned(),
1535 "triage".to_owned(),
1536 "Still needed?".to_owned(),
1537 String::new(),
1538 vec![
1539 "resume".to_owned(),
1540 "not yet".to_owned(),
1541 "discard".to_owned(),
1542 ],
1543 );
1544 questions.put(&mut triage_q).unwrap();
1545 triage_q
1546 .answer(Answer::Choice("not yet".to_owned()))
1547 .unwrap();
1548 questions.put(&mut triage_q).unwrap();
1549 assert!(!triage_q.status.open());
1550
1551 apply(
1552 &queue,
1553 &questions,
1554 &Verdict {
1555 decisions: vec![Decision {
1556 id: t.id.clone(),
1557 question: Some("what now?".to_owned()),
1558 ..Decision::default()
1559 }],
1560 },
1561 )
1562 .unwrap();
1563
1564 let after = queue.get(&t.id).unwrap();
1565 assert_eq!(
1566 after.status,
1567 TaskStatus::Held,
1568 "triage's own answer is not yet applied - conduct must wait"
1569 );
1570 assert_eq!(
1571 questions.list().len(),
1572 1,
1573 "no conductor question was filed over the pending triage answer"
1574 );
1575 }
1576
1577 #[test]
1578 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
1579 let dir = tempdir().unwrap();
1580 let queue = Queue::at(dir.path().join("queue"));
1581 let questions = Questions::at(dir.path().join("questions"));
1582 let mut held = task("manual recovery");
1583 held.priority = 300;
1584 held.runs.push("run20260912-224242-daf5".to_owned());
1585 held.hold_manual(Some(
1586 "active manual recovery run20260912-224242-daf5".to_owned(),
1587 ));
1588 queue.put(&mut held).unwrap();
1589
1590 for decision in [
1594 Decision {
1595 id: held.id.clone(),
1596 recovery: Some(Recovery::Requeue),
1597 ..Decision::default()
1598 },
1599 Decision {
1600 id: held.id.clone(),
1601 recovery: Some(Recovery::Hold),
1602 reason: Some("stale replacement reason".to_owned()),
1603 ..Decision::default()
1604 },
1605 Decision {
1606 id: held.id.clone(),
1607 recovery: Some(Recovery::Review),
1608 ..Decision::default()
1609 },
1610 Decision {
1611 id: held.id.clone(),
1612 blocked_by: vec!["other-task".to_owned()],
1613 question: Some("retry now?".to_owned()),
1614 ..Decision::default()
1615 },
1616 ] {
1617 apply(
1618 &queue,
1619 &questions,
1620 &Verdict {
1621 decisions: vec![decision],
1622 },
1623 )
1624 .unwrap();
1625 }
1626
1627 let after = queue.get(&held.id).unwrap();
1628 assert_eq!(after.status, TaskStatus::Held);
1629 assert!(after.operator_held());
1630 assert_eq!(after.priority, 300);
1631 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
1632 assert_eq!(
1633 after.hold_reason.as_deref(),
1634 Some("active manual recovery run20260912-224242-daf5")
1635 );
1636 assert!(after.blocked_by.is_empty());
1637 assert!(questions.list().is_empty());
1638 assert!(
1639 queue.next_runnable().is_none(),
1640 "must not dispatch a duplicate"
1641 );
1642 }
1643
1644 #[test]
1645 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
1646 let dir = tempdir().unwrap();
1647 let queue = Queue::at(dir.path().join("queue"));
1648 let questions = Questions::at(dir.path().join("questions"));
1649
1650 let mut automatic = task("disk gate");
1651 automatic.hold_machine(Some("disk full".to_owned()));
1652 queue.put(&mut automatic).unwrap();
1653 let requeue = || Verdict {
1654 decisions: vec![Decision {
1655 id: automatic.id.clone(),
1656 recovery: Some(Recovery::Requeue),
1657 ..Decision::default()
1658 }],
1659 };
1660 apply(&queue, &questions, &requeue()).unwrap();
1661 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
1662
1663 let mut manual = task("operator gate");
1664 manual.hold_manual(Some("wait for operator".to_owned()));
1665 queue.put(&mut manual).unwrap();
1666 apply(
1667 &queue,
1668 &questions,
1669 &Verdict {
1670 decisions: vec![Decision {
1671 id: manual.id.clone(),
1672 recovery: Some(Recovery::Requeue),
1673 ..Decision::default()
1674 }],
1675 },
1676 )
1677 .unwrap();
1678 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
1679
1680 let mut released = queue.get(&manual.id).unwrap();
1683 released.release();
1684 queue.put(&mut released).unwrap();
1685 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
1686 }
1687
1688 #[test]
1689 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
1690 let dir = tempdir().unwrap();
1691 let queue = Queue::at(dir.path().join("queue"));
1692 let questions = Questions::at(dir.path().join("questions"));
1693 let mut legacy = task("old explicit hold");
1694 legacy.status = TaskStatus::Held;
1695 legacy.hold_reason = Some("manual recovery already active".to_owned());
1696 legacy.hold_source = None;
1697 legacy.blocked_by = vec!["dependency".to_owned()];
1698 queue.put(&mut legacy).unwrap();
1699
1700 apply(
1701 &queue,
1702 &questions,
1703 &Verdict {
1704 decisions: vec![Decision {
1705 id: legacy.id.clone(),
1706 recovery: Some(Recovery::Requeue),
1707 ..Decision::default()
1708 }],
1709 },
1710 )
1711 .unwrap();
1712
1713 let after = queue.get(&legacy.id).unwrap();
1714 assert_eq!(after.status, TaskStatus::Held);
1715 assert_eq!(after.hold_source, None);
1716 assert_eq!(after.hold_reason, legacy.hold_reason);
1717 assert_eq!(after.blocked_by, legacy.blocked_by);
1718 }
1719
1720 #[test]
1721 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
1722 crate::run::set_home(std::env::temp_dir().join("magi-conduct-tests-home"));
1728 let dir = tempdir().unwrap();
1729 let queue = Queue::at(dir.path().join("queue"));
1730 let questions = Questions::at(dir.path().join("questions"));
1731 let mut t = task("blocked with no readable run");
1732 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
1734 queue.put(&mut t).unwrap();
1735
1736 apply(
1737 &queue,
1738 &questions,
1739 &Verdict {
1740 decisions: vec![Decision {
1741 id: t.id.clone(),
1742 recovery: Some(Recovery::Review),
1743 ..Decision::default()
1744 }],
1745 },
1746 )
1747 .unwrap();
1748
1749 let after = queue.get(&t.id).unwrap();
1750 assert_eq!(
1751 after.status,
1752 TaskStatus::Failed,
1753 "with nothing to reopen, the decision is dropped rather than guessed at"
1754 );
1755 assert!(after.review_branch.is_none());
1756 }
1757
1758 #[test]
1759 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
1760 let dir = tempdir().unwrap();
1764 let queue = Queue::at(dir.path().join("queue"));
1765 let questions = Questions::at(dir.path().join("questions"));
1766
1767 for recovery in [Recovery::Requeue, Recovery::Review] {
1768 let mut t = task("ordinary");
1769 queue.put(&mut t).unwrap();
1770
1771 apply(
1772 &queue,
1773 &questions,
1774 &Verdict {
1775 decisions: vec![Decision {
1776 id: t.id.clone(),
1777 recovery: Some(recovery),
1778 ..Decision::default()
1779 }],
1780 },
1781 )
1782 .unwrap();
1783
1784 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1785 }
1786 }
1787
1788 #[tokio::test]
1789 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
1790 let dir = tempdir().unwrap();
1791 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
1792 let queue = Queue::at(dir.path().join("queue"));
1793 let questions = Questions::at(dir.path().join("questions"));
1794 let mut t = task("normal");
1795 queue.put(&mut t).unwrap();
1796
1797 let mut conductor = Conductor::new();
1798 conductor
1799 .maybe_run(
1800 &cfg,
1801 dir.path(),
1802 &queue,
1803 &questions,
1804 dir.path(),
1805 &[t.clone()],
1806 &[],
1807 &[],
1808 2,
1809 )
1810 .await;
1811
1812 assert_eq!(
1813 queue.get(&t.id).unwrap().status,
1814 TaskStatus::Queued,
1815 "a failed invocation must change nothing"
1816 );
1817 assert!(
1818 queue.next_runnable().is_some(),
1819 "the loop must still be able to take the next task"
1820 );
1821 }
1822
1823 #[tokio::test]
1824 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
1825 let dir = tempdir().unwrap();
1826 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
1827 let queue = Queue::at(dir.path().join("queue"));
1828 let questions = Questions::at(dir.path().join("questions"));
1829 let mut t = task("normal");
1830 queue.put(&mut t).unwrap();
1831
1832 let mut conductor = Conductor::new();
1833 conductor
1834 .maybe_run(
1835 &cfg,
1836 dir.path(),
1837 &queue,
1838 &questions,
1839 dir.path(),
1840 &[t.clone()],
1841 &[],
1842 &[],
1843 2,
1844 )
1845 .await;
1846
1847 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1848 }
1849
1850 #[tokio::test]
1851 async fn json_survives_code_fences_and_a_preamble() {
1852 let dir = tempdir().unwrap();
1853 let mut t = task("fenced");
1854 let reply = format!(
1855 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
1856 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
1857 t.id
1858 );
1859 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1860 let queue = Queue::at(dir.path().join("queue"));
1861 let questions = Questions::at(dir.path().join("questions"));
1862 queue.put(&mut t).unwrap();
1863
1864 let mut conductor = Conductor::new();
1865 conductor
1866 .maybe_run(
1867 &cfg,
1868 dir.path(),
1869 &queue,
1870 &questions,
1871 dir.path(),
1872 &[t.clone()],
1873 &[],
1874 &[],
1875 2,
1876 )
1877 .await;
1878
1879 let back = queue.get(&t.id).unwrap();
1880 assert_eq!(back.status, TaskStatus::Blocked);
1881 assert_eq!(back.blocked_by, ["x"]);
1882 }
1883
1884 #[tokio::test]
1885 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
1886 let dir = tempdir().unwrap();
1889 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
1890 let queue = Queue::at(dir.path().join("queue"));
1891 let questions = Questions::at(dir.path().join("questions"));
1892 let mut t = task("stable");
1893 queue.put(&mut t).unwrap();
1894 let artifacts = dir.path().join("conduct").join("artifacts");
1895 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
1896
1897 let mut conductor = Conductor::new();
1898 conductor
1899 .maybe_run(
1900 &cfg,
1901 dir.path(),
1902 &queue,
1903 &questions,
1904 dir.path(),
1905 &[t.clone()],
1906 &[],
1907 &[],
1908 2,
1909 )
1910 .await;
1911 assert!(turn(1).is_file(), "the first cycle must call the conductor");
1912
1913 conductor
1914 .maybe_run(
1915 &cfg,
1916 dir.path(),
1917 &queue,
1918 &questions,
1919 dir.path(),
1920 &[t.clone()],
1921 &[],
1922 &[],
1923 2,
1924 )
1925 .await;
1926 assert!(
1927 !turn(2).is_file(),
1928 "an unchanged revision and an unchanged stalled/finished set must not call the \
1929 conductor twice"
1930 );
1931
1932 t.priority = 1;
1934 queue.put(&mut t).unwrap();
1935 conductor
1936 .maybe_run(
1937 &cfg,
1938 dir.path(),
1939 &queue,
1940 &questions,
1941 dir.path(),
1942 &[t.clone()],
1943 &[],
1944 &[],
1945 2,
1946 )
1947 .await;
1948 assert!(turn(2).is_file(), "a moved revision calls it again");
1949 }
1950
1951 #[tokio::test]
1952 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
1953 let dir = tempdir().unwrap();
1959 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
1960 let queue = Queue::at(dir.path().join("queue"));
1961 let questions = Questions::at(dir.path().join("questions"));
1962 let mut t = task("quiet");
1963 queue.put(&mut t).unwrap();
1964 let artifacts = dir.path().join("conduct").join("artifacts");
1965 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
1966
1967 let mut conductor = Conductor::new();
1968 conductor
1969 .maybe_run(
1970 &cfg,
1971 dir.path(),
1972 &queue,
1973 &questions,
1974 dir.path(),
1975 &[t.clone()],
1976 &[],
1977 &[],
1978 2,
1979 )
1980 .await;
1981 assert!(turn(1).is_file());
1982
1983 conductor
1984 .maybe_run(
1985 &cfg,
1986 dir.path(),
1987 &queue,
1988 &questions,
1989 dir.path(),
1990 &[],
1991 &[t.clone()],
1992 &[],
1993 2,
1994 )
1995 .await;
1996 assert!(
1997 turn(2).is_file(),
1998 "a task turning stalled must call the conductor again"
1999 );
2000
2001 conductor
2004 .maybe_run(
2005 &cfg,
2006 dir.path(),
2007 &queue,
2008 &questions,
2009 dir.path(),
2010 &[],
2011 &[t.clone()],
2012 &[],
2013 2,
2014 )
2015 .await;
2016 assert!(
2017 !turn(3).is_file(),
2018 "the same stalled task lingering must not call the conductor every cycle"
2019 );
2020 }
2021
2022 #[test]
2023 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
2024 let dir = tempdir().unwrap();
2025 let queue = Queue::at(dir.path().join("queue"));
2026 let mut t = task("t");
2027 queue.put(&mut t).unwrap();
2028
2029 let mut conductor = Conductor::new();
2030 assert!(
2031 conductor.worth_a_look(&queue, &[], &[]),
2032 "a conductor that has never run has something to look at"
2033 );
2034
2035 conductor.last_seen = Some(Conductor::snapshot(&queue, &[], &[]));
2036 assert!(
2037 !conductor.worth_a_look(&queue, &[], &[]),
2038 "nothing changed and nothing is stalled or finished"
2039 );
2040 assert!(
2041 conductor.worth_a_look(&queue, &[t.clone()], &[]),
2042 "a stalled task is worth a look even at the same revision"
2043 );
2044 assert!(
2045 conductor.worth_a_look(&queue, &[], &[t.clone()]),
2046 "a finished task is worth a look even at the same revision"
2047 );
2048 }
2049
2050 #[tokio::test]
2051 async fn the_conduct_path_never_calls_ask_and_wait() {
2052 let dir = tempdir().unwrap();
2058 let queue = Queue::at(dir.path().join("queue"));
2059 let questions = Questions::at(dir.path().join("questions"));
2060 let mut t = task("asks without blocking");
2061 queue.put(&mut t).unwrap();
2062
2063 apply(
2064 &queue,
2065 &questions,
2066 &Verdict {
2067 decisions: vec![Decision {
2068 id: t.id.clone(),
2069 question: Some("ok?".to_owned()),
2070 ..Decision::default()
2071 }],
2072 },
2073 )
2074 .unwrap();
2075 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2077 }
2078}