1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{ChoiceAction, Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104 Recover,
106 Resume,
108 Repair,
110 Complete,
112 Inspect,
114 Configure,
116}
117
118#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123 Task,
125 Run {
127 id: String,
129 },
130 Setting {
133 path: String,
135 keys: Vec<String>,
137 },
138 Source {
140 reference: String,
142 },
143}
144
145#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148 pub operation: QuestionOperation,
150 pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155 fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156 let target = match &self.target {
157 QuestionTarget::Task => format!("task:{}", task.id),
158 QuestionTarget::Run { id } => {
159 anyhow::ensure!(
160 task.runs.contains(id),
161 "question target is not this task's run"
162 );
163 format!("run:{id}")
164 }
165 QuestionTarget::Setting { path, keys } => {
166 use std::path::Component;
167 anyhow::ensure!(
168 !path.is_empty()
169 && Path::new(path)
170 .components()
171 .all(|c| matches!(c, Component::Normal(_)))
172 && !keys.is_empty()
173 && keys.iter().all(|key| !key.is_empty()),
174 "question setting must name a repository file and concrete keys"
175 );
176 let root = task.repo.canonicalize()?;
177 let file = root.join(path).canonicalize()?;
178 let relative = file
179 .strip_prefix(&root)
180 .context("question setting is outside the repository")?;
181 let content = std::fs::read_to_string(&file)?;
182 let value: toml::Value = toml::from_str(&content)?;
183 let mut setting = &value;
184 for key in keys {
185 setting = setting
186 .get(key)
187 .context("question setting key does not exist")?;
188 }
189 format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190 }
191 QuestionTarget::Source { reference } => {
192 use crate::proc::Quiet as _;
193 let output = std::process::Command::new("git")
194 .args([
195 "rev-parse",
196 "--verify",
197 "--end-of-options",
198 &format!("{reference}^{{commit}}"),
199 ])
200 .current_dir(&task.repo)
201 .quiet()
202 .output()
203 .context("resolve question source")?;
204 anyhow::ensure!(
205 output.status.success(),
206 "question source cannot be resolved"
207 );
208 let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209 anyhow::ensure!(
210 !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211 "invalid question source commit"
212 );
213 format!("source:{commit}")
214 }
215 };
216 Ok((self.operation, target))
217 }
218}
219
220#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224 pub id: String,
226 #[serde(default)]
230 pub blocked_by: Vec<String>,
231 #[serde(default)]
236 pub missing_dependencies: Vec<String>,
237 #[serde(default)]
239 pub reason: Option<String>,
240 #[serde(default)]
247 pub recovery: Option<Recovery>,
248 #[serde(default)]
254 pub question: Option<String>,
255 #[serde(default)]
257 pub choices: Vec<String>,
258 #[serde(default)]
260 pub actions: BTreeMap<String, ChoiceAction>,
261 #[serde(default)]
264 pub question_decision: Option<QuestionDecision>,
265 #[serde(default)]
267 pub evidence: Vec<String>,
268}
269
270#[derive(Debug, Clone, Default, Deserialize)]
283pub struct Verdict {
284 pub decisions: Vec<Decision>,
287}
288
289fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
292 prompt::ConductTask {
293 id: t.id.clone(),
294 title: t.title.clone(),
295 instruction: t.instruction.clone(),
296 repo: t.repo.display().to_string(),
297 priority: t.priority,
298 status: t.status.as_str().to_owned(),
299 attempts: t.attempts,
300 max_attempts: t.effective_attempt_limit(max_attempts),
301 last_error: t.last_error.clone(),
302 hold_reason: t.hold_reason.clone(),
303 held_at: t.held_at.map(|at| at.to_string()),
304 hold_source: t.hold_source.map(|source| source.label().to_owned()),
305 blocked_by: t.blocked_by.clone(),
306 answers: t
307 .answers
308 .iter()
309 .map(|a| prompt::ConductAnswer {
310 question: a.question.clone(),
311 answer: a.answer.clone(),
312 })
313 .collect(),
314 operator_resume: t.resumption.as_ref().map(|r| format!(
315 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
316 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
317 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
318 wording alone is insufficient.",
319 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
320 )).or_else(|| t.resume_override.as_ref().map(|o| {
321 format!(
322 "the operator explicitly answered \"resume\" at {}; do not hold this \
323 task again for the same reason unless there is new information",
324 o.at
325 )
326 })),
327 }
328}
329
330fn pinned_resume(task: &Task) -> bool {
334 task.resume_override
335 .as_ref()
336 .is_some_and(|o| o.pinned_run.is_some())
337}
338
339fn new_blocking_evidence(
342 queue: &Queue,
343 questions: &Questions,
344 task: &Task,
345 d: &Decision,
346) -> Vec<String> {
347 let Some(r) = &task.resumption else {
348 return Vec::new();
349 };
350 d.evidence
351 .iter()
352 .filter_map(|id| {
353 let version = if let Ok(q) = questions.get(id) {
354 (q.asked_at > r.at
355 && (q.run == task.id || task.runs.contains(&q.run))
356 && q.node != NODE
357 && q.status != crate::ask::QuestionStatus::Abandoned)
358 .then(|| {
359 format!(
360 "question:{id}:{}:{:?}:{}",
361 q.asked_at,
362 q.status,
363 q.resolution().unwrap_or_default()
364 )
365 })
366 } else if d.blocked_by.contains(id) {
367 queue
368 .get(id)
369 .ok()
370 .filter(|dep| {
371 dep.id != task.id
372 && dep.status != TaskStatus::Done
373 && (dep.created_at > r.at
374 || dep.held_at.is_some_and(|at| at > r.at)
375 || dep.operator_decided_at.is_some_and(|at| at > r.at)
376 || dep.passes.iter().any(|p| p.settled_at > r.at)
377 || (dep.status == TaskStatus::Blocked
378 && dep.blocked_at.is_some_and(|at| at > r.at)))
379 })
380 .map(|dep| {
381 format!(
382 "dependency:{id}:{:?}:{:?}:{:?}",
383 dep.status,
384 dep.chat_notice_key(),
385 dep.hold_notice_cause
386 )
387 })
388 } else if task.runs.iter().any(|run| run == id) {
389 RunState::load(id)
390 .ok()
391 .filter(|run| {
392 run.updated_at > r.at
395 && run
396 .passes
397 .last()
398 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
399 && matches!(
400 run.status,
401 crate::run::RunStatus::Failed
402 | crate::run::RunStatus::Blocked
403 | crate::run::RunStatus::Stalled
404 )
405 })
406 .map(|run| {
407 format!(
408 "run:{id}:{:?}:{:?}",
409 run.status,
410 run.passes.last().map(|p| p.index)
411 )
412 })
413 } else {
414 None
415 };
416 version.filter(|v| !r.blocking_evidence.contains(v))
417 })
418 .collect()
419}
420
421fn may_hold(task: &mut Task, reason: &str) -> bool {
425 if pinned_resume(task) {
426 return false;
427 }
428 if task.resumption.is_some() {
429 return true; }
431 let Some(o) = task.resume_override.as_mut() else {
432 return true;
433 };
434 if o.forced || o.pinned_run.is_some() {
435 tracing::warn!(
436 "conductor tried to hold task {} after the operator forced a resume: {reason}",
437 task.id
438 );
439 return false;
440 }
441 if o.conductor_rehold.is_some() {
442 return false;
443 }
444 o.conductor_rehold = Some(reason.to_owned());
445 true
446}
447
448fn severity_str(s: crate::verdict::Severity) -> &'static str {
451 match s {
452 crate::verdict::Severity::Nit => "nit",
453 crate::verdict::Severity::Minor => "minor",
454 crate::verdict::Severity::Major => "major",
455 crate::verdict::Severity::Blocker => "blocker",
456 }
457}
458
459fn surviving_branch(task: &Task) -> Option<String> {
464 let last = task.runs.last()?;
465 let state = RunState::load(last).ok()?;
466 if !crate::daemon::compatible_run_instruction(&state, task) {
467 return None;
468 }
469 state.winner().map(|c| c.branch.clone())
470}
471
472fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
499 let note = match &d.reason {
500 Some(reason) => reason.clone(),
501 None => match task.answers.last() {
502 Some(a) => format!(
503 "conduct held this again with no new reason given; last operator \
504 answer on record: {}",
505 a.answer
506 ),
507 None => "conduct held this again with no reason given".to_owned(),
508 },
509 };
510 match task.hold_reason.as_deref() {
511 Some(prior) if !prior.is_empty() => {
512 let outer = outermost_hold_note(prior);
513 if note.trim() == prior.trim() || note.trim() == outer {
514 None
515 } else {
516 Some(format!("{note}\n\n(previously: {outer})"))
517 }
518 }
519 _ => Some(note),
520 }
521}
522
523const PREVIOUSLY: &str = "\n\n(previously: ";
526
527fn outermost_hold_note(reason: &str) -> &str {
530 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
531}
532
533fn hold_note(d: &Decision) -> String {
535 d.reason
536 .clone()
537 .unwrap_or_else(|| "(no reason given)".to_owned())
538}
539
540async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
542 let Some(run_id) = task.runs.last().cloned() else {
543 return prompt::ConductOutcome {
544 run_id: "(none)".to_owned(),
545 unreadable: Some("this task has not produced a run yet".to_owned()),
546 run_status: None,
547 open_findings: Vec::new(),
548 rounds_used: 0,
549 rounds_max: 0,
550 rounds: Vec::new(),
551 branch: None,
552 branch_head: None,
553 references: None,
554 empty_candidate: false,
555 };
556 };
557 let state = match RunState::load(&run_id) {
558 Ok(s) => s,
559 Err(e) => {
560 tracing::warn!(
565 "conductor: could not read run {run_id} for task {}: {e:#}",
566 task.short()
567 );
568 return prompt::ConductOutcome {
569 run_id,
570 unreadable: Some(format!("{e:#}")),
571 run_status: None,
572 open_findings: Vec::new(),
573 rounds_used: 0,
574 rounds_max: 0,
575 rounds: Vec::new(),
576 branch: None,
577 branch_head: None,
578 references: None,
579 empty_candidate: false,
580 };
581 }
582 };
583
584 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
585 id: f.id.clone(),
586 title: f.title.clone(),
587 severity: severity_str(f.severity).to_owned(),
588 };
589 let open_findings = state
590 .open_findings()
591 .into_iter()
592 .map(finding_view)
593 .collect();
594 let rounds = state
595 .reviews
596 .iter()
597 .map(|r| prompt::ConductRound {
598 round: r.round,
599 findings: r
600 .reviews
601 .iter()
602 .flat_map(|rec| rec.findings.iter())
603 .map(finding_view)
604 .collect(),
605 addressed: r
606 .fix
607 .as_ref()
608 .map(|fx| fx.addressed.clone())
609 .unwrap_or_default(),
610 rejected: r
611 .fix
612 .as_ref()
613 .map(|fx| {
614 fx.rejected
615 .iter()
616 .map(|rej| prompt::ConductRejection {
617 id: rej.id.clone(),
618 why: rej.why.clone(),
619 })
620 .collect()
621 })
622 .unwrap_or_default(),
623 })
624 .collect();
625 let branch = state.winner().map(|c| c.branch.clone());
626 let branch_head = match &branch {
627 Some(b) => crate::git::rev_parse(repo, b)
628 .await
629 .ok()
630 .map(|h| h.chars().take(8).collect()),
631 None => None,
632 };
633
634 prompt::ConductOutcome {
635 run_id,
636 unreadable: None,
637 run_status: Some(state.status.as_str().to_owned()),
638 open_findings,
639 rounds_used: state.reviews.len(),
640 rounds_max: state.config.graph.review_rounds,
641 rounds,
642 branch,
643 branch_head,
644 references: None,
645 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
646 }
647}
648
649async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
653 let id = task.runs.last()?;
654 let state = match RunState::load(id) {
655 Ok(s) => s,
656 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
657 };
658 let branch = state.winner()?.branch.clone();
659 let base = state.base_branch.clone();
660 match crate::land::find_open_pr(repo, &branch, &base).await {
661 Ok(crate::land::OpenPr::None) => None,
662 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
663 "pull request {url} is already open for {branch} into {base}"
664 )),
665 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
666 "several pull requests are already open for {branch} into {base}: {}",
667 urls.join(" ")
668 )),
669 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
670 }
671}
672
673async fn attach_facts(_cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
680 for d in verdict
681 .decisions
682 .iter_mut()
683 .filter(|d| d.question.is_some())
684 {
685 let Ok(task) = queue.get(&d.id) else { continue };
686 if let (Some(fact), Some(q)) = (
687 open_pr_fact(&task, &repo_for(&task, repo)).await,
688 d.question.as_mut(),
689 ) {
690 q.push_str(&format!("\n\nChecked against the repository: {fact}"));
691 }
692 }
693}
694
695async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
697 prompt::ConductFinished {
698 task: view(t, max_attempts),
699 outcome: outcome_for(t, &repo_for(t, repo)).await,
700 }
701}
702
703fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
706 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
707 fallback.to_path_buf()
708 } else {
709 task.repo.clone()
710 }
711}
712
713fn apply_one(
730 queue: &Queue,
731 questions: &Questions,
732 d: &Decision,
733 snapshot: Option<&Task>,
734) -> Result<()> {
735 let _claim = queue
736 .claim(&d.id)
737 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
738 let mut task = queue.get(&d.id).context("no such task")?;
739
740 if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
743 return Ok(());
744 }
745 let blocking =
746 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
747 let evidence = new_blocking_evidence(queue, questions, &task, d);
748 if blocking && task.resumption.is_some() && evidence.is_empty() {
749 crate::notices::raise(crate::notices::conductor_resumption(
750 &task.id,
751 task.short(),
752 d.reason.as_deref(),
753 &d.blocked_by,
754 &d.evidence,
755 d.question.is_some(),
756 ));
757 return Ok(());
758 }
759
760 if blocking && !evidence.is_empty() {
761 let mut cause = evidence.clone();
762 cause.sort();
763 cause.dedup();
764 task.notice_cause(
765 &format!("conductor-evidence:{}", cause.join(",")),
766 None,
767 None,
768 );
769 }
770 if blocking {
771 if let Some(r) = task.resumption.as_mut() {
772 r.blocking_evidence.extend(evidence);
773 }
774 }
775
776 if task.operator_held() {
780 return Ok(());
781 }
782
783 if crate::base_sync_recovery::held_task(queue, &task)
784 && (d.recovery == Some(Recovery::Requeue)
785 || d.question_decision
786 .as_ref()
787 .is_some_and(|q| q.operation == QuestionOperation::Resume)
788 || d.actions
789 .values()
790 .any(|a| matches!(a, ChoiceAction::Resume { .. })))
791 {
792 return Ok(());
794 }
795
796 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
800 return Ok(());
801 }
802
803 let pinned = pinned_resume(&task);
806 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
807 return Ok(());
808 }
809
810 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
814 && !d.missing_dependencies.is_empty()
815 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
816 && d.missing_dependencies
817 .iter()
818 .any(|id| !d.blocked_by.contains(id))
819 && d.missing_dependencies
820 .iter()
821 .chain(&d.blocked_by)
822 .any(|id| {
823 matches!(
824 crate::queue::dependency(queue, questions, id),
825 crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
826 )
827 })
828 {
829 return Ok(());
830 }
831
832 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
838 && !d.missing_dependencies.is_empty()
839 && d.missing_dependencies.iter().all(|id| {
840 let claim = crate::queue::dependency(queue, questions, id);
841 claim != crate::queue::Dependency::Missing
842 && d.blocked_by.iter().any(|blocker| {
843 blocker == id
844 || (matches!(
845 &claim,
846 crate::queue::Dependency::Task(_)
847 | crate::queue::Dependency::Question(_)
848 | crate::queue::Dependency::Deleted(_)
849 ) && crate::queue::dependency(queue, questions, blocker) == claim)
850 })
851 })
852 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
853 && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
854 {
855 let mut ids = d.blocked_by.clone();
856 crate::queue::normalize_blockers(queue, questions, &mut ids);
857 let reason = format!("Waiting on dependencies: {}", ids.join(", "));
858 task.block(ids, Some(reason));
859 return queue.put(&mut task);
860 }
861
862 if let Some(text) = &d.question {
863 if task.status == TaskStatus::Done {
864 return Ok(());
865 }
866 let mut held = task.clone();
867 held.status = TaskStatus::Held;
868 let targets: BTreeSet<_> = d
869 .blocked_by
870 .iter()
871 .filter_map(|id| {
872 if let Ok(dep) = queue.get(id) {
873 Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
874 } else if questions.get(id).is_ok() {
875 Some(format!("question:{id}"))
876 } else {
877 None
878 }
879 })
880 .collect();
881 let decision = d
882 .question_decision
883 .as_ref()
884 .and_then(|q| match q.identity(&task) {
885 Ok(identity) => Some(identity),
886 Err(error) => {
887 tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
890 None
891 }
892 });
893 let context_value =
894 serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
895 let context = serde_json::to_string(&context_value)?;
896 if questions.list().iter().any(|q| {
900 q.node == NODE
901 && q.run == task.id
902 && q.status == crate::ask::QuestionStatus::Answered
903 && q.decision_context.as_ref().is_some_and(|stored| {
904 if stored == &context {
905 return true;
906 }
907 decision.is_none()
910 && serde_json::from_str::<serde_json::Value>(stored)
911 .ok()
912 .and_then(|v| {
913 v.as_array().map(|items| {
914 items.len() == 4
915 && items[..3]
916 == context_value.as_array().expect("context")[..3]
917 })
918 })
919 == Some(true)
920 })
921 }) {
922 return Ok(());
923 }
924 let question_id = match questions
928 .list()
929 .into_iter()
930 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
931 {
932 Some(existing) => existing.id,
933 None => {
934 let mut q = Question::new(
935 task.id.clone(),
936 NODE.to_owned(),
937 SEAT.to_owned(),
938 text.clone(),
939 d.reason.clone().unwrap_or_default(),
940 d.choices.clone(),
941 );
942 q.actions = d.actions.clone();
943 crate::choice_effect::executable_conductor(&mut q);
944 q.decision_context = Some(context);
945 q.deputy = Some(Deputy::new(crate::deputy::brief(
950 &task.id,
951 d.reason.as_deref().unwrap_or_default(),
952 &q.choices,
953 &q.actions,
954 )));
955 if task.repo.is_dir() && task.repo != Path::new(".") {
956 q.cwd = Some(task.repo.to_string_lossy().into_owned());
957 }
958 questions.put_executable(&mut q)?;
959 q.id
960 }
961 };
962 task.block(vec![question_id], d.reason.clone());
963 return queue.put(&mut task);
964 }
965
966 match task.status {
967 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
971 if may_hold(&mut task, &hold_note(d)) {
972 task.hold_machine(d.reason.clone());
973 queue.put(&mut task)?;
974 }
975 }
976 TaskStatus::Queued if !d.blocked_by.is_empty() => {
977 let mut ids = d.blocked_by.clone();
978 crate::queue::normalize_blockers(queue, questions, &mut ids);
979 task.block(ids, d.reason.clone());
980 queue.put(&mut task)?;
981 }
982 TaskStatus::Running => match d.recovery {
983 Some(Recovery::Requeue) if pinned_resume(&task) => {}
984 Some(Recovery::Requeue) => {
985 task.requeue();
986 queue.put(&mut task)?;
987 }
988 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
989 task.hold_machine(d.reason.clone());
990 queue.put(&mut task)?;
991 }
992 _ => {}
996 },
997 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
998 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1001 Some(Recovery::Requeue) => {
1002 task.requeue();
1003 queue.put(&mut task)?;
1004 }
1005 Some(Recovery::Hold) => {
1006 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1010 if may_hold(&mut task, &hold_note(d)) {
1011 task.hold_machine(Some(reason));
1012 queue.put(&mut task)?;
1013 }
1014 }
1015 }
1016 Some(Recovery::Review) => {
1017 if let Some(branch) = surviving_branch(&task) {
1018 task.request_review(branch);
1019 queue.put(&mut task)?;
1020 }
1021 }
1027 Some(Recovery::Done) => {
1028 task.succeed();
1029 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1033 queue.put(&mut task)?;
1034 }
1035 None => {}
1036 },
1037 _ => {}
1040 }
1041 Ok(())
1042}
1043
1044pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1048 for d in &verdict.decisions {
1049 if let Err(e) = apply_one(queue, questions, d, None) {
1050 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1051 }
1052 }
1053 Ok(())
1054}
1055
1056pub fn seat_path(home: &Path) -> PathBuf {
1062 home.join("conduct").join("seat.json")
1063}
1064
1065pub fn load_seat(home: &Path) -> Option<SeatState> {
1067 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1068}
1069
1070fn busy_path(home: &Path) -> PathBuf {
1071 home.join("conduct").join("busy")
1072}
1073
1074pub fn busy(home: &Path) -> bool {
1080 std::fs::metadata(busy_path(home))
1081 .and_then(|m| m.modified())
1082 .ok()
1083 .and_then(|t| t.elapsed().ok())
1084 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1085}
1086
1087struct Busy(PathBuf);
1089
1090impl Busy {
1091 fn mark(home: &Path) -> Self {
1092 let path = busy_path(home);
1093 if let Some(dir) = path.parent() {
1094 let _ = std::fs::create_dir_all(dir);
1095 }
1096 let _ = std::fs::write(&path, std::process::id().to_string());
1097 Self(path)
1098 }
1099}
1100
1101impl Drop for Busy {
1102 fn drop(&mut self) {
1103 let _ = std::fs::remove_file(&self.0);
1104 }
1105}
1106
1107#[derive(Debug, Default, Serialize, Deserialize)]
1110pub struct Conductor {
1111 seat: Option<SeatState>,
1112 last_seen: Option<(u64, BTreeSet<String>)>,
1113 considered: BTreeMap<String, String>,
1117}
1118
1119fn decision_input(task: &Task) -> serde_json::Value {
1124 let mut v = serde_json::to_value(task).unwrap_or_default();
1125 if let Some(o) = v.as_object_mut() {
1126 o.remove("chat_notice");
1127 o.remove("chat_notices");
1128 o.remove("updated_at");
1130 }
1131 v
1132}
1133
1134fn input_fingerprint(task: &Task) -> String {
1135 let mut v = decision_input(task);
1136 if let Some(o) = v.as_object_mut() {
1137 o.remove("hold_reason");
1138 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1139 ro.remove("conductor_rehold");
1140 }
1141 }
1142 v.to_string()
1143}
1144
1145impl Conductor {
1146 #[must_use]
1148 pub fn new() -> Self {
1149 Self::default()
1150 }
1151
1152 fn settled(
1157 considered: &BTreeMap<String, String>,
1158 stalled: &[Task],
1159 finished: &[Task],
1160 ) -> BTreeSet<String> {
1161 stalled
1162 .iter()
1163 .chain(finished)
1164 .filter(|t| t.status == TaskStatus::Held)
1165 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1166 .map(|t| t.id.clone())
1167 .collect()
1168 }
1169
1170 fn snapshot_with(
1171 considered: &BTreeMap<String, String>,
1172 queue: &Queue,
1173 stalled: &[Task],
1174 finished: &[Task],
1175 ) -> (u64, BTreeSet<String>) {
1176 let ids = stalled
1177 .iter()
1178 .chain(finished)
1179 .map(|t| t.id.clone())
1180 .collect();
1181 let skip = Self::settled(considered, stalled, finished);
1182 (queue.revision_excluding(&skip), ids)
1183 }
1184
1185 fn snapshot(
1186 &self,
1187 queue: &Queue,
1188 stalled: &[Task],
1189 finished: &[Task],
1190 ) -> (u64, BTreeSet<String>) {
1191 Self::snapshot_with(&self.considered, queue, stalled, finished)
1192 }
1193
1194 #[must_use]
1211 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1212 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1213 }
1214
1215 #[allow(clippy::too_many_arguments)]
1219 pub async fn maybe_run(
1220 &mut self,
1221 cfg: &Config,
1222 repo: &Path,
1223 queue: &Queue,
1224 questions: &Questions,
1225 home: &Path,
1226 queued: &[Task],
1227 stalled: &[Task],
1228 finished: &[Task],
1229 max_attempts: usize,
1230 ) {
1231 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1232 return;
1233 }
1234 self.considered = stalled
1237 .iter()
1238 .chain(finished)
1239 .filter(|t| t.status == TaskStatus::Held)
1240 .map(|t| (t.id.clone(), input_fingerprint(t)))
1241 .collect();
1242 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1243 if let Err(e) = self
1244 .run_once(
1245 cfg,
1246 repo,
1247 queue,
1248 questions,
1249 home,
1250 queued,
1251 stalled,
1252 finished,
1253 max_attempts,
1254 )
1255 .await
1256 {
1257 tracing::warn!("conductor: {e:#}");
1263 }
1264 }
1265
1266 #[allow(clippy::too_many_arguments)]
1267 async fn run_once(
1268 &mut self,
1269 cfg: &Config,
1270 repo: &Path,
1271 queue: &Queue,
1272 questions: &Questions,
1273 home: &Path,
1274 queued: &[Task],
1275 stalled: &[Task],
1276 finished: &[Task],
1277 max_attempts: usize,
1278 ) -> Result<()> {
1279 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1280 return Ok(());
1281 }
1282
1283 let primary = cfg
1284 .resolve_roles()
1285 .context("resolving the conductor seat")?
1286 .conductor;
1287 let chain = match cfg.roles.conductor.as_ref() {
1291 Some(c) if c.ids().len() > 1 => {
1292 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1293 }
1294 _ => vec![primary],
1295 };
1296
1297 let runnable_views: Vec<prompt::ConductTask> =
1298 queued.iter().map(|t| view(t, max_attempts)).collect();
1299 let stalled_views: Vec<prompt::ConductTask> =
1300 stalled.iter().map(|t| view(t, max_attempts)).collect();
1301 let mut finished_views = Vec::with_capacity(finished.len());
1302 for t in finished {
1303 finished_views.push(finished_view(t, repo, max_attempts).await);
1304 }
1305
1306 let body = prompt::with_overlay(
1307 prompt::conduct(
1308 &runnable_views,
1309 &stalled_views,
1310 &finished_views,
1311 &cfg.graph.language,
1312 ),
1313 cfg.prompts.overlay(NODE),
1314 );
1315
1316 let body = format!(
1317 "{body}\n\n{}",
1318 crate::queue::dependency_facts(queue, questions)
1319 );
1320 let artifacts = home.join("conduct").join("artifacts");
1321 let cache_dir = cfg.cache_dir();
1324
1325 let busy = Busy::mark(home);
1327 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1328 for spec in &chain {
1329 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1332 if needs_new_seat {
1333 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1334 }
1335 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1336 let stem = if std::ptr::eq(spec, &chain[0]) {
1339 format!("turn-{}", seat.turns + 1)
1340 } else {
1341 format!("turn-{}-{}", seat.turns + 1, spec.id)
1342 };
1343 let inv = Invocation {
1344 coordination: cfg.coordination.as_ref(),
1345 fetched_base: None,
1346 cwd: repo,
1347 prompt: &body,
1348 timeout: TURN_TIMEOUT,
1349 allow_write: false,
1352 sessions: cfg.graph.sessions,
1353 artifacts: &artifacts,
1354 stem: &stem,
1355 run: NODE,
1356 node: NODE,
1357 cache_dir: cache_dir.as_deref(),
1358 attachments: &[],
1359 writable: &[],
1360 };
1361 let out = agent::invoke(spec, seat, &inv).await;
1362 if let Ok(body) = serde_json::to_string(&*seat) {
1365 let path = seat_path(home);
1366 if let Some(dir) = path.parent() {
1367 let _ = std::fs::create_dir_all(dir);
1368 }
1369 let _ = std::fs::write(path, body);
1370 }
1371 let advance = agent::chain_advances(&out);
1372 result = match out {
1373 Err(e) => Err(e.context("invoking the conductor")),
1374 Ok(out) if advance => Err(anyhow::anyhow!(
1375 "no usable reply ({})",
1376 crate::notices::turn_status(
1377 out.exit_code,
1378 out.timed_out,
1379 out.quota_exhausted()
1380 )
1381 )),
1382 Ok(out) => verdict::extract_json(&out.text)
1383 .context("the conductor's reply could not be parsed"),
1384 };
1385 if result.is_ok() {
1386 break;
1387 }
1388 if chain.len() > 1 {
1389 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1390 }
1391 }
1392 drop(busy);
1393 let mut verdict = result?;
1394 attach_facts(cfg, repo, queue, &mut verdict).await;
1395 for d in &verdict.decisions {
1396 let snapshot = queued
1397 .iter()
1398 .chain(stalled)
1399 .chain(finished)
1400 .find(|t| t.id == d.id);
1401 if let Some(snapshot) = snapshot {
1402 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1403 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1404 }
1405 }
1406 }
1407 Ok(())
1408 }
1409}
1410
1411#[cfg(test)]
1412mod tests {
1413 use std::collections::BTreeMap;
1414
1415 use tempfile::tempdir;
1416
1417 use super::*;
1418 use crate::ask::{Answer, QuestionStatus};
1419 use crate::config::{AgentKind, AgentSpec, Graph};
1420 use crate::queue::Source;
1421
1422 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1423 let path = dir.join("mock-conduct-agent.sh");
1424 std::fs::write(&path, script).expect("write mock");
1425 AgentSpec {
1426 id: "mock".to_owned(),
1427 kind: AgentKind::Command,
1428 model: None,
1429 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1430 extra_args: Vec::new(),
1431 env,
1432 prompt_delivery: None,
1433 max_turn_secs: None,
1434 }
1435 }
1436
1437 fn config(spec: AgentSpec) -> Config {
1438 Config {
1439 agents: vec![spec],
1440 graph: Graph {
1441 language: "en".to_owned(),
1442 ..Graph::default()
1443 },
1444 ..Config::default()
1445 }
1446 }
1447
1448 fn task(title: &str) -> Task {
1449 Task::new(
1450 title.to_owned(),
1451 format!("do {title}"),
1452 std::path::PathBuf::from("."),
1453 Source::Human,
1454 )
1455 }
1456
1457 #[test]
1458 fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1459 let mut t = task("waiting");
1460 t.hold_machine(Some("waiting for recovery".into()));
1461 let before = input_fingerprint(&t);
1462 let snapshot = decision_input(&t);
1463 t.chat_notice = Some(crate::queue::ChatNotice {
1464 key: t.chat_notice_key().unwrap(),
1465 at: jiff::Timestamp::now(),
1466 outcome: crate::queue::ChatNoticeOutcome::Posted,
1467 });
1468 t.chat_notices.push(t.chat_notice_key().unwrap());
1469 t.updated_at = jiff::Timestamp::now();
1470 assert_eq!(before, input_fingerprint(&t));
1471 assert_eq!(snapshot, decision_input(&t));
1472 t.hold_reason = Some("recovery remains under consideration".into());
1473 assert_eq!(before, input_fingerprint(&t));
1474 t.record_answer("a different question".into(), "a new decision".into());
1475 assert_ne!(before, input_fingerprint(&t));
1476 }
1477
1478 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1479 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1480
1481 fn env(reply: &str) -> BTreeMap<String, String> {
1482 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1483 }
1484
1485 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1486
1487 fn init_repo_with_branch(dir: &Path, branch: &str) {
1491 use crate::proc::Quiet as _;
1492 let run = |args: &[&str]| {
1493 let out = std::process::Command::new("git")
1494 .args(args)
1495 .current_dir(dir)
1496 .quiet()
1497 .output()
1498 .expect("spawn git");
1499 assert!(
1500 out.status.success(),
1501 "git {args:?} failed: {}",
1502 String::from_utf8_lossy(&out.stderr)
1503 );
1504 };
1505 run(&["init", "-b", "main"]);
1506 run(&["config", "user.name", "magi test"]);
1507 run(&["config", "user.email", "magi@example.com"]);
1508 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1509 run(&["add", "-A"]);
1510 run(&["commit", "-m", "init"]);
1511 run(&["checkout", "-b", branch]);
1512 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1513 run(&["add", "-A"]);
1514 run(&["commit", "-m", "candidate work"]);
1515 }
1516
1517 fn review_round_with_finding(
1518 round: usize,
1519 finding_id: &str,
1520 title: &str,
1521 addressed: &[&str],
1522 rejected: &[(&str, &str)],
1523 ) -> crate::run::ReviewRound {
1524 crate::run::ReviewRound {
1525 round,
1526 head: "deadbeef".to_owned(),
1527 verified_head: None,
1528 verified_at: None,
1529 reviews: vec![crate::run::ReviewRecord {
1530 attempts: 0,
1531 reviewer: 1,
1532 agent: "mock".to_owned(),
1533 summary: String::new(),
1534 findings: vec![crate::verdict::Finding {
1535 id: finding_id.to_owned(),
1536 severity: crate::verdict::Severity::Major,
1537 file: None,
1538 line: None,
1539 title: title.to_owned(),
1540 detail: String::new(),
1541 }],
1542 vote: None,
1543 failed: None,
1544 duration_ms: 0,
1545 }],
1546 e2e: Vec::new(),
1547 verify_retried: false,
1548 e2e_deferred: false,
1549 e2e_defer_reason: None,
1550 fix: Some(crate::run::FixRecord {
1551 agent: "mock".to_owned(),
1552 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1553 rejected: rejected
1554 .iter()
1555 .map(|(id, why)| crate::verdict::Rejection {
1556 id: (*id).to_owned(),
1557 why: (*why).to_owned(),
1558 })
1559 .collect(),
1560 notes: String::new(),
1561 committed: false,
1562 failed: None,
1563 duration_ms: 0,
1564 continuation: None,
1565 }),
1566 blocking: 1,
1567 answered: 1,
1568 expected: 1,
1569 clean: false,
1570 progressed: true,
1571 vote_split: false,
1572 reconsideration: Vec::new(),
1573 verdict: None,
1574 }
1575 }
1576
1577 #[test]
1578 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1579 crate::run::pin_test_home();
1580 let dir = tempdir().unwrap();
1581 let queue = Queue::at(dir.path().join("queue"));
1582 let questions = Questions::at(dir.path().join("questions"));
1583 let mut t = task("recover");
1584 t.answers.push(crate::queue::AnsweredQuestion {
1585 question: "Inspect remote first?".into(),
1586 answer: "Inspect, then await resumption instructions".into(),
1587 });
1588 t.hold_machine(Some("waiting for remote inspection".into()));
1589 queue.put(&mut t).unwrap();
1590 let snapshot = queue.get(&t.id).unwrap();
1591 t.edit(
1592 "recover".into(),
1593 "Reuse the measured PR head; fix Clippy; review".into(),
1594 )
1595 .unwrap();
1596 t.release_operator("cli", None);
1597 queue.put(&mut t).unwrap();
1598 let approval = t.resumption.clone();
1599 let d = Decision {
1600 id: t.id.clone(),
1601 recovery: Some(Recovery::Hold),
1602 reason: Some("remote inspection still required".into()),
1603 ..Decision::default()
1604 };
1605 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1606 let stale_question = Decision {
1607 question: Some("Inspect remote?".into()),
1608 ..d.clone()
1609 };
1610 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1611 assert!(questions.list().is_empty());
1612 for reason in [
1613 "waiting for remote inspection",
1614 "inspection has not been confirmed",
1615 ] {
1616 let repeated = Decision {
1617 reason: Some(reason.into()),
1618 question: Some("How should we proceed?".into()),
1619 ..d.clone()
1620 };
1621 apply(
1622 &queue,
1623 &questions,
1624 &Verdict {
1625 decisions: vec![repeated],
1626 },
1627 )
1628 .unwrap();
1629 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1630 }
1631 assert!(
1632 questions.list().is_empty(),
1633 "a rejected proposal registers no question"
1634 );
1635 let notice = crate::notices::Notices::open()
1636 .get(&crate::notices::id_of(&format!(
1637 "conductor-resumption:{}",
1638 t.id
1639 )))
1640 .unwrap();
1641 assert!(!notice.message.contains("How should we proceed?"));
1642 assert!(!notice.message.contains("Some("));
1643 assert!(notice.message.contains("No reply to this notice is needed"));
1644 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1645 assert_eq!(reloaded.resumption, approval);
1646 assert_eq!(reloaded.answers, t.answers);
1647 let v = view(&reloaded, 3);
1648 assert!(v.operator_resume.unwrap().contains("superseded"));
1649 let mut dep = task("new prerequisite");
1650 dep.hold_machine(Some("new authentication failure".into()));
1651 queue.put(&mut dep).unwrap();
1652 let new_blocker = Decision {
1653 blocked_by: vec![dep.id.clone()],
1654 evidence: vec![dep.id.clone()],
1655 recovery: None,
1657 reason: Some("new prerequisite failed".into()),
1658 ..d.clone()
1659 };
1660 apply(
1661 &queue,
1662 &questions,
1663 &Verdict {
1664 decisions: vec![new_blocker.clone()],
1665 },
1666 )
1667 .unwrap();
1668 let mut after = queue.get(&t.id).unwrap();
1669 assert_eq!(after.status, TaskStatus::Blocked);
1670 after.release();
1673 queue.put(&mut after).unwrap();
1674 apply(
1675 &queue,
1676 &questions,
1677 &Verdict {
1678 decisions: vec![new_blocker],
1679 },
1680 )
1681 .unwrap();
1682 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1683 after.start("new-run".into());
1684 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1685 after.hold_manual(Some("operator says wait".into()));
1686 queue.put(&mut after).unwrap();
1687 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1688 assert!(queue.get(&t.id).unwrap().operator_held());
1689 }
1690
1691 #[test]
1692 fn newly_blocked_dependency_is_material_evidence_after_approval() {
1693 let dir = tempdir().unwrap();
1694 let queue = Queue::at(dir.path().join("queue"));
1695 let questions = Questions::at(dir.path().join("questions"));
1696 let mut dep = task("old queued dependency");
1697 queue.put(&mut dep).unwrap();
1698 let mut t = task("approved task");
1699 t.release_operator("test", None);
1700 queue.put(&mut t).unwrap();
1701 let mut q = Question::new(
1702 dep.id.clone(),
1703 "check".into(),
1704 "check".into(),
1705 "New dependency approval".into(),
1706 String::new(),
1707 Vec::new(),
1708 );
1709 questions.put(&mut q).unwrap();
1710 dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1711 let changed = dep.blocked_at;
1712 queue.put(&mut dep).unwrap();
1713 dep.block(vec![q.id], Some("same decision paraphrased".into()));
1714 assert_eq!(
1715 dep.blocked_at, changed,
1716 "prose is not new blocking evidence"
1717 );
1718 queue.put(&mut dep).unwrap();
1719 let d = Decision {
1720 id: t.id.clone(),
1721 blocked_by: vec![dep.id.clone()],
1722 evidence: vec![dep.id],
1723 ..Decision::default()
1724 };
1725 assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1726 apply_one(&queue, &questions, &d, None).unwrap();
1727 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1728 }
1729
1730 #[test]
1731 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1732 crate::run::pin_test_home();
1733 let dir = tempdir().unwrap();
1734 let queue = Queue::at(dir.path().join("queue"));
1735 let questions = Questions::at(dir.path().join("questions"));
1736 let mut t = task("resume evidence");
1737 let mut run = RunState::new(
1738 t.repo.clone(),
1739 "main".into(),
1740 "base".into(),
1741 "work".into(),
1742 crate::config::Config::default(),
1743 );
1744 run.begin_pass();
1745 run.status = crate::run::RunStatus::Failed;
1746 run.save().unwrap();
1747 t.start(run.id.clone());
1748 t.runs.push(run.id.clone());
1750 t.release_operator("cli", None);
1751 let approval = t.resumption.as_ref().unwrap().at;
1752 let decision = Decision {
1753 evidence: vec![run.id.clone()],
1754 ..Decision::default()
1755 };
1756 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1757 t.start(run.id.clone());
1758 assert_eq!(t.runs.len(), 1);
1759 run.begin_pass();
1760 run.save().unwrap();
1761 assert!(run.passes.last().unwrap().started_at > approval);
1762 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1763 assert_eq!(evidence.len(), 1);
1764 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1765 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1766 run.begin_pass();
1767 run.save().unwrap();
1768 assert_eq!(
1769 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1770 1
1771 );
1772
1773 let mut q = Question::new(
1774 run.id.clone(),
1775 "review".into(),
1776 "check".into(),
1777 "New failure".into(),
1778 String::new(),
1779 Vec::new(),
1780 );
1781 questions.put(&mut q).unwrap();
1782 let decision = Decision {
1783 evidence: vec![q.id.clone()],
1784 ..Decision::default()
1785 };
1786 assert_eq!(
1787 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1788 1
1789 );
1790 q.asked_at = approval;
1791 questions.put(&mut q).unwrap();
1792 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1793 q.asked_at = jiff::Timestamp::now();
1794 q.run = "unrelated".into();
1795 questions.put(&mut q).unwrap();
1796 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1797
1798 let mut fresh = RunState::new(
1799 t.repo.clone(),
1800 "main".into(),
1801 "base".into(),
1802 "fresh work".into(),
1803 crate::config::Config::default(),
1804 );
1805 fresh.status = crate::run::RunStatus::Failed;
1806 fresh.save().unwrap();
1807 t.start(fresh.id.clone());
1808 let decision = Decision {
1809 evidence: vec![fresh.id.clone()],
1810 ..Decision::default()
1811 };
1812 assert_eq!(
1813 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1814 1
1815 );
1816 }
1817
1818 #[test]
1819 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1820 crate::run::pin_test_home();
1821 let dir = tempdir().unwrap();
1822 let default_repo = dir.path().join("default");
1823 let task_repo = dir.path().join("task");
1824 std::fs::create_dir_all(&default_repo).unwrap();
1825 std::fs::create_dir_all(&task_repo).unwrap();
1826 init_repo_with_branch(&default_repo, "other-branch");
1827 init_repo_with_branch(&task_repo, "magi/f00d/A");
1828
1829 let mut config = Config::default();
1830 config.graph.review_rounds = 6;
1831 let mut state = crate::run::RunState::new(
1832 task_repo.clone(),
1833 "main".to_owned(),
1834 "deadbeef".to_owned(),
1835 "task".to_owned(),
1836 config,
1837 );
1838 state.status = crate::run::RunStatus::Blocked;
1839 state.candidates.push(crate::run::Candidate {
1840 index: 0,
1841 label: 'A',
1842 agent: "mock".to_owned(),
1843 branch: "magi/f00d/A".to_owned(),
1844 worktree: task_repo.clone(),
1845 summary: String::new(),
1846 stat: String::new(),
1847 files: 1,
1848 commits: 1,
1849 empty: false,
1850 failed: None,
1851 verified_noop: None,
1852 duration_ms: 0,
1853 folded: false,
1854 });
1855 state.tally = Some(crate::run::Tally {
1856 first_choice: std::collections::BTreeMap::new(),
1857 borda: std::collections::BTreeMap::new(),
1858 winner: 'A',
1859 rankings: 0,
1860 unanimous_initial: false,
1861 deliberated: false,
1862 changed_votes: 0,
1863 unanimous_final: false,
1864 tie_break: None,
1865 judges: 0,
1866 present: 0,
1867 quorum: 0,
1868 met_quorum: true,
1869 uncontested: Some("solo".to_owned()),
1870 });
1871 state.reviews = vec![
1872 review_round_with_finding(
1873 1,
1874 "R1-1-2",
1875 "answer content is dropped",
1876 &[],
1877 &[("R1-1-2", "the id leaving blocked_by is enough")],
1878 ),
1879 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1880 ];
1881 state.save().unwrap();
1882
1883 let mut t = task("outcome test");
1884 t.repo = task_repo;
1885 t.runs.push(state.id.clone());
1886
1887 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1888 let outcome = finished.outcome;
1889
1890 assert!(outcome.unreadable.is_none());
1891 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1892 assert_eq!(outcome.rounds_used, 2);
1893 assert_eq!(outcome.rounds_max, 6);
1894 assert_eq!(outcome.rounds.len(), 2);
1895 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1896 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1897 assert!(outcome.rounds[1].addressed.is_empty());
1898 assert!(outcome.rounds[1].rejected.is_empty());
1899 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1900 assert!(
1901 outcome.branch_head.is_some(),
1902 "a real branch must resolve a head commit: {outcome:?}"
1903 );
1904 }
1905
1906 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1910 tokio::runtime::Builder::new_current_thread()
1911 .enable_all()
1912 .build()
1913 .unwrap()
1914 .block_on(f)
1915 }
1916
1917 #[test]
1918 fn view_resolves_task_attempt_limit_before_daemon_default() {
1919 let mut t = task("budget");
1920 assert_eq!(view(&t, 7).max_attempts, 7);
1921 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1922 .unwrap();
1923 assert_eq!(view(&t, 7).max_attempts, 3);
1924 t.change_attempt_limit(None).unwrap();
1925 assert_eq!(view(&t, 7).max_attempts, 7);
1926 }
1927
1928 #[test]
1929 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1930 let mut t = task("answered");
1931 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1932 let v = view(&t, 2);
1933 assert_eq!(v.answers.len(), 1);
1934 assert_eq!(v.answers[0].question, "Which backend?");
1935 assert_eq!(v.answers[0].answer, "SQLite");
1936 }
1937
1938 #[tokio::test]
1939 async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1940 for held in [false, true] {
1941 for reference in ["ca5b", "20261013-193611-ca5b"] {
1942 for question in [false, true] {
1943 let dir = tempdir().unwrap();
1944 let queue = Queue::at(dir.path().join("queue"));
1945 let questions = Questions::at(dir.path().join("questions"));
1946 let mut dep = task("prerequisite");
1947 dep.id = "20261013-193611-ca5b".into();
1948 dep.start("existing-run".into());
1949 queue.put(&mut dep).unwrap();
1950 let mut waiting = task("waiting");
1951 if held {
1952 waiting.hold_machine(Some("dependency could not be located".into()));
1953 }
1954 queue.put(&mut waiting).unwrap();
1955 let missing_reference = if reference == "ca5b" {
1956 dep.id.as_str()
1957 } else {
1958 "ca5b"
1959 };
1960 let reply = serde_json::json!({"decisions": [{
1961 "id": waiting.id, "blocked_by": [reference],
1962 "missing_dependencies": [missing_reference],
1963 "question": question.then_some("ca5b is missing; where is its replacement?"),
1964 "recovery": (!question).then_some("hold"),
1965 "reason": "ca5b is missing"
1966 }]}).to_string();
1967 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1968 let (queued, finished) = if held {
1969 (vec![], vec![waiting.clone()])
1970 } else {
1971 (vec![waiting.clone()], vec![])
1972 };
1973 Conductor::new()
1974 .run_once(
1975 &cfg,
1976 dir.path(),
1977 &queue,
1978 &questions,
1979 dir.path(),
1980 &queued,
1981 &[],
1982 &finished,
1983 2,
1984 )
1985 .await
1986 .unwrap();
1987 let after = queue.get(&waiting.id).unwrap();
1988 assert_eq!(after.status, TaskStatus::Blocked);
1989 assert_eq!(after.blocked_by, [dep.id]);
1990 assert_eq!(after.blocked_from, Some(waiting.status));
1991 assert!(!after.block_reason.unwrap().contains("missing"));
1992 assert!(questions.list().is_empty());
1993 assert_eq!(after.instruction, waiting.instruction);
1994 }
1995 }
1996 }
1997 }
1998
1999 #[tokio::test]
2000 async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2001 for held in [false, true] {
2002 for unknown in [false, true] {
2003 for question in [false, true] {
2004 let dir = tempdir().unwrap();
2005 let queue = Queue::at(dir.path().join("queue"));
2006 let questions = Questions::at(dir.path().join("questions"));
2007 let mut dep = task("running prerequisite");
2008 dep.id = "20261013-193611-ca5b".into();
2009 dep.start("existing-run".into());
2010 queue.put(&mut dep).unwrap();
2011 let reference = if unknown {
2012 std::fs::write(questions.root(), "not a directory").unwrap();
2015 dep.id.clone()
2016 } else {
2017 let mut other = task("another prerequisite");
2018 other.id = "20261014-193611-ca5b".into();
2019 queue.put(&mut other).unwrap();
2020 "ca5b".to_owned()
2021 };
2022 assert_eq!(
2023 crate::queue::dependency(&queue, &questions, &reference),
2024 if unknown {
2025 crate::queue::Dependency::Unknown
2026 } else {
2027 crate::queue::Dependency::Ambiguous
2028 }
2029 );
2030 let mut waiting = task("waiting");
2031 if held {
2032 waiting.hold_machine(Some("dependency could not be located".into()));
2033 }
2034 queue.put(&mut waiting).unwrap();
2035 let reply = serde_json::json!({"decisions": [{
2036 "id": waiting.id, "blocked_by": [reference],
2037 "missing_dependencies": [reference],
2038 "question": question.then_some("ca5b is missing"),
2039 "recovery": (!question).then_some("hold"),
2040 "reason": "ca5b is missing"
2041 }]})
2042 .to_string();
2043 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2044 let (queued, finished) = if held {
2045 (vec![], vec![waiting.clone()])
2046 } else {
2047 (vec![waiting.clone()], vec![])
2048 };
2049 Conductor::new()
2050 .run_once(
2051 &cfg,
2052 dir.path(),
2053 &queue,
2054 &questions,
2055 dir.path(),
2056 &queued,
2057 &[],
2058 &finished,
2059 2,
2060 )
2061 .await
2062 .unwrap();
2063 let after = queue.get(&waiting.id).unwrap();
2064 assert_eq!(after.status, TaskStatus::Blocked);
2065 assert_eq!(after.blocked_by, [reference]);
2066 assert_eq!(after.blocked_from, Some(waiting.status));
2067 assert!(!after.block_reason.unwrap().contains("missing"));
2068 assert_eq!(after.instruction, waiting.instruction);
2069 if unknown {
2070 assert!(questions.root().is_file());
2071 } else {
2072 assert!(questions.list().is_empty());
2073 }
2074 }
2075 }
2076 }
2077 }
2078
2079 #[tokio::test]
2080 async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2081 for held in [false, true] {
2082 for unknown in [false, true] {
2083 for question in [false, true] {
2084 for short_blocker in [false, true] {
2085 let dir = tempdir().unwrap();
2086 let queue = Queue::at(dir.path().join("queue"));
2087 let questions = Questions::at(dir.path().join("questions"));
2088 let mut dep = task("running prerequisite");
2089 dep.id = "20261013-193611-ca5b".into();
2090 dep.start("existing-run".into());
2091 queue.put(&mut dep).unwrap();
2092 if unknown {
2093 std::fs::write(questions.root(), "not a directory").unwrap();
2094 } else {
2095 let mut other = task("another prerequisite");
2096 other.id = "20261014-193611-ca5b".into();
2097 queue.put(&mut other).unwrap();
2098 }
2099 let (blocker, missing) = if short_blocker {
2100 ("ca5b", dep.id.as_str())
2101 } else {
2102 (dep.id.as_str(), "ca5b")
2103 };
2104 let mut waiting = task("waiting");
2105 if held {
2106 waiting.hold_machine(Some("dependency could not be located".into()));
2107 }
2108 queue.put(&mut waiting).unwrap();
2109 let reply = serde_json::json!({"decisions": [{
2110 "id": waiting.id, "blocked_by": [blocker],
2111 "missing_dependencies": [missing],
2112 "question": question.then_some("ca5b is missing"),
2113 "recovery": (!question).then_some("hold"),
2114 "reason": "ca5b is missing"
2115 }]})
2116 .to_string();
2117 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2118 let (queued, finished) = if held {
2119 (vec![], vec![waiting.clone()])
2120 } else {
2121 (vec![waiting.clone()], vec![])
2122 };
2123 Conductor::new()
2124 .run_once(
2125 &cfg,
2126 dir.path(),
2127 &queue,
2128 &questions,
2129 dir.path(),
2130 &queued,
2131 &[],
2132 &finished,
2133 2,
2134 )
2135 .await
2136 .unwrap();
2137 assert_eq!(
2138 serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2139 serde_json::to_value(&waiting).unwrap(),
2140 "unverifiable aliases must leave the entire task unchanged"
2141 );
2142 if unknown {
2143 assert!(questions.root().is_file());
2144 } else {
2145 assert!(questions.list().is_empty());
2146 }
2147 }
2148 }
2149 }
2150 }
2151 }
2152
2153 #[tokio::test]
2154 async fn dependency_resolution_genuine_missing_reference_still_asks() {
2155 let dir = tempdir().unwrap();
2156 let queue = Queue::at(dir.path().join("queue"));
2157 let questions = Questions::at(dir.path().join("questions"));
2158 let mut waiting = task("missing prerequisite");
2159 queue.put(&mut waiting).unwrap();
2160 let reply = serde_json::json!({"decisions": [{
2161 "id": waiting.id, "blocked_by": ["not-present"],
2162 "missing_dependencies": ["not-present"],
2163 "question": "Which prerequisite replaces the missing task?"
2164 }]})
2165 .to_string();
2166 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2167 Conductor::new()
2168 .run_once(
2169 &cfg,
2170 dir.path(),
2171 &queue,
2172 &questions,
2173 dir.path(),
2174 &[waiting.clone()],
2175 &[],
2176 &[],
2177 2,
2178 )
2179 .await
2180 .unwrap();
2181 let after = queue.get(&waiting.id).unwrap();
2182 assert_eq!(after.status, TaskStatus::Blocked);
2183 assert_eq!(
2184 questions.get(&after.blocked_by[0]).unwrap().summary,
2185 "Which prerequisite replaces the missing task?"
2186 );
2187 }
2188
2189 #[tokio::test]
2190 async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2191 let dir = tempdir().unwrap();
2192 let queue = Queue::at(dir.path().join("queue"));
2193 let questions = Questions::at(dir.path().join("questions"));
2194 let mut dep = task("prerequisite");
2195 dep.id = "20261013-193611-ca5b".into();
2196 dep.start("existing-run".into());
2197 queue.put(&mut dep).unwrap();
2198 let mut unrelated = task("unrelated work");
2199 unrelated.start("other-run".into());
2200 queue.put(&mut unrelated).unwrap();
2201 std::fs::File::options()
2204 .write(true)
2205 .open(queue.path_of(&unrelated.id))
2206 .unwrap()
2207 .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2208 .unwrap();
2209 unrelated.succeed();
2210 let mut waiting = task("waiting");
2211 let mut asking = task("needs a product decision");
2212 let mut holding = task("must stay held");
2213 queue.put(&mut waiting).unwrap();
2214 queue.put(&mut asking).unwrap();
2215 queue.put(&mut holding).unwrap();
2216 let reply = serde_json::json!({"decisions": [
2217 {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2218 {"id": asking.id, "question": "Which behavior do you want?"},
2219 {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2220 ]})
2221 .to_string();
2222 let mut vars = env(&reply);
2223 vars.insert(
2224 "UNRELATED_PATH".into(),
2225 queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2226 );
2227 vars.insert(
2228 "UNRELATED_RECORD".into(),
2229 serde_json::to_string(&unrelated).unwrap(),
2230 );
2231 let script = r#"#!/bin/sh
2232cat >/dev/null
2233printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2234printf '%s\n' "$MOCK_REPLY"
2235"#;
2236 let cfg = config(mock_agent(dir.path(), script, vars));
2237 Conductor::new()
2238 .run_once(
2239 &cfg,
2240 dir.path(),
2241 &queue,
2242 &questions,
2243 dir.path(),
2244 &[waiting.clone(), asking.clone(), holding.clone()],
2245 &[],
2246 &[],
2247 2,
2248 )
2249 .await
2250 .unwrap();
2251 assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2252 let after = queue.get(&waiting.id).unwrap();
2253 assert_eq!(after.status, TaskStatus::Blocked);
2254 assert_eq!(after.blocked_by, [dep.id]);
2255 let asked = queue.get(&asking.id).unwrap();
2256 assert_eq!(asked.status, TaskStatus::Blocked);
2257 assert_eq!(
2258 questions.get(&asked.blocked_by[0]).unwrap().summary,
2259 "Which behavior do you want?"
2260 );
2261 assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2262 }
2263
2264 #[test]
2265 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2266 let dir = tempdir().unwrap();
2267 let queue = Queue::at(dir.path().join("queue"));
2268 let questions = Questions::at(dir.path().join("questions"));
2269 let mut a = task("a");
2270 a.priority = 9;
2271 queue.put(&mut a).unwrap();
2272
2273 let verdict = Verdict {
2274 decisions: vec![Decision {
2275 id: a.id.clone(),
2276 blocked_by: vec!["20260101-000000-dead".to_owned()],
2277 reason: Some("waits on the other task".to_owned()),
2278 recovery: None,
2279 question: None,
2280 choices: Vec::new(),
2281 actions: BTreeMap::new(),
2282 question_decision: None,
2283 evidence: Vec::new(),
2284 missing_dependencies: Vec::new(),
2285 }],
2286 };
2287 apply(&queue, &questions, &verdict).unwrap();
2288
2289 let back = queue.get(&a.id).unwrap();
2290 assert_eq!(back.status, TaskStatus::Blocked);
2291 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2292 assert_eq!(
2293 back.priority, 9,
2294 "the conductor's reply cannot carry priority"
2295 );
2296 }
2297
2298 #[test]
2299 fn a_question_decision_files_one_and_blocks_on_its_id() {
2300 let dir = tempdir().unwrap();
2301 let queue = Queue::at(dir.path().join("queue"));
2302 let questions = Questions::at(dir.path().join("questions"));
2303 let mut t = task("ambiguous");
2304 queue.put(&mut t).unwrap();
2305
2306 let verdict = Verdict {
2307 decisions: vec![Decision {
2308 id: t.id.clone(),
2309 blocked_by: Vec::new(),
2310 reason: Some("which backend?".to_owned()),
2311 recovery: None,
2312 question: Some("Which storage backend?".to_owned()),
2313 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2314 actions: BTreeMap::new(),
2315 question_decision: None,
2316 evidence: Vec::new(),
2317 missing_dependencies: Vec::new(),
2318 }],
2319 };
2320 apply(&queue, &questions, &verdict).unwrap();
2321
2322 let back = queue.get(&t.id).unwrap();
2323 assert_eq!(back.status, TaskStatus::Blocked);
2324 assert_eq!(back.blocked_by.len(), 1);
2325 let q = questions.get(&back.blocked_by[0]).unwrap();
2326 assert_eq!(q.summary, "Which storage backend?");
2327 assert_eq!(q.node, NODE);
2328 assert!(q.status.open());
2329 }
2330
2331 #[test]
2332 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2333 let dir = tempdir().unwrap();
2334 let queue = Queue::at(dir.path().join("queue"));
2335 let questions = Questions::at(dir.path().join("questions"));
2336 let mut t = task("asked once");
2337 queue.put(&mut t).unwrap();
2338
2339 let decision = Decision {
2340 id: t.id.clone(),
2341 reason: Some("still deciding".to_owned()),
2342 question: Some("Which backend?".to_owned()),
2343 ..Decision::default()
2344 };
2345 apply(
2346 &queue,
2347 &questions,
2348 &Verdict {
2349 decisions: vec![decision.clone()],
2350 },
2351 )
2352 .unwrap();
2353 assert_eq!(questions.list().len(), 1);
2354 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2355
2356 let mut released = queue.get(&t.id).unwrap();
2361 released.release();
2362 queue.put(&mut released).unwrap();
2363
2364 apply(
2365 &queue,
2366 &questions,
2367 &Verdict {
2368 decisions: vec![decision],
2369 },
2370 )
2371 .unwrap();
2372 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2373 let after = queue.get(&t.id).unwrap();
2374 assert_eq!(
2375 after.blocked_by,
2376 [first_question_id],
2377 "the existing open question is reused, not replaced"
2378 );
2379 }
2380
2381 #[test]
2382 fn a_same_id_question_from_another_node_is_not_reused() {
2383 let dir = tempdir().unwrap();
2384 let queue = Queue::at(dir.path().join("queue"));
2385 let questions = Questions::at(dir.path().join("questions"));
2386 let mut t = task("must ask the conductor");
2387 queue.put(&mut t).unwrap();
2388
2389 let mut unrelated = Question::new(
2390 t.id.clone(),
2391 "review".to_owned(),
2392 "reviewer-1".to_owned(),
2393 "An unrelated review question".to_owned(),
2394 String::new(),
2395 Vec::new(),
2396 );
2397 questions.put(&mut unrelated).unwrap();
2398
2399 apply(
2400 &queue,
2401 &questions,
2402 &Verdict {
2403 decisions: vec![Decision {
2404 id: t.id.clone(),
2405 question: Some("Which backend?".to_owned()),
2406 ..Decision::default()
2407 }],
2408 },
2409 )
2410 .unwrap();
2411
2412 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2413 assert_eq!(blocked_by.len(), 1);
2414 assert_ne!(blocked_by[0], unrelated.id);
2415 assert!(questions.get(&unrelated.id).unwrap().status.open());
2416 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2417 }
2418
2419 #[test]
2420 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2421 let dir = tempdir().unwrap();
2422 let queue = Queue::at(dir.path().join("queue"));
2423 let questions = Questions::at(dir.path().join("questions"));
2424 let mut t = task("waits on an answer");
2425 queue.put(&mut t).unwrap();
2426
2427 apply(
2428 &queue,
2429 &questions,
2430 &Verdict {
2431 decisions: vec![Decision {
2432 id: t.id.clone(),
2433 blocked_by: Vec::new(),
2434 reason: None,
2435 recovery: None,
2436 question: Some("Which backend?".to_owned()),
2437 choices: Vec::new(),
2438 actions: BTreeMap::new(),
2439 question_decision: None,
2440 evidence: Vec::new(),
2441 missing_dependencies: Vec::new(),
2442 }],
2443 },
2444 )
2445 .unwrap();
2446 let blocked = queue.get(&t.id).unwrap();
2447 let question_id = blocked.blocked_by[0].clone();
2448
2449 let mut q = questions.get(&question_id).unwrap();
2450 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2451 questions.put(&mut q).unwrap();
2452 assert_eq!(q.status, QuestionStatus::Answered);
2453
2454 let mut task_after = queue.get(&t.id).unwrap();
2458 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2459 task_after.unblock(&question_id);
2460 assert_eq!(task_after.status, TaskStatus::Queued);
2461 assert_eq!(task_after.answers[0].answer, "SQLite");
2462 }
2463
2464 #[test]
2465 fn a_stalled_task_can_be_requeued_or_held() {
2466 let dir = tempdir().unwrap();
2467 let queue = Queue::at(dir.path().join("queue"));
2468 let questions = Questions::at(dir.path().join("questions"));
2469
2470 let mut requeue_me = task("stuck a");
2471 requeue_me.start("run-1".to_owned());
2472 queue.put(&mut requeue_me).unwrap();
2473
2474 let mut hold_me = task("stuck b");
2475 hold_me.start("run-2".to_owned());
2476 queue.put(&mut hold_me).unwrap();
2477
2478 apply(
2479 &queue,
2480 &questions,
2481 &Verdict {
2482 decisions: vec![
2483 Decision {
2484 id: requeue_me.id.clone(),
2485 recovery: Some(Recovery::Requeue),
2486 ..Decision::default()
2487 },
2488 Decision {
2489 id: hold_me.id.clone(),
2490 recovery: Some(Recovery::Hold),
2491 reason: Some("looks broken".to_owned()),
2492 ..Decision::default()
2493 },
2494 ],
2495 },
2496 )
2497 .unwrap();
2498
2499 let requeued = queue.get(&requeue_me.id).unwrap();
2500 assert_eq!(requeued.status, TaskStatus::Queued);
2501 assert_eq!(requeued.attempts, 0);
2502
2503 let held = queue.get(&hold_me.id).unwrap();
2504 assert_eq!(held.status, TaskStatus::Held);
2505 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2506 }
2507
2508 #[test]
2509 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2510 let dir = tempdir().unwrap();
2515 let queue = Queue::at(dir.path().join("queue"));
2516 let questions = Questions::at(dir.path().join("questions"));
2517 let mut t = task("held out of attempts");
2518 t.hold_machine(Some("out of attempts".to_owned()));
2519 queue.put(&mut t).unwrap();
2520
2521 apply(
2522 &queue,
2523 &questions,
2524 &Verdict {
2525 decisions: vec![Decision {
2526 id: t.id.clone(),
2527 reason: Some("what should happen to this one?".to_owned()),
2528 question: Some("Hold it, or try again?".to_owned()),
2529 ..Decision::default()
2530 }],
2531 },
2532 )
2533 .unwrap();
2534 let blocked = queue.get(&t.id).unwrap();
2535 assert_eq!(blocked.status, TaskStatus::Blocked);
2536 let question_id = blocked.blocked_by[0].clone();
2537
2538 let mut q = questions.get(&question_id).unwrap();
2539 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2540 questions.put(&mut q).unwrap();
2541
2542 let mut after = queue.get(&t.id).unwrap();
2544 after.record_answer(q.summary.clone(), "leave it held".to_owned());
2545 after.unblock(&question_id);
2546 assert_eq!(
2547 after.status,
2548 TaskStatus::Held,
2549 "must not fall back to queued"
2550 );
2551 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2552 }
2553
2554 #[test]
2555 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2556 let dir = tempdir().unwrap();
2566 let queue = Queue::at(dir.path().join("queue"));
2567 let questions = Questions::at(dir.path().join("questions"));
2568 let mut t = task("disk pressure, then reconsidered");
2569 t.hold_machine(Some(
2570 "not enough free space to start a run: 10 bytes free, 100 required by \
2571 `[disk] min_free_bytes`"
2572 .to_owned(),
2573 ));
2574 t.record_answer(
2575 "How should this be handled?".to_owned(),
2576 "keep it held, a human will look at it later".to_owned(),
2577 );
2578 queue.put(&mut t).unwrap();
2579
2580 apply(
2581 &queue,
2582 &questions,
2583 &Verdict {
2584 decisions: vec![Decision {
2585 id: t.id.clone(),
2586 recovery: Some(Recovery::Hold),
2587 ..Decision::default()
2588 }],
2589 },
2590 )
2591 .unwrap();
2592
2593 let after = queue.get(&t.id).unwrap();
2594 assert_eq!(after.status, TaskStatus::Held);
2595 assert!(
2596 !after
2597 .hold_reason
2598 .as_deref()
2599 .unwrap_or_default()
2600 .starts_with("not enough free space"),
2601 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2602 after.hold_reason
2603 );
2604
2605 let cfg_dir = tempdir().unwrap();
2608 let config = cfg_dir.path().join("magi.toml");
2609 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2610 let report =
2611 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2612 assert!(report.resumed.is_empty(), "must not be auto-released");
2613 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2614 }
2615
2616 #[test]
2617 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2618 crate::run::pin_test_home();
2619 let dir = tempdir().unwrap();
2620 let queue = Queue::at(dir.path().join("queue"));
2621 let questions = Questions::at(dir.path().join("questions"));
2622 let config = dir.path().join("magi.toml");
2623 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2624 let now = jiff::Timestamp::now();
2625 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2626 let open = || {
2627 questions
2628 .list()
2629 .into_iter()
2630 .filter(|q| q.node == "triage" && q.status.open())
2631 .collect::<Vec<_>>()
2632 };
2633 let hold = Decision {
2634 recovery: Some(Recovery::Hold),
2635 reason: Some("waiting on manual worktree cleanup".to_owned()),
2636 ..Decision::default()
2637 };
2638
2639 let mut t = task("looping hold");
2640 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2641 queue.put(&mut t).unwrap();
2642 let hold = Decision {
2643 id: t.id.clone(),
2644 ..hold
2645 };
2646
2647 assert_eq!(triage().asked.len(), 1);
2649 let first = open().remove(0);
2650 let mut q = questions.get(&first.id).unwrap();
2651 let resume = q.choices[0].clone();
2652 q.answer(Answer::Choice(resume)).unwrap();
2653 questions.put(&mut q).unwrap();
2654 assert_eq!(triage().answered.len(), 1);
2655 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2656
2657 apply(
2659 &queue,
2660 &questions,
2661 &Verdict {
2662 decisions: vec![hold.clone()],
2663 },
2664 )
2665 .unwrap();
2666 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2667 assert!(triage().asked.is_empty());
2668
2669 let mut evidence = Question::new(
2671 t.id.clone(),
2672 "preflight".into(),
2673 "check".into(),
2674 "New permission failure".into(),
2675 String::new(),
2676 Vec::new(),
2677 );
2678 questions.put(&mut evidence).unwrap();
2679 let hold = Decision {
2680 evidence: vec![evidence.id.clone()],
2681 reason: Some("new permission check failed".into()),
2682 ..hold
2683 };
2684 apply(
2685 &queue,
2686 &questions,
2687 &Verdict {
2688 decisions: vec![hold.clone()],
2689 },
2690 )
2691 .unwrap();
2692 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2693
2694 assert_eq!(triage().asked.len(), 1);
2696 let second = open().remove(0);
2697 assert!(second.detail.contains("new permission check failed"));
2698 assert!(triage().asked.is_empty(), "no duplicate question");
2699 assert_eq!(open().len(), 1);
2700
2701 let mut q = questions.get(&second.id).unwrap();
2703 let force = q.choices[0].clone();
2704 q.answer(Answer::Choice(force)).unwrap();
2705 questions.put(&mut q).unwrap();
2706 assert_eq!(triage().answered.len(), 1);
2707 apply(
2708 &queue,
2709 &questions,
2710 &Verdict {
2711 decisions: vec![hold],
2712 },
2713 )
2714 .unwrap();
2715 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2716 }
2717
2718 #[test]
2719 fn conductor_resumption_notice_does_not_solicit_unregistered_questions() {
2720 crate::run::pin_test_home();
2721 let dir = tempdir().unwrap();
2722 let queue = Queue::at(dir.path().join("queue"));
2723 let questions = Questions::at(dir.path().join("questions"));
2724
2725 let mut t = task("task with explicit resumption");
2726 t.status = TaskStatus::Queued;
2727 t.resumption = Some(crate::queue::Resumption {
2728 generation: 1,
2729 source: "test".into(),
2730 question_id: Some("q1".into()),
2731 at: jiff::Timestamp::now(),
2732 instruction: "resume".into(),
2733 answers_through: 0,
2734 runs_through: 0,
2735 cleared_reason: None,
2736 blocking_evidence: Vec::new(),
2737 });
2738 queue.put(&mut t).unwrap();
2739
2740 let decision = Decision {
2742 id: t.id.clone(),
2743 question: Some("How should we proceed with this task?".into()),
2744 blocked_by: vec!["dep-x".into()],
2745 reason: Some("needs clarification".into()),
2746 recovery: Some(Recovery::Hold),
2747 ..Decision::default()
2748 };
2749
2750 apply(
2751 &queue,
2752 &questions,
2753 &Verdict {
2754 decisions: vec![decision],
2755 },
2756 )
2757 .unwrap();
2758
2759 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2761 assert!(questions.list().is_empty());
2763
2764 let notices = crate::notices::Notices::open();
2766 let notice = notices
2767 .get(&crate::notices::id_of(&format!(
2768 "conductor-resumption:{}",
2769 t.id
2770 )))
2771 .expect("conductor-resumption notice filed");
2772
2773 assert_eq!(
2774 notice.link,
2775 Some(crate::notices::Link::Task { id: t.id.clone() })
2776 );
2777 assert!(!notice.message.contains("How should we proceed"));
2779 assert!(
2781 notice
2782 .message
2783 .contains("No reply to this notice is needed.")
2784 );
2785 assert!(!notice.message.contains("Some("));
2787 assert!(!notice.message.contains("None"));
2788 assert!(!notice.message.contains("[]"));
2789 assert!(!notice.message.contains("{:?}"));
2790 assert!(notice.message.contains("Proposed dependencies: dep-x."));
2792 }
2793
2794 #[test]
2795 fn a_runnable_task_can_be_held_directly_without_a_question() {
2796 let dir = tempdir().unwrap();
2797 let queue = Queue::at(dir.path().join("queue"));
2798 let questions = Questions::at(dir.path().join("questions"));
2799 let mut t = task("already answered, should stay put");
2800 queue.put(&mut t).unwrap();
2801
2802 apply(
2803 &queue,
2804 &questions,
2805 &Verdict {
2806 decisions: vec![Decision {
2807 id: t.id.clone(),
2808 recovery: Some(Recovery::Hold),
2809 reason: Some("operator already said keep this held".to_owned()),
2810 ..Decision::default()
2811 }],
2812 },
2813 )
2814 .unwrap();
2815
2816 let after = queue.get(&t.id).unwrap();
2817 assert_eq!(after.status, TaskStatus::Held);
2818 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2819 }
2820
2821 #[test]
2822 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2823 crate::run::pin_test_home();
2824 let dir = tempdir().unwrap();
2830 let queue = Queue::at(dir.path().join("queue"));
2831 let questions = Questions::at(dir.path().join("questions"));
2832 let mut t = task("already merged by hand");
2833 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2834 t.record_answer(
2835 "Handle this one?".to_owned(),
2836 "already merged and cleaned up, close it".to_owned(),
2837 );
2838 queue.put(&mut t).unwrap();
2839
2840 apply(
2841 &queue,
2842 &questions,
2843 &Verdict {
2844 decisions: vec![Decision {
2845 id: t.id.clone(),
2846 recovery: Some(Recovery::Done),
2847 reason: Some("operator confirmed this already landed".to_owned()),
2848 ..Decision::default()
2849 }],
2850 },
2851 )
2852 .unwrap();
2853
2854 let after = queue.get(&t.id).unwrap();
2855 assert_eq!(after.status, TaskStatus::Done);
2856 assert!(after.hold_reason.is_none());
2857 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2858 }
2859
2860 #[test]
2861 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2862 let dir = tempdir().unwrap();
2863 let queue = Queue::at(dir.path().join("queue"));
2864 let questions = Questions::at(dir.path().join("questions"));
2865
2866 let mut queued = task("never ran yet");
2867 queue.put(&mut queued).unwrap();
2868
2869 let mut running = task("mid-run");
2870 running.start("run-1".to_owned());
2871 queue.put(&mut running).unwrap();
2872
2873 for id in [queued.id.clone(), running.id.clone()] {
2874 apply(
2875 &queue,
2876 &questions,
2877 &Verdict {
2878 decisions: vec![Decision {
2879 id,
2880 recovery: Some(Recovery::Done),
2881 ..Decision::default()
2882 }],
2883 },
2884 )
2885 .unwrap();
2886 }
2887
2888 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2889 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2890 }
2891
2892 #[test]
2893 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2894 let dir = tempdir().unwrap();
2897 let queue = Queue::at(dir.path().join("queue"));
2898 let questions = Questions::at(dir.path().join("questions"));
2899 let mut t = task("asked about repeatedly");
2900 t.hold_machine(Some("out of attempts".to_owned()));
2901 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2902 t.record_answer(
2903 "Handle this one? (2)".to_owned(),
2904 "still not yet".to_owned(),
2905 );
2906 queue.put(&mut t).unwrap();
2907 assert_eq!(questions.list().len(), 0);
2908
2909 let verdict = Verdict {
2910 decisions: vec![Decision {
2911 id: t.id.clone(),
2912 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2913 ..Decision::default()
2914 }],
2915 };
2916 apply(&queue, &questions, &verdict).unwrap();
2917
2918 let filed = questions.list();
2919 assert_eq!(filed.len(), 1, "the question was filed");
2920 assert_eq!(
2921 filed[0].summary,
2922 "Branch conflicts with origin/main, how do we proceed?"
2923 );
2924 let deputy = filed[0]
2925 .deputy
2926 .as_ref()
2927 .expect("the wait is handed to a deputy");
2928 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2929 assert_eq!(
2930 deputy.starts, 0,
2931 "filing never starts anything: the loop does not wait"
2932 );
2933 assert!(filed[0].status.open());
2934 assert_eq!(filed[0].node, NODE);
2935 let after = queue.get(&t.id).unwrap();
2936 assert_eq!(after.status, TaskStatus::Blocked);
2937 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2938 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2939 assert!(
2940 !after
2941 .hold_reason
2942 .clone()
2943 .unwrap_or_default()
2944 .contains("conduct tried to ask"),
2945 "no hold was applied"
2946 );
2947
2948 apply(&queue, &questions, &verdict).unwrap();
2949 let again = questions.list();
2950 assert_eq!(again.len(), 1, "the open question is reused");
2951 assert_eq!(again[0].id, filed[0].id);
2952 }
2953
2954 #[test]
2955 fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
2956 let dir = tempdir().unwrap();
2957 init_repo_with_branch(dir.path(), "source-a");
2958 let mut t = task("source decision");
2959 t.repo = dir.path().into();
2960 let decision = |reference: &str| QuestionDecision {
2961 operation: QuestionOperation::Recover,
2962 target: QuestionTarget::Source {
2963 reference: reference.into(),
2964 },
2965 };
2966 assert_eq!(
2967 decision("source-a").identity(&t).unwrap(),
2968 decision("refs/heads/source-a").identity(&t).unwrap()
2969 );
2970 assert_ne!(
2971 decision("source-a").identity(&t).unwrap(),
2972 decision("main").identity(&t).unwrap()
2973 );
2974 assert!(decision("invented-source-label").identity(&t).is_err());
2975 let unrelated = QuestionDecision {
2976 operation: QuestionOperation::Resume,
2977 target: QuestionTarget::Run {
2978 id: "unrelated-run".into(),
2979 },
2980 };
2981 assert!(unrelated.identity(&t).is_err());
2982 }
2983
2984 #[test]
2985 fn base_recovery_conductor_cannot_offer_or_requeue_an_unchanged_stop() {
2986 let dir = tempdir().unwrap();
2987 let queue = Queue::at(dir.path().join("queue"));
2988 let questions = Questions::at(dir.path().join("questions"));
2989 let mut t = task("retained base stop");
2990 let mut state = RunState::new(
2991 dir.path().into(),
2992 "main".into(),
2993 "base".into(),
2994 "retained work".into(),
2995 Config::default(),
2996 );
2997 state.base_sync = Some(crate::run::BaseSync {
2998 tip: "tip".into(),
2999 behind: 3,
3000 attempts: 4,
3001 conflict: Some("exhausted".into()),
3002 already_in: None,
3003 });
3004 state.save_under(dir.path()).unwrap();
3005 t.link_run(&state.id);
3006 t.hold_machine(Some("base stopped".into()));
3007 queue.put(&mut t).unwrap();
3008 let d = Decision {
3009 id: t.id.clone(),
3010 recovery: Some(Recovery::Requeue),
3011 ..Decision::default()
3012 };
3013 apply_one(&queue, &questions, &d, None).unwrap();
3014 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3015 let d = Decision {
3016 id: t.id.clone(),
3017 question: Some("Resume?".into()),
3018 question_decision: Some(QuestionDecision {
3019 operation: QuestionOperation::Resume,
3020 target: QuestionTarget::Run { id: state.id },
3021 }),
3022 ..Decision::default()
3023 };
3024 apply_one(&queue, &questions, &d, None).unwrap();
3025 assert!(questions.list().is_empty());
3026 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3027 }
3028
3029 #[test]
3030 fn invalid_question_targets_still_file_and_block_queued_tasks() {
3031 let dir = tempdir().unwrap();
3032 init_repo_with_branch(dir.path(), "source-a");
3033 let queue = Queue::at(dir.path().join("queue"));
3034 let questions = Questions::at(dir.path().join("questions"));
3035 for target in [
3036 QuestionTarget::Source {
3037 reference: "missing-source".into(),
3038 },
3039 QuestionTarget::Run {
3040 id: "unrelated-run".into(),
3041 },
3042 ] {
3043 let mut t = task("decision with invalid metadata");
3044 t.repo = dir.path().into();
3045 queue.put(&mut t).unwrap();
3046 let d = Decision {
3047 id: t.id.clone(),
3048 question: Some("Which recovery should proceed?".into()),
3049 choices: vec!["inspect first".into(), "keep waiting".into()],
3050 question_decision: Some(QuestionDecision {
3051 operation: QuestionOperation::Recover,
3052 target,
3053 }),
3054 ..Decision::default()
3055 };
3056 apply_one(&queue, &questions, &d, None).unwrap();
3057 let q = questions
3058 .list()
3059 .into_iter()
3060 .find(|q| q.run == t.id)
3061 .unwrap();
3062 assert_eq!(q.summary, d.question.as_deref().unwrap());
3063 assert!(
3064 q.choices.iter().all(|c| q.actions.contains_key(c)),
3065 "manual-only choices must become an executable hold"
3066 );
3067 assert!(q.status.open());
3068 let context: serde_json::Value =
3069 serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
3070 assert!(context[3].is_null(), "invalid metadata was discarded");
3071 let after = queue.get(&t.id).unwrap();
3072 assert_eq!(after.status, TaskStatus::Blocked);
3073 assert_eq!(after.blocked_by, vec![q.id.clone()]);
3074 apply_one(&queue, &questions, &d, None).unwrap();
3075 assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
3076 }
3077 }
3078
3079 #[test]
3080 fn answered_conductor_question_needs_changed_facts_before_reasking() {
3081 let dir = tempdir().unwrap();
3082 let queue = Queue::at(dir.path().join("queue"));
3083 let questions = Questions::at(dir.path().join("questions"));
3084 let mut t = task("recovery decision");
3085 t.hold_machine(Some("out of attempts".into()));
3086 t.runs = vec!["source-run-a".into(), "source-run-b".into()];
3087 queue.put(&mut t).unwrap();
3088 let mut d = Decision {
3089 id: t.id.clone(),
3090 question: Some("Recover from source A?".into()),
3091 choices: vec!["approve source A".into(), "keep held".into()],
3092 question_decision: Some(QuestionDecision {
3093 operation: QuestionOperation::Recover,
3094 target: QuestionTarget::Run {
3095 id: "source-run-a".into(),
3096 },
3097 }),
3098 ..Decision::default()
3099 };
3100 apply_one(&queue, &questions, &d, None).unwrap();
3101 let mut q = questions.list().pop().unwrap();
3102 q.answer(crate::ask::Answer::Choice(q.choices[0].clone()))
3103 .unwrap();
3104 questions.put(&mut q).unwrap();
3105 queue
3106 .modify(&t.id, |task| {
3107 task.unblock(&q.id);
3108 task.hold_machine(Some("recovery remains under consideration".into()));
3109 true
3110 })
3111 .unwrap();
3112 d.question = Some("Should we keep waiting on recovery?".into());
3113 d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3114 apply_one(&queue, &questions, &d, None).unwrap();
3115 assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3116 d.question = Some("Recover from source B?".into());
3117 d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3119 id: "source-run-b".into(),
3120 };
3121 apply_one(&queue, &questions, &d, None).unwrap();
3122 assert_eq!(
3123 questions.list().len(),
3124 2,
3125 "new decision with unchanged task facts"
3126 );
3127 let after = queue.get(&t.id).unwrap();
3128 assert_eq!(after.instruction, t.instruction);
3129 assert_eq!(after.runs, t.runs);
3130 assert_eq!(after.status, TaskStatus::Blocked);
3131 }
3132
3133 #[test]
3134 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3135 let dir = tempdir().unwrap();
3136 let queue = Queue::at(dir.path().join("queue"));
3137 let questions = Questions::at(dir.path().join("questions"));
3138 let mut t = task("asked about once already");
3139 t.hold_machine(Some("out of attempts".to_owned()));
3140 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3141 queue.put(&mut t).unwrap();
3142
3143 apply(
3144 &queue,
3145 &questions,
3146 &Verdict {
3147 decisions: vec![Decision {
3148 id: t.id.clone(),
3149 question: Some("Still not sure - now what?".to_owned()),
3150 ..Decision::default()
3151 }],
3152 },
3153 )
3154 .unwrap();
3155
3156 assert_eq!(questions.list().len(), 1, "the second question was filed");
3157 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3158 }
3159
3160 #[test]
3161 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3162 let dir = tempdir().unwrap();
3168 let queue = Queue::at(dir.path().join("queue"));
3169 let questions = Questions::at(dir.path().join("questions"));
3170 let mut t = task("held, triage already asking about it");
3171 t.hold_machine(Some("cause unclear".to_owned()));
3172 queue.put(&mut t).unwrap();
3173
3174 let mut triage_q = Question::new(
3175 t.id.clone(),
3176 crate::triage::NODE.to_owned(),
3177 "triage".to_owned(),
3178 "Still needed?".to_owned(),
3179 String::new(),
3180 vec![
3181 "resume".to_owned(),
3182 "not yet".to_owned(),
3183 "discard".to_owned(),
3184 ],
3185 );
3186 questions.put(&mut triage_q).unwrap();
3187
3188 for decision in [
3189 Decision {
3190 id: t.id.clone(),
3191 question: Some("what now?".to_owned()),
3192 ..Decision::default()
3193 },
3194 Decision {
3195 id: t.id.clone(),
3196 recovery: Some(Recovery::Requeue),
3197 ..Decision::default()
3198 },
3199 ] {
3200 apply(
3201 &queue,
3202 &questions,
3203 &Verdict {
3204 decisions: vec![decision],
3205 },
3206 )
3207 .unwrap();
3208 }
3209
3210 let after = queue.get(&t.id).unwrap();
3211 assert_eq!(
3212 after.status,
3213 TaskStatus::Held,
3214 "triage still owns this hold"
3215 );
3216 assert!(after.blocked_by.is_empty());
3217 assert_eq!(
3218 questions.list().len(),
3219 1,
3220 "no second, conductor-owned question was filed"
3221 );
3222 }
3223
3224 #[test]
3225 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3226 let dir = tempdir().unwrap();
3234 let queue = Queue::at(dir.path().join("queue"));
3235 let questions = Questions::at(dir.path().join("questions"));
3236 let mut t = task("held, triage question answered but not yet applied");
3237 t.hold_machine(Some("cause unclear".to_owned()));
3238 queue.put(&mut t).unwrap();
3239
3240 let mut triage_q = Question::new(
3241 t.id.clone(),
3242 crate::triage::NODE.to_owned(),
3243 "triage".to_owned(),
3244 "Still needed?".to_owned(),
3245 String::new(),
3246 vec![
3247 "resume".to_owned(),
3248 "not yet".to_owned(),
3249 "discard".to_owned(),
3250 ],
3251 );
3252 questions.put(&mut triage_q).unwrap();
3253 triage_q
3254 .answer(Answer::Choice("not yet".to_owned()))
3255 .unwrap();
3256 questions.put(&mut triage_q).unwrap();
3257 assert!(!triage_q.status.open());
3258
3259 apply(
3260 &queue,
3261 &questions,
3262 &Verdict {
3263 decisions: vec![Decision {
3264 id: t.id.clone(),
3265 question: Some("what now?".to_owned()),
3266 ..Decision::default()
3267 }],
3268 },
3269 )
3270 .unwrap();
3271
3272 let after = queue.get(&t.id).unwrap();
3273 assert_eq!(
3274 after.status,
3275 TaskStatus::Held,
3276 "triage's own answer is not yet applied - conduct must wait"
3277 );
3278 assert_eq!(
3279 questions.list().len(),
3280 1,
3281 "no conductor question was filed over the pending triage answer"
3282 );
3283 }
3284
3285 #[test]
3286 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3287 let dir = tempdir().unwrap();
3288 let queue = Queue::at(dir.path().join("queue"));
3289 let questions = Questions::at(dir.path().join("questions"));
3290 let mut held = task("manual recovery");
3291 held.priority = 300;
3292 held.runs.push("run20260912-224242-daf5".to_owned());
3293 held.hold_manual(Some(
3294 "active manual recovery run20260912-224242-daf5".to_owned(),
3295 ));
3296 queue.put(&mut held).unwrap();
3297
3298 for decision in [
3302 Decision {
3303 id: held.id.clone(),
3304 recovery: Some(Recovery::Requeue),
3305 ..Decision::default()
3306 },
3307 Decision {
3308 id: held.id.clone(),
3309 recovery: Some(Recovery::Hold),
3310 reason: Some("stale replacement reason".to_owned()),
3311 ..Decision::default()
3312 },
3313 Decision {
3314 id: held.id.clone(),
3315 recovery: Some(Recovery::Review),
3316 ..Decision::default()
3317 },
3318 Decision {
3319 id: held.id.clone(),
3320 blocked_by: vec!["other-task".to_owned()],
3321 question: Some("retry now?".to_owned()),
3322 ..Decision::default()
3323 },
3324 ] {
3325 apply(
3326 &queue,
3327 &questions,
3328 &Verdict {
3329 decisions: vec![decision],
3330 },
3331 )
3332 .unwrap();
3333 }
3334
3335 let after = queue.get(&held.id).unwrap();
3336 assert_eq!(after.status, TaskStatus::Held);
3337 assert!(after.operator_held());
3338 assert_eq!(after.priority, 300);
3339 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3340 assert_eq!(
3341 after.hold_reason.as_deref(),
3342 Some("active manual recovery run20260912-224242-daf5")
3343 );
3344 assert!(after.blocked_by.is_empty());
3345 assert!(questions.list().is_empty());
3346 assert!(
3347 queue.next_runnable().is_none(),
3348 "must not dispatch a duplicate"
3349 );
3350 }
3351
3352 #[test]
3353 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3354 let dir = tempdir().unwrap();
3355 let queue = Queue::at(dir.path().join("queue"));
3356 let questions = Questions::at(dir.path().join("questions"));
3357
3358 let mut automatic = task("disk gate");
3359 automatic.hold_machine(Some("disk full".to_owned()));
3360 queue.put(&mut automatic).unwrap();
3361 let requeue = || Verdict {
3362 decisions: vec![Decision {
3363 id: automatic.id.clone(),
3364 recovery: Some(Recovery::Requeue),
3365 ..Decision::default()
3366 }],
3367 };
3368 apply(&queue, &questions, &requeue()).unwrap();
3369 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3370
3371 let mut manual = task("operator gate");
3372 manual.hold_manual(Some("wait for operator".to_owned()));
3373 queue.put(&mut manual).unwrap();
3374 apply(
3375 &queue,
3376 &questions,
3377 &Verdict {
3378 decisions: vec![Decision {
3379 id: manual.id.clone(),
3380 recovery: Some(Recovery::Requeue),
3381 ..Decision::default()
3382 }],
3383 },
3384 )
3385 .unwrap();
3386 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3387
3388 let mut released = queue.get(&manual.id).unwrap();
3391 released.release();
3392 queue.put(&mut released).unwrap();
3393 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3394 }
3395
3396 #[test]
3397 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3398 let dir = tempdir().unwrap();
3399 let queue = Queue::at(dir.path().join("queue"));
3400 let questions = Questions::at(dir.path().join("questions"));
3401 let mut legacy = task("old explicit hold");
3402 legacy.status = TaskStatus::Held;
3403 legacy.hold_reason = Some("manual recovery already active".to_owned());
3404 legacy.hold_source = None;
3405 legacy.blocked_by = vec!["dependency".to_owned()];
3406 queue.put(&mut legacy).unwrap();
3407
3408 apply(
3409 &queue,
3410 &questions,
3411 &Verdict {
3412 decisions: vec![Decision {
3413 id: legacy.id.clone(),
3414 recovery: Some(Recovery::Requeue),
3415 ..Decision::default()
3416 }],
3417 },
3418 )
3419 .unwrap();
3420
3421 let after = queue.get(&legacy.id).unwrap();
3422 assert_eq!(after.status, TaskStatus::Held);
3423 assert_eq!(after.hold_source, None);
3424 assert_eq!(after.hold_reason, legacy.hold_reason);
3425 assert_eq!(after.blocked_by, legacy.blocked_by);
3426 }
3427
3428 #[test]
3429 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3430 crate::run::pin_test_home();
3436 let dir = tempdir().unwrap();
3437 let queue = Queue::at(dir.path().join("queue"));
3438 let questions = Questions::at(dir.path().join("questions"));
3439 let mut t = task("blocked with no readable run");
3440 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
3442 queue.put(&mut t).unwrap();
3443
3444 apply(
3445 &queue,
3446 &questions,
3447 &Verdict {
3448 decisions: vec![Decision {
3449 id: t.id.clone(),
3450 recovery: Some(Recovery::Review),
3451 ..Decision::default()
3452 }],
3453 },
3454 )
3455 .unwrap();
3456
3457 let after = queue.get(&t.id).unwrap();
3458 assert_eq!(
3459 after.status,
3460 TaskStatus::Failed,
3461 "with nothing to reopen, the decision is dropped rather than guessed at"
3462 );
3463 assert!(after.review_branch.is_none());
3464 }
3465
3466 #[test]
3467 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3468 let dir = tempdir().unwrap();
3472 let queue = Queue::at(dir.path().join("queue"));
3473 let questions = Questions::at(dir.path().join("questions"));
3474
3475 for recovery in [Recovery::Requeue, Recovery::Review] {
3476 let mut t = task("ordinary");
3477 queue.put(&mut t).unwrap();
3478
3479 apply(
3480 &queue,
3481 &questions,
3482 &Verdict {
3483 decisions: vec![Decision {
3484 id: t.id.clone(),
3485 recovery: Some(recovery),
3486 ..Decision::default()
3487 }],
3488 },
3489 )
3490 .unwrap();
3491
3492 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3493 }
3494 }
3495
3496 #[tokio::test]
3497 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3498 let dir = tempdir().unwrap();
3499 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3500 let queue = Queue::at(dir.path().join("queue"));
3501 let questions = Questions::at(dir.path().join("questions"));
3502 let mut t = task("normal");
3503 queue.put(&mut t).unwrap();
3504
3505 let mut conductor = Conductor::new();
3506 conductor
3507 .maybe_run(
3508 &cfg,
3509 dir.path(),
3510 &queue,
3511 &questions,
3512 dir.path(),
3513 &[t.clone()],
3514 &[],
3515 &[],
3516 2,
3517 )
3518 .await;
3519
3520 assert_eq!(
3521 queue.get(&t.id).unwrap().status,
3522 TaskStatus::Queued,
3523 "a failed invocation must change nothing"
3524 );
3525 assert!(
3526 queue.next_runnable().is_some(),
3527 "the loop must still be able to take the next task"
3528 );
3529 }
3530
3531 #[tokio::test]
3532 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3533 let dir = tempdir().unwrap();
3534 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3535 let queue = Queue::at(dir.path().join("queue"));
3536 let questions = Questions::at(dir.path().join("questions"));
3537 let mut t = task("normal");
3538 queue.put(&mut t).unwrap();
3539
3540 let mut conductor = Conductor::new();
3541 conductor
3542 .maybe_run(
3543 &cfg,
3544 dir.path(),
3545 &queue,
3546 &questions,
3547 dir.path(),
3548 &[t.clone()],
3549 &[],
3550 &[],
3551 2,
3552 )
3553 .await;
3554
3555 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3556 }
3557
3558 #[tokio::test]
3559 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3560 let dir = tempdir().unwrap();
3561 let mut t = task("chained");
3562 let reply = format!(
3563 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3564 t.id
3565 );
3566 let counted = |id: &str, tail: &str| {
3567 let calls = dir.path().join(format!("{id}.calls"));
3568 let path = dir.path().join(format!("{id}.sh"));
3569 std::fs::write(
3570 &path,
3571 format!(
3572 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3573 calls.display()
3574 ),
3575 )
3576 .unwrap();
3577 AgentSpec {
3578 id: id.to_owned(),
3579 kind: AgentKind::Command,
3580 model: None,
3581 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3582 extra_args: Vec::new(),
3583 env: BTreeMap::new(),
3584 prompt_delivery: None,
3585 max_turn_secs: None,
3586 }
3587 };
3588 let n_calls = |id: &str| {
3589 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3590 .map_or(0, |s| s.lines().count())
3591 };
3592 let mut cfg = config(counted("a", "exit 3"));
3593 cfg.agents = vec![
3594 counted("a", "exit 3"),
3595 counted("b", &format!("printf '%s' '{reply}'")),
3596 ];
3597 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3598 "a".into(),
3599 "b".into(),
3600 "a".into(),
3601 ]));
3602 let queue = Queue::at(dir.path().join("queue"));
3603 let questions = Questions::at(dir.path().join("questions"));
3604 queue.put(&mut t).unwrap();
3605
3606 let mut conductor = Conductor::new();
3607 conductor
3608 .maybe_run(
3609 &cfg,
3610 dir.path(),
3611 &queue,
3612 &questions,
3613 dir.path(),
3614 &[t.clone()],
3615 &[],
3616 &[],
3617 2,
3618 )
3619 .await;
3620
3621 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3622 assert_eq!(n_calls("a"), 1, "each id is tried once");
3623 assert_eq!(n_calls("b"), 1);
3624 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3625 assert!(
3626 saved.contains("\"b\""),
3627 "the seat on disk is the one that ran: {saved}"
3628 );
3629 }
3630
3631 #[tokio::test]
3632 async fn json_survives_code_fences_and_a_preamble() {
3633 let dir = tempdir().unwrap();
3634 let mut t = task("fenced");
3635 let reply = format!(
3636 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3637 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3638 t.id
3639 );
3640 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3641 let queue = Queue::at(dir.path().join("queue"));
3642 let questions = Questions::at(dir.path().join("questions"));
3643 queue.put(&mut t).unwrap();
3644
3645 let mut conductor = Conductor::new();
3646 conductor
3647 .maybe_run(
3648 &cfg,
3649 dir.path(),
3650 &queue,
3651 &questions,
3652 dir.path(),
3653 &[t.clone()],
3654 &[],
3655 &[],
3656 2,
3657 )
3658 .await;
3659
3660 let back = queue.get(&t.id).unwrap();
3661 assert_eq!(back.status, TaskStatus::Blocked);
3662 assert_eq!(back.blocked_by, ["x"]);
3663 }
3664
3665 #[tokio::test]
3666 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3667 let dir = tempdir().unwrap();
3670 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3671 let queue = Queue::at(dir.path().join("queue"));
3672 let questions = Questions::at(dir.path().join("questions"));
3673 let mut t = task("stable");
3674 queue.put(&mut t).unwrap();
3675 let artifacts = dir.path().join("conduct").join("artifacts");
3676 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3677
3678 let mut conductor = Conductor::new();
3679 conductor
3680 .maybe_run(
3681 &cfg,
3682 dir.path(),
3683 &queue,
3684 &questions,
3685 dir.path(),
3686 &[t.clone()],
3687 &[],
3688 &[],
3689 2,
3690 )
3691 .await;
3692 assert!(turn(1).is_file(), "the first cycle must call the conductor");
3693
3694 conductor
3695 .maybe_run(
3696 &cfg,
3697 dir.path(),
3698 &queue,
3699 &questions,
3700 dir.path(),
3701 &[t.clone()],
3702 &[],
3703 &[],
3704 2,
3705 )
3706 .await;
3707 assert!(
3708 !turn(2).is_file(),
3709 "an unchanged revision and an unchanged stalled/finished set must not call the \
3710 conductor twice"
3711 );
3712
3713 t.priority = 1;
3715 queue.put(&mut t).unwrap();
3716 conductor
3717 .maybe_run(
3718 &cfg,
3719 dir.path(),
3720 &queue,
3721 &questions,
3722 dir.path(),
3723 &[t.clone()],
3724 &[],
3725 &[],
3726 2,
3727 )
3728 .await;
3729 assert!(turn(2).is_file(), "a moved revision calls it again");
3730 }
3731
3732 #[tokio::test]
3733 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3734 let dir = tempdir().unwrap();
3740 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3741 let queue = Queue::at(dir.path().join("queue"));
3742 let questions = Questions::at(dir.path().join("questions"));
3743 let mut t = task("quiet");
3744 queue.put(&mut t).unwrap();
3745 let artifacts = dir.path().join("conduct").join("artifacts");
3746 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3747
3748 let mut conductor = Conductor::new();
3749 conductor
3750 .maybe_run(
3751 &cfg,
3752 dir.path(),
3753 &queue,
3754 &questions,
3755 dir.path(),
3756 &[t.clone()],
3757 &[],
3758 &[],
3759 2,
3760 )
3761 .await;
3762 assert!(turn(1).is_file());
3763
3764 conductor
3765 .maybe_run(
3766 &cfg,
3767 dir.path(),
3768 &queue,
3769 &questions,
3770 dir.path(),
3771 &[],
3772 &[t.clone()],
3773 &[],
3774 2,
3775 )
3776 .await;
3777 assert!(
3778 turn(2).is_file(),
3779 "a task turning stalled must call the conductor again"
3780 );
3781
3782 conductor
3785 .maybe_run(
3786 &cfg,
3787 dir.path(),
3788 &queue,
3789 &questions,
3790 dir.path(),
3791 &[],
3792 &[t.clone()],
3793 &[],
3794 2,
3795 )
3796 .await;
3797 assert!(
3798 !turn(3).is_file(),
3799 "the same stalled task lingering must not call the conductor every cycle"
3800 );
3801 }
3802
3803 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3804 apply(
3805 queue,
3806 questions,
3807 &Verdict {
3808 decisions: vec![Decision {
3809 id: id.to_owned(),
3810 reason: Some(note.to_owned()),
3811 recovery: Some(Recovery::Hold),
3812 ..Decision::default()
3813 }],
3814 },
3815 )
3816 .unwrap();
3817 }
3818
3819 #[test]
3820 fn reholding_with_an_identical_note_changes_nothing() {
3821 let dir = tempdir().unwrap();
3822 let queue = Queue::at(dir.path().join("queue"));
3823 let questions = Questions::at(dir.path().join("questions"));
3824 let mut t = task("held");
3825 t.hold_machine(Some("waiting on a human".to_owned()));
3826 queue.put(&mut t).unwrap();
3827
3828 rehold(&queue, &questions, &t.id, "owner said keep it held");
3829 let first = queue.get(&t.id).unwrap();
3830 let rev = queue.revision();
3831 for _ in 0..5 {
3832 rehold(&queue, &questions, &t.id, "owner said keep it held");
3833 }
3834 let after = queue.get(&t.id).unwrap();
3835 assert_eq!(after.hold_reason, first.hold_reason);
3836 assert_eq!(after.updated_at, first.updated_at);
3837 assert_eq!(queue.revision(), rev, "nothing was written");
3838 }
3839
3840 #[test]
3841 fn reholding_with_different_notes_keeps_one_previously_level() {
3842 let dir = tempdir().unwrap();
3843 let queue = Queue::at(dir.path().join("queue"));
3844 let questions = Questions::at(dir.path().join("questions"));
3845 let mut t = task("held");
3846 t.hold_machine(Some("original".to_owned()));
3847 queue.put(&mut t).unwrap();
3848
3849 for i in 0..20 {
3850 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3851 }
3852 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3853 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3854 assert!(reason.starts_with("note "), "the new note leads: {reason}");
3855 }
3856
3857 #[test]
3858 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3859 let mut t = task("deep");
3860 let nested = "same\n\n(previously: same\n\n(previously: same))";
3861 t.hold_machine(Some(nested.to_owned()));
3862 let d = Decision {
3863 id: t.id.clone(),
3864 reason: Some("same".to_owned()),
3865 ..Decision::default()
3866 };
3867 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3868 let d2 = Decision {
3869 reason: Some("other".to_owned()),
3870 ..d
3871 };
3872 assert_eq!(
3873 reaffirmed_hold_reason(&t, &d2).as_deref(),
3874 Some("other\n\n(previously: same)")
3875 );
3876 }
3877
3878 #[tokio::test]
3879 async fn a_repeated_hold_decision_settles_instead_of_looping() {
3880 let dir = tempdir().unwrap();
3881 let queue = Queue::at(dir.path().join("queue"));
3882 let questions = Questions::at(dir.path().join("questions"));
3883 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3884 let mut t = task("held");
3885 t.hold_machine(Some("first".to_owned()));
3886 queue.put(&mut t).unwrap();
3887 let cfg = config(mock_agent(
3888 dir.path(),
3889 REPLY,
3890 env(&reply.replace("ID", &t.id)),
3891 ));
3892 let mut conductor = Conductor::new();
3893 for _ in 0..2 {
3895 let held = queue.get(&t.id).unwrap();
3896 conductor
3897 .maybe_run(
3898 &cfg,
3899 dir.path(),
3900 &queue,
3901 &questions,
3902 dir.path(),
3903 &[],
3904 &[],
3905 &[held],
3906 2,
3907 )
3908 .await;
3909 }
3910 let held = queue.get(&t.id).unwrap();
3911 assert!(
3912 held.hold_reason
3913 .as_deref()
3914 .unwrap()
3915 .starts_with("keep held")
3916 );
3917 assert!(
3918 !conductor.worth_a_look(&queue, &[], &[held]),
3919 "an unchanged held task must stop being reconsidered"
3920 );
3921 }
3922
3923 #[tokio::test]
3924 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3925 let dir = tempdir().unwrap();
3926 let queue = Queue::at(dir.path().join("queue"));
3927 let questions = Questions::at(dir.path().join("questions"));
3928 let mut t = task("held");
3929 t.hold_machine(Some("first".to_owned()));
3930 queue.put(&mut t).unwrap();
3931 let reply = |note: &str| {
3932 format!(
3933 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3934 t.id
3935 )
3936 };
3937 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3938 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3939 let mut conductor = Conductor::new();
3940 let mut writes = 0;
3941 for i in 0..6 {
3942 let held = queue.get(&t.id).unwrap();
3943 let before = queue.revision();
3944 conductor
3945 .maybe_run(
3946 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3947 dir.path(),
3948 &queue,
3949 &questions,
3950 dir.path(),
3951 &[],
3952 &[],
3953 &[held],
3954 2,
3955 )
3956 .await;
3957 if queue.revision() != before {
3958 writes += 1;
3959 }
3960 }
3961 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3962 let mut held = queue.get(&t.id).unwrap();
3963 held.instruction.push_str(" (edited)");
3964 queue.put(&mut held).unwrap();
3965 let held = queue.get(&t.id).unwrap();
3966 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3967 }
3968
3969 #[test]
3970 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3971 let dir = tempdir().unwrap();
3972 let queue = Queue::at(dir.path().join("queue"));
3973 let mut t = task("t");
3974 queue.put(&mut t).unwrap();
3975
3976 let mut conductor = Conductor::new();
3977 assert!(
3978 conductor.worth_a_look(&queue, &[], &[]),
3979 "a conductor that has never run has something to look at"
3980 );
3981
3982 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3983 assert!(
3984 !conductor.worth_a_look(&queue, &[], &[]),
3985 "nothing changed and nothing is stalled or finished"
3986 );
3987 assert!(
3988 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3989 "a stalled task is worth a look even at the same revision"
3990 );
3991 assert!(
3992 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3993 "a finished task is worth a look even at the same revision"
3994 );
3995 }
3996
3997 #[tokio::test]
3998 async fn the_conduct_path_never_calls_ask_and_wait() {
3999 let dir = tempdir().unwrap();
4005 let queue = Queue::at(dir.path().join("queue"));
4006 let questions = Questions::at(dir.path().join("questions"));
4007 let mut t = task("asks without blocking");
4008 queue.put(&mut t).unwrap();
4009
4010 apply(
4011 &queue,
4012 &questions,
4013 &Verdict {
4014 decisions: vec![Decision {
4015 id: t.id.clone(),
4016 question: Some("ok?".to_owned()),
4017 ..Decision::default()
4018 }],
4019 },
4020 )
4021 .unwrap();
4022 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
4024 }
4025
4026 #[test]
4027 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
4028 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
4029 t.resume_override = Some(crate::queue::OperatorResume {
4030 question_id: "q".into(),
4031 at: jiff::Timestamp::now(),
4032 conductor_rehold: None,
4033 forced: false,
4034 pinned_run: Some("run-1".into()),
4035 });
4036 assert!(pinned_resume(&t));
4037 assert!(!may_hold(&mut t, "waiting for magi resume"));
4038 assert!(
4039 t.resume_override
4040 .as_ref()
4041 .unwrap()
4042 .conductor_rehold
4043 .is_none(),
4044 "a refused hold is not recorded as an override"
4045 );
4046 t.resume_override = None;
4047 assert!(!pinned_resume(&t));
4048 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
4049 }
4050
4051 #[test]
4052 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
4053 let dir = tempdir().unwrap();
4054 let queue = Queue::at(dir.path().join("queue"));
4055 let questions = Questions::at(dir.path().join("questions"));
4056 let mut t = task("resume me");
4057 t.resume_override = Some(crate::queue::OperatorResume {
4058 question_id: "q".into(),
4059 at: jiff::Timestamp::now(),
4060 conductor_rehold: None,
4061 forced: true,
4062 pinned_run: Some("run-1".into()),
4063 });
4064 queue.put(&mut t).unwrap();
4065
4066 apply(
4067 &queue,
4068 &questions,
4069 &Verdict {
4070 decisions: vec![
4071 Decision {
4072 id: t.id.clone(),
4073 question: Some("really?".to_owned()),
4074 ..Decision::default()
4075 },
4076 Decision {
4077 id: t.id.clone(),
4078 blocked_by: vec!["other".to_owned()],
4079 ..Decision::default()
4080 },
4081 ],
4082 },
4083 )
4084 .unwrap();
4085
4086 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
4087 assert!(questions.list().is_empty());
4088 }
4089}