1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{ChoiceAction, Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104 Recover,
106 Resume,
108 Repair,
110 Complete,
112 Inspect,
114 Configure,
116}
117
118#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123 Task,
125 Run {
127 id: String,
129 },
130 Setting {
133 path: String,
135 keys: Vec<String>,
137 },
138 Source {
140 reference: String,
142 },
143}
144
145#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148 pub operation: QuestionOperation,
150 pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155 fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156 let target = match &self.target {
157 QuestionTarget::Task => format!("task:{}", task.id),
158 QuestionTarget::Run { id } => {
159 anyhow::ensure!(
160 task.runs.contains(id),
161 "question target is not this task's run"
162 );
163 format!("run:{id}")
164 }
165 QuestionTarget::Setting { path, keys } => {
166 use std::path::Component;
167 anyhow::ensure!(
168 !path.is_empty()
169 && Path::new(path)
170 .components()
171 .all(|c| matches!(c, Component::Normal(_)))
172 && !keys.is_empty()
173 && keys.iter().all(|key| !key.is_empty()),
174 "question setting must name a repository file and concrete keys"
175 );
176 let root = task.repo.canonicalize()?;
177 let file = root.join(path).canonicalize()?;
178 let relative = file
179 .strip_prefix(&root)
180 .context("question setting is outside the repository")?;
181 let content = std::fs::read_to_string(&file)?;
182 let value: toml::Value = toml::from_str(&content)?;
183 let mut setting = &value;
184 for key in keys {
185 setting = setting
186 .get(key)
187 .context("question setting key does not exist")?;
188 }
189 format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190 }
191 QuestionTarget::Source { reference } => {
192 use crate::proc::Quiet as _;
193 let output = std::process::Command::new("git")
194 .args([
195 "rev-parse",
196 "--verify",
197 "--end-of-options",
198 &format!("{reference}^{{commit}}"),
199 ])
200 .current_dir(&task.repo)
201 .quiet()
202 .output()
203 .context("resolve question source")?;
204 anyhow::ensure!(
205 output.status.success(),
206 "question source cannot be resolved"
207 );
208 let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209 anyhow::ensure!(
210 !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211 "invalid question source commit"
212 );
213 format!("source:{commit}")
214 }
215 };
216 Ok((self.operation, target))
217 }
218}
219
220#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224 pub id: String,
226 #[serde(default)]
230 pub blocked_by: Vec<String>,
231 #[serde(default)]
236 pub missing_dependencies: Vec<String>,
237 #[serde(default)]
239 pub reason: Option<String>,
240 #[serde(default)]
247 pub recovery: Option<Recovery>,
248 #[serde(default)]
254 pub question: Option<String>,
255 #[serde(default)]
257 pub choices: Vec<String>,
258 #[serde(default)]
260 pub actions: BTreeMap<String, ChoiceAction>,
261 #[serde(default)]
264 pub question_decision: Option<QuestionDecision>,
265 #[serde(default)]
267 pub evidence: Vec<String>,
268}
269
270#[derive(Debug, Clone, Default, Deserialize)]
283pub struct Verdict {
284 pub decisions: Vec<Decision>,
287}
288
289fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
292 prompt::ConductTask {
293 id: t.id.clone(),
294 title: t.title.clone(),
295 instruction: t.instruction.clone(),
296 repo: t.repo.display().to_string(),
297 priority: t.priority,
298 status: t.status.as_str().to_owned(),
299 attempts: t.attempts,
300 max_attempts: t.effective_attempt_limit(max_attempts),
301 last_error: t.last_error.clone(),
302 hold_reason: t.hold_reason.clone(),
303 held_at: t.held_at.map(|at| at.to_string()),
304 hold_source: t.hold_source.map(|source| source.label().to_owned()),
305 blocked_by: t.blocked_by.clone(),
306 answers: t
307 .answers
308 .iter()
309 .map(|a| prompt::ConductAnswer {
310 question: a.question.clone(),
311 answer: a.answer.clone(),
312 })
313 .collect(),
314 operator_resume: t.resumption.as_ref().map(|r| format!(
315 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
316 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
317 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
318 wording alone is insufficient.",
319 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
320 )).or_else(|| t.resume_override.as_ref().map(|o| {
321 format!(
322 "the operator explicitly answered \"resume\" at {}; do not hold this \
323 task again for the same reason unless there is new information",
324 o.at
325 )
326 })),
327 }
328}
329
330fn pinned_resume(task: &Task) -> bool {
334 task.resume_override
335 .as_ref()
336 .is_some_and(|o| o.pinned_run.is_some())
337}
338
339fn new_blocking_evidence(
342 queue: &Queue,
343 questions: &Questions,
344 task: &Task,
345 d: &Decision,
346) -> Vec<String> {
347 let Some(r) = &task.resumption else {
348 return Vec::new();
349 };
350 d.evidence
351 .iter()
352 .filter_map(|id| {
353 let version = if let Ok(q) = questions.get(id) {
354 (q.asked_at > r.at
355 && (q.run == task.id || task.runs.contains(&q.run))
356 && q.node != NODE
357 && q.status != crate::ask::QuestionStatus::Abandoned)
358 .then(|| {
359 format!(
360 "question:{id}:{}:{:?}:{}",
361 q.asked_at,
362 q.status,
363 q.resolution().unwrap_or_default()
364 )
365 })
366 } else if d.blocked_by.contains(id) {
367 queue
368 .get(id)
369 .ok()
370 .filter(|dep| {
371 dep.id != task.id
372 && dep.status != TaskStatus::Done
373 && (dep.created_at > r.at
374 || dep.held_at.is_some_and(|at| at > r.at)
375 || dep.operator_decided_at.is_some_and(|at| at > r.at)
376 || dep.passes.iter().any(|p| p.settled_at > r.at)
377 || (dep.status == TaskStatus::Blocked
378 && dep.blocked_at.is_some_and(|at| at > r.at)))
379 })
380 .map(|dep| {
381 format!(
382 "dependency:{id}:{:?}:{:?}:{:?}",
383 dep.status,
384 dep.chat_notice_key(),
385 dep.hold_notice_cause
386 )
387 })
388 } else if task.runs.iter().any(|run| run == id) {
389 RunState::load(id)
390 .ok()
391 .filter(|run| {
392 run.updated_at > r.at
395 && run
396 .passes
397 .last()
398 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
399 && matches!(
400 run.status,
401 crate::run::RunStatus::Failed
402 | crate::run::RunStatus::Blocked
403 | crate::run::RunStatus::Stalled
404 )
405 })
406 .map(|run| {
407 format!(
408 "run:{id}:{:?}:{:?}",
409 run.status,
410 run.passes.last().map(|p| p.index)
411 )
412 })
413 } else {
414 None
415 };
416 version.filter(|v| !r.blocking_evidence.contains(v))
417 })
418 .collect()
419}
420
421fn may_hold(task: &mut Task, reason: &str) -> bool {
425 if pinned_resume(task) {
426 return false;
427 }
428 if task.resumption.is_some() {
429 return true; }
431 let Some(o) = task.resume_override.as_mut() else {
432 return true;
433 };
434 if o.forced || o.pinned_run.is_some() {
435 tracing::warn!(
436 "conductor tried to hold task {} after the operator forced a resume: {reason}",
437 task.id
438 );
439 return false;
440 }
441 if o.conductor_rehold.is_some() {
442 return false;
443 }
444 o.conductor_rehold = Some(reason.to_owned());
445 true
446}
447
448fn severity_str(s: crate::verdict::Severity) -> &'static str {
451 match s {
452 crate::verdict::Severity::Nit => "nit",
453 crate::verdict::Severity::Minor => "minor",
454 crate::verdict::Severity::Major => "major",
455 crate::verdict::Severity::Blocker => "blocker",
456 }
457}
458
459fn surviving_branch(task: &Task) -> Option<String> {
464 let last = task.runs.last()?;
465 let state = RunState::load(last).ok()?;
466 if !crate::daemon::compatible_run_instruction(&state, task) {
467 return None;
468 }
469 state.winner().map(|c| c.branch.clone())
470}
471
472fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
499 let note = match &d.reason {
500 Some(reason) => reason.clone(),
501 None => match task.answers.last() {
502 Some(a) => format!(
503 "conduct held this again with no new reason given; last operator \
504 answer on record: {}",
505 a.answer
506 ),
507 None => "conduct held this again with no reason given".to_owned(),
508 },
509 };
510 match task.hold_reason.as_deref() {
511 Some(prior) if !prior.is_empty() => {
512 let outer = outermost_hold_note(prior);
513 if note.trim() == prior.trim() || note.trim() == outer {
514 None
515 } else {
516 Some(format!("{note}\n\n(previously: {outer})"))
517 }
518 }
519 _ => Some(note),
520 }
521}
522
523const PREVIOUSLY: &str = "\n\n(previously: ";
526
527fn outermost_hold_note(reason: &str) -> &str {
530 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
531}
532
533fn hold_note(d: &Decision) -> String {
535 d.reason
536 .clone()
537 .unwrap_or_else(|| "(no reason given)".to_owned())
538}
539
540async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
542 let Some(run_id) = task.runs.last().cloned() else {
543 return prompt::ConductOutcome {
544 run_id: "(none)".to_owned(),
545 unreadable: Some("this task has not produced a run yet".to_owned()),
546 run_status: None,
547 open_findings: Vec::new(),
548 rounds_used: 0,
549 rounds_max: 0,
550 rounds: Vec::new(),
551 branch: None,
552 branch_head: None,
553 references: None,
554 empty_candidate: false,
555 };
556 };
557 let state = match RunState::load(&run_id) {
558 Ok(s) => s,
559 Err(e) => {
560 tracing::warn!(
565 "conductor: could not read run {run_id} for task {}: {e:#}",
566 task.short()
567 );
568 return prompt::ConductOutcome {
569 run_id,
570 unreadable: Some(format!("{e:#}")),
571 run_status: None,
572 open_findings: Vec::new(),
573 rounds_used: 0,
574 rounds_max: 0,
575 rounds: Vec::new(),
576 branch: None,
577 branch_head: None,
578 references: None,
579 empty_candidate: false,
580 };
581 }
582 };
583
584 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
585 id: f.id.clone(),
586 title: f.title.clone(),
587 severity: severity_str(f.severity).to_owned(),
588 };
589 let open_findings = state
590 .open_findings()
591 .into_iter()
592 .map(finding_view)
593 .collect();
594 let rounds = state
595 .reviews
596 .iter()
597 .map(|r| prompt::ConductRound {
598 round: r.round,
599 findings: r
600 .reviews
601 .iter()
602 .flat_map(|rec| rec.findings.iter())
603 .map(finding_view)
604 .collect(),
605 addressed: r
606 .fix
607 .as_ref()
608 .map(|fx| fx.addressed.clone())
609 .unwrap_or_default(),
610 rejected: r
611 .fix
612 .as_ref()
613 .map(|fx| {
614 fx.rejected
615 .iter()
616 .map(|rej| prompt::ConductRejection {
617 id: rej.id.clone(),
618 why: rej.why.clone(),
619 })
620 .collect()
621 })
622 .unwrap_or_default(),
623 })
624 .collect();
625 let branch = state.winner().map(|c| c.branch.clone());
626 let branch_head = match &branch {
627 Some(b) => crate::git::rev_parse(repo, b)
628 .await
629 .ok()
630 .map(|h| h.chars().take(8).collect()),
631 None => None,
632 };
633
634 prompt::ConductOutcome {
635 run_id,
636 unreadable: None,
637 run_status: Some(state.status.as_str().to_owned()),
638 open_findings,
639 rounds_used: state.reviews.len(),
640 rounds_max: state.config.graph.review_rounds,
641 rounds,
642 branch,
643 branch_head,
644 references: crate::refs::describe(&state.seeds),
645 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
646 }
647}
648
649async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
653 let id = task.runs.last()?;
654 let state = match RunState::load(id) {
655 Ok(s) => s,
656 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
657 };
658 let branch = state.winner()?.branch.clone();
659 let base = state.base_branch.clone();
660 match crate::land::find_open_pr(repo, &branch, &base).await {
661 Ok(crate::land::OpenPr::None) => None,
662 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
663 "pull request {url} is already open for {branch} into {base}"
664 )),
665 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
666 "several pull requests are already open for {branch} into {base}: {}",
667 urls.join(" ")
668 )),
669 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
670 }
671}
672
673async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
680 for d in verdict
681 .decisions
682 .iter_mut()
683 .filter(|d| d.question.is_some())
684 {
685 let Ok(task) = queue.get(&d.id) else {
686 continue;
687 };
688 let text = format!("{}\n{}", task.title, task.instruction);
689 let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
690 if crate::refs::scan(&text).is_empty() {
691 if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
692 q.push_str(&format!(
693 "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
694 ));
695 }
696 continue;
697 }
698 let repo = repo_for(&task, repo);
699 let remote = &cfg.merge.remote;
700 let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
701 .await
702 .ok();
703 let facts = match base {
704 Some(base) => {
705 let base_name = base.clone();
706 let tracking = format!("{remote}/{base}");
707 let refreshed = crate::git::fetch(&repo, remote, &base)
710 .await
711 .is_ok_and(|o| o.ok());
712 let against = tracking;
715 match crate::git::rev_parse(&repo, &against).await {
716 Ok(tip) => crate::refs::describe(
717 &crate::refs::resolve(&repo, &tip, remote, &text).await,
718 )
719 .map(|facts| {
720 if refreshed {
721 facts
722 } else {
723 format!(
724 "{facts}\n(could not fetch {remote}/{base_name}: this is \
725 against the last fetched `{against}`, which may be behind \
726 the remote)"
727 )
728 }
729 }),
730 Err(_) => Some(format!(
731 "could NOT be checked against the repository: {against} is not \
732 available here, so whether the work is in the base is unknown"
733 )),
734 }
735 }
736 None => Some("could not check the repository: no base branch known".to_owned()),
737 };
738 let facts = match (facts, pr_fact) {
739 (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
740 (f, p) => f.or(p),
741 };
742 if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
743 q.push_str(&format!(
744 "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
745 ));
746 }
747 }
748}
749
750async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
752 prompt::ConductFinished {
753 task: view(t, max_attempts),
754 outcome: outcome_for(t, &repo_for(t, repo)).await,
755 }
756}
757
758fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
761 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
762 fallback.to_path_buf()
763 } else {
764 task.repo.clone()
765 }
766}
767
768fn apply_one(
785 queue: &Queue,
786 questions: &Questions,
787 d: &Decision,
788 snapshot: Option<&Task>,
789) -> Result<()> {
790 let _claim = queue
791 .claim(&d.id)
792 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
793 let mut task = queue.get(&d.id).context("no such task")?;
794
795 if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
798 return Ok(());
799 }
800 let blocking =
801 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
802 let evidence = new_blocking_evidence(queue, questions, &task, d);
803 if blocking && task.resumption.is_some() && evidence.is_empty() {
804 crate::notices::raise(crate::notices::conductor_resumption(
805 &task.id,
806 task.short(),
807 d.reason.as_deref(),
808 &d.blocked_by,
809 &d.evidence,
810 d.question.is_some(),
811 ));
812 return Ok(());
813 }
814
815 if blocking && !evidence.is_empty() {
816 let mut cause = evidence.clone();
817 cause.sort();
818 cause.dedup();
819 task.notice_cause(
820 &format!("conductor-evidence:{}", cause.join(",")),
821 None,
822 None,
823 );
824 }
825 if blocking {
826 if let Some(r) = task.resumption.as_mut() {
827 r.blocking_evidence.extend(evidence);
828 }
829 }
830
831 if task.operator_held() {
835 return Ok(());
836 }
837
838 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
842 return Ok(());
843 }
844
845 let pinned = pinned_resume(&task);
848 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
849 return Ok(());
850 }
851
852 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
856 && !d.missing_dependencies.is_empty()
857 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
858 && d.missing_dependencies
859 .iter()
860 .any(|id| !d.blocked_by.contains(id))
861 && d.missing_dependencies
862 .iter()
863 .chain(&d.blocked_by)
864 .any(|id| {
865 matches!(
866 crate::queue::dependency(queue, questions, id),
867 crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
868 )
869 })
870 {
871 return Ok(());
872 }
873
874 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
880 && !d.missing_dependencies.is_empty()
881 && d.missing_dependencies.iter().all(|id| {
882 let claim = crate::queue::dependency(queue, questions, id);
883 claim != crate::queue::Dependency::Missing
884 && d.blocked_by.iter().any(|blocker| {
885 blocker == id
886 || (matches!(
887 &claim,
888 crate::queue::Dependency::Task(_)
889 | crate::queue::Dependency::Question(_)
890 | crate::queue::Dependency::Deleted(_)
891 ) && crate::queue::dependency(queue, questions, blocker) == claim)
892 })
893 })
894 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
895 && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
896 {
897 let mut ids = d.blocked_by.clone();
898 crate::queue::normalize_blockers(queue, questions, &mut ids);
899 let reason = format!("Waiting on dependencies: {}", ids.join(", "));
900 task.block(ids, Some(reason));
901 return queue.put(&mut task);
902 }
903
904 if let Some(text) = &d.question {
905 if task.status == TaskStatus::Done {
906 return Ok(());
907 }
908 let mut held = task.clone();
909 held.status = TaskStatus::Held;
910 let targets: BTreeSet<_> = d
911 .blocked_by
912 .iter()
913 .filter_map(|id| {
914 if let Ok(dep) = queue.get(id) {
915 Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
916 } else if questions.get(id).is_ok() {
917 Some(format!("question:{id}"))
918 } else {
919 None
920 }
921 })
922 .collect();
923 let decision = d
924 .question_decision
925 .as_ref()
926 .and_then(|q| match q.identity(&task) {
927 Ok(identity) => Some(identity),
928 Err(error) => {
929 tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
932 None
933 }
934 });
935 let context_value =
936 serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
937 let context = serde_json::to_string(&context_value)?;
938 if questions.list().iter().any(|q| {
942 q.node == NODE
943 && q.run == task.id
944 && q.status == crate::ask::QuestionStatus::Answered
945 && q.decision_context.as_ref().is_some_and(|stored| {
946 if stored == &context {
947 return true;
948 }
949 decision.is_none()
952 && serde_json::from_str::<serde_json::Value>(stored)
953 .ok()
954 .and_then(|v| {
955 v.as_array().map(|items| {
956 items.len() == 4
957 && items[..3]
958 == context_value.as_array().expect("context")[..3]
959 })
960 })
961 == Some(true)
962 })
963 }) {
964 return Ok(());
965 }
966 let question_id = match questions
970 .list()
971 .into_iter()
972 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
973 {
974 Some(existing) => existing.id,
975 None => {
976 let mut q = Question::new(
977 task.id.clone(),
978 NODE.to_owned(),
979 SEAT.to_owned(),
980 text.clone(),
981 d.reason.clone().unwrap_or_default(),
982 d.choices.clone(),
983 );
984 q.actions = d.actions.clone();
985 crate::choice_effect::executable_conductor(&mut q);
986 q.decision_context = Some(context);
987 q.deputy = Some(Deputy::new(crate::deputy::brief(
992 &task.id,
993 d.reason.as_deref().unwrap_or_default(),
994 &q.choices,
995 &q.actions,
996 )));
997 if task.repo.is_dir() && task.repo != Path::new(".") {
998 q.cwd = Some(task.repo.to_string_lossy().into_owned());
999 }
1000 questions.put_executable(&mut q)?;
1001 q.id
1002 }
1003 };
1004 task.block(vec![question_id], d.reason.clone());
1005 return queue.put(&mut task);
1006 }
1007
1008 match task.status {
1009 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
1013 if may_hold(&mut task, &hold_note(d)) {
1014 task.hold_machine(d.reason.clone());
1015 queue.put(&mut task)?;
1016 }
1017 }
1018 TaskStatus::Queued if !d.blocked_by.is_empty() => {
1019 let mut ids = d.blocked_by.clone();
1020 crate::queue::normalize_blockers(queue, questions, &mut ids);
1021 task.block(ids, d.reason.clone());
1022 queue.put(&mut task)?;
1023 }
1024 TaskStatus::Running => match d.recovery {
1025 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1026 Some(Recovery::Requeue) => {
1027 task.requeue();
1028 queue.put(&mut task)?;
1029 }
1030 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
1031 task.hold_machine(d.reason.clone());
1032 queue.put(&mut task)?;
1033 }
1034 _ => {}
1038 },
1039 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
1040 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1043 Some(Recovery::Requeue) => {
1044 task.requeue();
1045 queue.put(&mut task)?;
1046 }
1047 Some(Recovery::Hold) => {
1048 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1052 if may_hold(&mut task, &hold_note(d)) {
1053 task.hold_machine(Some(reason));
1054 queue.put(&mut task)?;
1055 }
1056 }
1057 }
1058 Some(Recovery::Review) => {
1059 if let Some(branch) = surviving_branch(&task) {
1060 task.request_review(branch);
1061 queue.put(&mut task)?;
1062 }
1063 }
1069 Some(Recovery::Done) => {
1070 task.succeed();
1071 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1075 queue.put(&mut task)?;
1076 }
1077 None => {}
1078 },
1079 _ => {}
1082 }
1083 Ok(())
1084}
1085
1086pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1090 for d in &verdict.decisions {
1091 if let Err(e) = apply_one(queue, questions, d, None) {
1092 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1093 }
1094 }
1095 Ok(())
1096}
1097
1098pub fn seat_path(home: &Path) -> PathBuf {
1104 home.join("conduct").join("seat.json")
1105}
1106
1107pub fn load_seat(home: &Path) -> Option<SeatState> {
1109 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1110}
1111
1112fn busy_path(home: &Path) -> PathBuf {
1113 home.join("conduct").join("busy")
1114}
1115
1116pub fn busy(home: &Path) -> bool {
1122 std::fs::metadata(busy_path(home))
1123 .and_then(|m| m.modified())
1124 .ok()
1125 .and_then(|t| t.elapsed().ok())
1126 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1127}
1128
1129struct Busy(PathBuf);
1131
1132impl Busy {
1133 fn mark(home: &Path) -> Self {
1134 let path = busy_path(home);
1135 if let Some(dir) = path.parent() {
1136 let _ = std::fs::create_dir_all(dir);
1137 }
1138 let _ = std::fs::write(&path, std::process::id().to_string());
1139 Self(path)
1140 }
1141}
1142
1143impl Drop for Busy {
1144 fn drop(&mut self) {
1145 let _ = std::fs::remove_file(&self.0);
1146 }
1147}
1148
1149#[derive(Debug, Default)]
1152pub struct Conductor {
1153 seat: Option<SeatState>,
1154 last_seen: Option<(u64, BTreeSet<String>)>,
1155 considered: BTreeMap<String, String>,
1159}
1160
1161fn decision_input(task: &Task) -> serde_json::Value {
1166 let mut v = serde_json::to_value(task).unwrap_or_default();
1167 if let Some(o) = v.as_object_mut() {
1168 o.remove("chat_notice");
1169 o.remove("chat_notices");
1170 o.remove("updated_at");
1172 }
1173 v
1174}
1175
1176fn input_fingerprint(task: &Task) -> String {
1177 let mut v = decision_input(task);
1178 if let Some(o) = v.as_object_mut() {
1179 o.remove("hold_reason");
1180 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1181 ro.remove("conductor_rehold");
1182 }
1183 }
1184 v.to_string()
1185}
1186
1187impl Conductor {
1188 #[must_use]
1190 pub fn new() -> Self {
1191 Self::default()
1192 }
1193
1194 fn settled(
1199 considered: &BTreeMap<String, String>,
1200 stalled: &[Task],
1201 finished: &[Task],
1202 ) -> BTreeSet<String> {
1203 stalled
1204 .iter()
1205 .chain(finished)
1206 .filter(|t| t.status == TaskStatus::Held)
1207 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1208 .map(|t| t.id.clone())
1209 .collect()
1210 }
1211
1212 fn snapshot_with(
1213 considered: &BTreeMap<String, String>,
1214 queue: &Queue,
1215 stalled: &[Task],
1216 finished: &[Task],
1217 ) -> (u64, BTreeSet<String>) {
1218 let ids = stalled
1219 .iter()
1220 .chain(finished)
1221 .map(|t| t.id.clone())
1222 .collect();
1223 let skip = Self::settled(considered, stalled, finished);
1224 (queue.revision_excluding(&skip), ids)
1225 }
1226
1227 fn snapshot(
1228 &self,
1229 queue: &Queue,
1230 stalled: &[Task],
1231 finished: &[Task],
1232 ) -> (u64, BTreeSet<String>) {
1233 Self::snapshot_with(&self.considered, queue, stalled, finished)
1234 }
1235
1236 #[must_use]
1253 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1254 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1255 }
1256
1257 #[allow(clippy::too_many_arguments)]
1261 pub async fn maybe_run(
1262 &mut self,
1263 cfg: &Config,
1264 repo: &Path,
1265 queue: &Queue,
1266 questions: &Questions,
1267 home: &Path,
1268 queued: &[Task],
1269 stalled: &[Task],
1270 finished: &[Task],
1271 max_attempts: usize,
1272 ) {
1273 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1274 return;
1275 }
1276 self.considered = stalled
1279 .iter()
1280 .chain(finished)
1281 .filter(|t| t.status == TaskStatus::Held)
1282 .map(|t| (t.id.clone(), input_fingerprint(t)))
1283 .collect();
1284 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1285 if let Err(e) = self
1286 .run_once(
1287 cfg,
1288 repo,
1289 queue,
1290 questions,
1291 home,
1292 queued,
1293 stalled,
1294 finished,
1295 max_attempts,
1296 )
1297 .await
1298 {
1299 tracing::warn!("conductor: {e:#}");
1305 }
1306 }
1307
1308 #[allow(clippy::too_many_arguments)]
1309 async fn run_once(
1310 &mut self,
1311 cfg: &Config,
1312 repo: &Path,
1313 queue: &Queue,
1314 questions: &Questions,
1315 home: &Path,
1316 queued: &[Task],
1317 stalled: &[Task],
1318 finished: &[Task],
1319 max_attempts: usize,
1320 ) -> Result<()> {
1321 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1322 return Ok(());
1323 }
1324
1325 let primary = cfg
1326 .resolve_roles()
1327 .context("resolving the conductor seat")?
1328 .conductor;
1329 let chain = match cfg.roles.conductor.as_ref() {
1333 Some(c) if c.ids().len() > 1 => {
1334 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1335 }
1336 _ => vec![primary],
1337 };
1338
1339 let runnable_views: Vec<prompt::ConductTask> =
1340 queued.iter().map(|t| view(t, max_attempts)).collect();
1341 let stalled_views: Vec<prompt::ConductTask> =
1342 stalled.iter().map(|t| view(t, max_attempts)).collect();
1343 let mut finished_views = Vec::with_capacity(finished.len());
1344 for t in finished {
1345 finished_views.push(finished_view(t, repo, max_attempts).await);
1346 }
1347
1348 let body = prompt::with_overlay(
1349 prompt::conduct(
1350 &runnable_views,
1351 &stalled_views,
1352 &finished_views,
1353 &cfg.graph.language,
1354 ),
1355 cfg.prompts.overlay(NODE),
1356 );
1357
1358 let body = format!(
1359 "{body}\n\n{}",
1360 crate::queue::dependency_facts(queue, questions)
1361 );
1362 let artifacts = home.join("conduct").join("artifacts");
1363 let cache_dir = cfg.cache_dir();
1366
1367 let busy = Busy::mark(home);
1369 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1370 for spec in &chain {
1371 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1374 if needs_new_seat {
1375 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1376 }
1377 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1378 let stem = if std::ptr::eq(spec, &chain[0]) {
1381 format!("turn-{}", seat.turns + 1)
1382 } else {
1383 format!("turn-{}-{}", seat.turns + 1, spec.id)
1384 };
1385 let inv = Invocation {
1386 scope: crate::agent::InvocationScope::Repository,
1387 coordination: cfg.coordination.as_ref(),
1388 fetched_base: None,
1389 cwd: repo,
1390 prompt: &body,
1391 timeout: TURN_TIMEOUT,
1392 allow_write: false,
1395 unsandboxed: false,
1396 sessions: cfg.graph.sessions,
1397 artifacts: &artifacts,
1398 stem: &stem,
1399 run: NODE,
1400 node: NODE,
1401 cache_dir: cache_dir.as_deref(),
1402 attachments: &[],
1403 writable: &[],
1404 };
1405 let out = agent::invoke(spec, seat, &inv).await;
1406 if let Ok(body) = serde_json::to_string(&*seat) {
1409 let path = seat_path(home);
1410 if let Some(dir) = path.parent() {
1411 let _ = std::fs::create_dir_all(dir);
1412 }
1413 let _ = std::fs::write(path, body);
1414 }
1415 let advance = agent::chain_advances(&out);
1416 result = match out {
1417 Err(e) => Err(e.context("invoking the conductor")),
1418 Ok(out) if advance => Err(anyhow::anyhow!(
1419 "no usable reply ({})",
1420 crate::notices::turn_status(
1421 out.exit_code,
1422 out.timed_out,
1423 out.quota_exhausted()
1424 )
1425 )),
1426 Ok(out) => verdict::extract_json(&out.text)
1427 .context("the conductor's reply could not be parsed"),
1428 };
1429 if result.is_ok() {
1430 break;
1431 }
1432 if chain.len() > 1 {
1433 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1434 }
1435 }
1436 drop(busy);
1437 let mut verdict = result?;
1438 attach_facts(cfg, repo, queue, &mut verdict).await;
1439 for d in &verdict.decisions {
1440 let snapshot = queued
1441 .iter()
1442 .chain(stalled)
1443 .chain(finished)
1444 .find(|t| t.id == d.id);
1445 if let Some(snapshot) = snapshot {
1446 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1447 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1448 }
1449 }
1450 }
1451 Ok(())
1452 }
1453}
1454
1455#[cfg(test)]
1456mod tests {
1457 use std::collections::BTreeMap;
1458
1459 use tempfile::tempdir;
1460
1461 use super::*;
1462 use crate::ask::{Answer, QuestionStatus};
1463 use crate::config::{AgentKind, AgentSpec, Graph};
1464 use crate::queue::Source;
1465
1466 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1467 let path = dir.join("mock-conduct-agent.sh");
1468 std::fs::write(&path, script).expect("write mock");
1469 AgentSpec {
1470 id: "mock".to_owned(),
1471 kind: AgentKind::Command,
1472 model: None,
1473 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1474 extra_args: Vec::new(),
1475 env,
1476 prompt_delivery: None,
1477 }
1478 }
1479
1480 fn config(spec: AgentSpec) -> Config {
1481 Config {
1482 agents: vec![spec],
1483 graph: Graph {
1484 language: "en".to_owned(),
1485 ..Graph::default()
1486 },
1487 ..Config::default()
1488 }
1489 }
1490
1491 fn task(title: &str) -> Task {
1492 Task::new(
1493 title.to_owned(),
1494 format!("do {title}"),
1495 std::path::PathBuf::from("."),
1496 Source::Human,
1497 )
1498 }
1499
1500 #[test]
1501 fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1502 let mut t = task("waiting");
1503 t.hold_machine(Some("waiting for recovery".into()));
1504 let before = input_fingerprint(&t);
1505 let snapshot = decision_input(&t);
1506 t.chat_notice = Some(crate::queue::ChatNotice {
1507 key: t.chat_notice_key().unwrap(),
1508 at: jiff::Timestamp::now(),
1509 outcome: crate::queue::ChatNoticeOutcome::Posted,
1510 });
1511 t.chat_notices.push(t.chat_notice_key().unwrap());
1512 t.updated_at = jiff::Timestamp::now();
1513 assert_eq!(before, input_fingerprint(&t));
1514 assert_eq!(snapshot, decision_input(&t));
1515 t.hold_reason = Some("recovery remains under consideration".into());
1516 assert_eq!(before, input_fingerprint(&t));
1517 t.record_answer("a different question".into(), "a new decision".into());
1518 assert_ne!(before, input_fingerprint(&t));
1519 }
1520
1521 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1522 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1523
1524 fn env(reply: &str) -> BTreeMap<String, String> {
1525 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1526 }
1527
1528 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1529
1530 fn init_repo_with_branch(dir: &Path, branch: &str) {
1534 use crate::proc::Quiet as _;
1535 let run = |args: &[&str]| {
1536 let out = std::process::Command::new("git")
1537 .args(args)
1538 .current_dir(dir)
1539 .quiet()
1540 .output()
1541 .expect("spawn git");
1542 assert!(
1543 out.status.success(),
1544 "git {args:?} failed: {}",
1545 String::from_utf8_lossy(&out.stderr)
1546 );
1547 };
1548 run(&["init", "-b", "main"]);
1549 run(&["config", "user.name", "magi test"]);
1550 run(&["config", "user.email", "magi@example.com"]);
1551 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1552 run(&["add", "-A"]);
1553 run(&["commit", "-m", "init"]);
1554 run(&["checkout", "-b", branch]);
1555 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1556 run(&["add", "-A"]);
1557 run(&["commit", "-m", "candidate work"]);
1558 }
1559
1560 fn review_round_with_finding(
1561 round: usize,
1562 finding_id: &str,
1563 title: &str,
1564 addressed: &[&str],
1565 rejected: &[(&str, &str)],
1566 ) -> crate::run::ReviewRound {
1567 crate::run::ReviewRound {
1568 round,
1569 head: "deadbeef".to_owned(),
1570 verified_head: None,
1571 verified_at: None,
1572 reviews: vec![crate::run::ReviewRecord {
1573 attempts: 0,
1574 reviewer: 1,
1575 agent: "mock".to_owned(),
1576 summary: String::new(),
1577 findings: vec![crate::verdict::Finding {
1578 id: finding_id.to_owned(),
1579 severity: crate::verdict::Severity::Major,
1580 file: None,
1581 line: None,
1582 title: title.to_owned(),
1583 detail: String::new(),
1584 }],
1585 vote: None,
1586 failed: None,
1587 duration_ms: 0,
1588 }],
1589 e2e: Vec::new(),
1590 verify_retried: false,
1591 e2e_deferred: false,
1592 e2e_defer_reason: None,
1593 fix: Some(crate::run::FixRecord {
1594 agent: "mock".to_owned(),
1595 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1596 rejected: rejected
1597 .iter()
1598 .map(|(id, why)| crate::verdict::Rejection {
1599 id: (*id).to_owned(),
1600 why: (*why).to_owned(),
1601 })
1602 .collect(),
1603 notes: String::new(),
1604 committed: false,
1605 failed: None,
1606 duration_ms: 0,
1607 continuation: None,
1608 }),
1609 blocking: 1,
1610 answered: 1,
1611 expected: 1,
1612 clean: false,
1613 progressed: true,
1614 vote_split: false,
1615 reconsideration: Vec::new(),
1616 verdict: None,
1617 }
1618 }
1619
1620 #[test]
1621 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1622 crate::run::pin_test_home();
1623 let dir = tempdir().unwrap();
1624 let queue = Queue::at(dir.path().join("queue"));
1625 let questions = Questions::at(dir.path().join("questions"));
1626 let mut t = task("recover");
1627 t.answers.push(crate::queue::AnsweredQuestion {
1628 question: "Inspect remote first?".into(),
1629 answer: "Inspect, then await resumption instructions".into(),
1630 });
1631 t.hold_machine(Some("waiting for remote inspection".into()));
1632 queue.put(&mut t).unwrap();
1633 let snapshot = queue.get(&t.id).unwrap();
1634 t.edit(
1635 "recover".into(),
1636 "Reuse the measured PR head; fix Clippy; review".into(),
1637 )
1638 .unwrap();
1639 t.release_operator("cli", None);
1640 queue.put(&mut t).unwrap();
1641 let approval = t.resumption.clone();
1642 let d = Decision {
1643 id: t.id.clone(),
1644 recovery: Some(Recovery::Hold),
1645 reason: Some("remote inspection still required".into()),
1646 ..Decision::default()
1647 };
1648 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1649 let stale_question = Decision {
1650 question: Some("Inspect remote?".into()),
1651 ..d.clone()
1652 };
1653 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1654 assert!(questions.list().is_empty());
1655 for reason in [
1656 "waiting for remote inspection",
1657 "inspection has not been confirmed",
1658 ] {
1659 let repeated = Decision {
1660 reason: Some(reason.into()),
1661 question: Some("How should we proceed?".into()),
1662 ..d.clone()
1663 };
1664 apply(
1665 &queue,
1666 &questions,
1667 &Verdict {
1668 decisions: vec![repeated],
1669 },
1670 )
1671 .unwrap();
1672 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1673 }
1674 assert!(
1675 questions.list().is_empty(),
1676 "a rejected proposal registers no question"
1677 );
1678 let notice = crate::notices::Notices::open()
1679 .get(&crate::notices::id_of(&format!(
1680 "conductor-resumption:{}",
1681 t.id
1682 )))
1683 .unwrap();
1684 assert!(!notice.message.contains("How should we proceed?"));
1685 assert!(!notice.message.contains("Some("));
1686 assert!(notice.message.contains("No reply to this notice is needed"));
1687 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1688 assert_eq!(reloaded.resumption, approval);
1689 assert_eq!(reloaded.answers, t.answers);
1690 let v = view(&reloaded, 3);
1691 assert!(v.operator_resume.unwrap().contains("superseded"));
1692 let mut dep = task("new prerequisite");
1693 dep.hold_machine(Some("new authentication failure".into()));
1694 queue.put(&mut dep).unwrap();
1695 let new_blocker = Decision {
1696 blocked_by: vec![dep.id.clone()],
1697 evidence: vec![dep.id.clone()],
1698 recovery: None,
1700 reason: Some("new prerequisite failed".into()),
1701 ..d.clone()
1702 };
1703 apply(
1704 &queue,
1705 &questions,
1706 &Verdict {
1707 decisions: vec![new_blocker.clone()],
1708 },
1709 )
1710 .unwrap();
1711 let mut after = queue.get(&t.id).unwrap();
1712 assert_eq!(after.status, TaskStatus::Blocked);
1713 after.release();
1716 queue.put(&mut after).unwrap();
1717 apply(
1718 &queue,
1719 &questions,
1720 &Verdict {
1721 decisions: vec![new_blocker],
1722 },
1723 )
1724 .unwrap();
1725 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1726 after.start("new-run".into());
1727 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1728 after.hold_manual(Some("operator says wait".into()));
1729 queue.put(&mut after).unwrap();
1730 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1731 assert!(queue.get(&t.id).unwrap().operator_held());
1732 }
1733
1734 #[test]
1735 fn newly_blocked_dependency_is_material_evidence_after_approval() {
1736 let dir = tempdir().unwrap();
1737 let queue = Queue::at(dir.path().join("queue"));
1738 let questions = Questions::at(dir.path().join("questions"));
1739 let mut dep = task("old queued dependency");
1740 queue.put(&mut dep).unwrap();
1741 let mut t = task("approved task");
1742 t.release_operator("test", None);
1743 queue.put(&mut t).unwrap();
1744 let mut q = Question::new(
1745 dep.id.clone(),
1746 "check".into(),
1747 "check".into(),
1748 "New dependency approval".into(),
1749 String::new(),
1750 Vec::new(),
1751 );
1752 questions.put(&mut q).unwrap();
1753 dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1754 let changed = dep.blocked_at;
1755 queue.put(&mut dep).unwrap();
1756 dep.block(vec![q.id], Some("same decision paraphrased".into()));
1757 assert_eq!(
1758 dep.blocked_at, changed,
1759 "prose is not new blocking evidence"
1760 );
1761 queue.put(&mut dep).unwrap();
1762 let d = Decision {
1763 id: t.id.clone(),
1764 blocked_by: vec![dep.id.clone()],
1765 evidence: vec![dep.id],
1766 ..Decision::default()
1767 };
1768 assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1769 apply_one(&queue, &questions, &d, None).unwrap();
1770 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1771 }
1772
1773 #[test]
1774 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1775 crate::run::pin_test_home();
1776 let dir = tempdir().unwrap();
1777 let queue = Queue::at(dir.path().join("queue"));
1778 let questions = Questions::at(dir.path().join("questions"));
1779 let mut t = task("resume evidence");
1780 let mut run = RunState::new(
1781 t.repo.clone(),
1782 "main".into(),
1783 "base".into(),
1784 "work".into(),
1785 crate::config::Config::default(),
1786 );
1787 run.begin_pass();
1788 run.status = crate::run::RunStatus::Failed;
1789 run.save().unwrap();
1790 t.start(run.id.clone());
1791 t.runs.push(run.id.clone());
1793 t.release_operator("cli", None);
1794 let approval = t.resumption.as_ref().unwrap().at;
1795 let decision = Decision {
1796 evidence: vec![run.id.clone()],
1797 ..Decision::default()
1798 };
1799 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1800 t.start(run.id.clone());
1801 assert_eq!(t.runs.len(), 1);
1802 run.begin_pass();
1803 run.save().unwrap();
1804 assert!(run.passes.last().unwrap().started_at > approval);
1805 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1806 assert_eq!(evidence.len(), 1);
1807 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1808 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1809 run.begin_pass();
1810 run.save().unwrap();
1811 assert_eq!(
1812 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1813 1
1814 );
1815
1816 let mut q = Question::new(
1817 run.id.clone(),
1818 "review".into(),
1819 "check".into(),
1820 "New failure".into(),
1821 String::new(),
1822 Vec::new(),
1823 );
1824 questions.put(&mut q).unwrap();
1825 let decision = Decision {
1826 evidence: vec![q.id.clone()],
1827 ..Decision::default()
1828 };
1829 assert_eq!(
1830 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1831 1
1832 );
1833 q.asked_at = approval;
1834 questions.put(&mut q).unwrap();
1835 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1836 q.asked_at = jiff::Timestamp::now();
1837 q.run = "unrelated".into();
1838 questions.put(&mut q).unwrap();
1839 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1840
1841 let mut fresh = RunState::new(
1842 t.repo.clone(),
1843 "main".into(),
1844 "base".into(),
1845 "fresh work".into(),
1846 crate::config::Config::default(),
1847 );
1848 fresh.status = crate::run::RunStatus::Failed;
1849 fresh.save().unwrap();
1850 t.start(fresh.id.clone());
1851 let decision = Decision {
1852 evidence: vec![fresh.id.clone()],
1853 ..Decision::default()
1854 };
1855 assert_eq!(
1856 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1857 1
1858 );
1859 }
1860
1861 #[test]
1862 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1863 crate::run::pin_test_home();
1864 let dir = tempdir().unwrap();
1865 let default_repo = dir.path().join("default");
1866 let task_repo = dir.path().join("task");
1867 std::fs::create_dir_all(&default_repo).unwrap();
1868 std::fs::create_dir_all(&task_repo).unwrap();
1869 init_repo_with_branch(&default_repo, "other-branch");
1870 init_repo_with_branch(&task_repo, "magi/f00d/A");
1871
1872 let mut config = Config::default();
1873 config.graph.review_rounds = 6;
1874 let mut state = crate::run::RunState::new(
1875 task_repo.clone(),
1876 "main".to_owned(),
1877 "deadbeef".to_owned(),
1878 "task".to_owned(),
1879 config,
1880 );
1881 state.status = crate::run::RunStatus::Blocked;
1882 state.candidates.push(crate::run::Candidate {
1883 index: 0,
1884 label: 'A',
1885 agent: "mock".to_owned(),
1886 branch: "magi/f00d/A".to_owned(),
1887 worktree: task_repo.clone(),
1888 summary: String::new(),
1889 stat: String::new(),
1890 files: 1,
1891 commits: 1,
1892 empty: false,
1893 failed: None,
1894 verified_noop: None,
1895 duration_ms: 0,
1896 folded: false,
1897 });
1898 state.tally = Some(crate::run::Tally {
1899 first_choice: std::collections::BTreeMap::new(),
1900 borda: std::collections::BTreeMap::new(),
1901 winner: 'A',
1902 rankings: 0,
1903 unanimous_initial: false,
1904 deliberated: false,
1905 changed_votes: 0,
1906 unanimous_final: false,
1907 tie_break: None,
1908 judges: 0,
1909 present: 0,
1910 quorum: 0,
1911 met_quorum: true,
1912 uncontested: Some("solo".to_owned()),
1913 });
1914 state.reviews = vec![
1915 review_round_with_finding(
1916 1,
1917 "R1-1-2",
1918 "answer content is dropped",
1919 &[],
1920 &[("R1-1-2", "the id leaving blocked_by is enough")],
1921 ),
1922 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1923 ];
1924 state.save().unwrap();
1925
1926 let mut t = task("outcome test");
1927 t.repo = task_repo;
1928 t.runs.push(state.id.clone());
1929
1930 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1931 let outcome = finished.outcome;
1932
1933 assert!(outcome.unreadable.is_none());
1934 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1935 assert_eq!(outcome.rounds_used, 2);
1936 assert_eq!(outcome.rounds_max, 6);
1937 assert_eq!(outcome.rounds.len(), 2);
1938 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1939 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1940 assert!(outcome.rounds[1].addressed.is_empty());
1941 assert!(outcome.rounds[1].rejected.is_empty());
1942 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1943 assert!(
1944 outcome.branch_head.is_some(),
1945 "a real branch must resolve a head commit: {outcome:?}"
1946 );
1947 }
1948
1949 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1953 tokio::runtime::Builder::new_current_thread()
1954 .enable_all()
1955 .build()
1956 .unwrap()
1957 .block_on(f)
1958 }
1959
1960 #[test]
1961 fn view_resolves_task_attempt_limit_before_daemon_default() {
1962 let mut t = task("budget");
1963 assert_eq!(view(&t, 7).max_attempts, 7);
1964 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1965 .unwrap();
1966 assert_eq!(view(&t, 7).max_attempts, 3);
1967 t.change_attempt_limit(None).unwrap();
1968 assert_eq!(view(&t, 7).max_attempts, 7);
1969 }
1970
1971 #[test]
1972 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1973 let mut t = task("answered");
1974 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1975 let v = view(&t, 2);
1976 assert_eq!(v.answers.len(), 1);
1977 assert_eq!(v.answers[0].question, "Which backend?");
1978 assert_eq!(v.answers[0].answer, "SQLite");
1979 }
1980
1981 #[tokio::test]
1982 async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1983 for held in [false, true] {
1984 for reference in ["ca5b", "20261013-193611-ca5b"] {
1985 for question in [false, true] {
1986 let dir = tempdir().unwrap();
1987 let queue = Queue::at(dir.path().join("queue"));
1988 let questions = Questions::at(dir.path().join("questions"));
1989 let mut dep = task("prerequisite");
1990 dep.id = "20261013-193611-ca5b".into();
1991 dep.start("existing-run".into());
1992 queue.put(&mut dep).unwrap();
1993 let mut waiting = task("waiting");
1994 if held {
1995 waiting.hold_machine(Some("dependency could not be located".into()));
1996 }
1997 queue.put(&mut waiting).unwrap();
1998 let missing_reference = if reference == "ca5b" {
1999 dep.id.as_str()
2000 } else {
2001 "ca5b"
2002 };
2003 let reply = serde_json::json!({"decisions": [{
2004 "id": waiting.id, "blocked_by": [reference],
2005 "missing_dependencies": [missing_reference],
2006 "question": question.then_some("ca5b is missing; where is its replacement?"),
2007 "recovery": (!question).then_some("hold"),
2008 "reason": "ca5b is missing"
2009 }]}).to_string();
2010 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2011 let (queued, finished) = if held {
2012 (vec![], vec![waiting.clone()])
2013 } else {
2014 (vec![waiting.clone()], vec![])
2015 };
2016 Conductor::new()
2017 .run_once(
2018 &cfg,
2019 dir.path(),
2020 &queue,
2021 &questions,
2022 dir.path(),
2023 &queued,
2024 &[],
2025 &finished,
2026 2,
2027 )
2028 .await
2029 .unwrap();
2030 let after = queue.get(&waiting.id).unwrap();
2031 assert_eq!(after.status, TaskStatus::Blocked);
2032 assert_eq!(after.blocked_by, [dep.id]);
2033 assert_eq!(after.blocked_from, Some(waiting.status));
2034 assert!(!after.block_reason.unwrap().contains("missing"));
2035 assert!(questions.list().is_empty());
2036 assert_eq!(after.instruction, waiting.instruction);
2037 }
2038 }
2039 }
2040 }
2041
2042 #[tokio::test]
2043 async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2044 for held in [false, true] {
2045 for unknown in [false, true] {
2046 for question in [false, true] {
2047 let dir = tempdir().unwrap();
2048 let queue = Queue::at(dir.path().join("queue"));
2049 let questions = Questions::at(dir.path().join("questions"));
2050 let mut dep = task("running prerequisite");
2051 dep.id = "20261013-193611-ca5b".into();
2052 dep.start("existing-run".into());
2053 queue.put(&mut dep).unwrap();
2054 let reference = if unknown {
2055 std::fs::write(questions.root(), "not a directory").unwrap();
2058 dep.id.clone()
2059 } else {
2060 let mut other = task("another prerequisite");
2061 other.id = "20261014-193611-ca5b".into();
2062 queue.put(&mut other).unwrap();
2063 "ca5b".to_owned()
2064 };
2065 assert_eq!(
2066 crate::queue::dependency(&queue, &questions, &reference),
2067 if unknown {
2068 crate::queue::Dependency::Unknown
2069 } else {
2070 crate::queue::Dependency::Ambiguous
2071 }
2072 );
2073 let mut waiting = task("waiting");
2074 if held {
2075 waiting.hold_machine(Some("dependency could not be located".into()));
2076 }
2077 queue.put(&mut waiting).unwrap();
2078 let reply = serde_json::json!({"decisions": [{
2079 "id": waiting.id, "blocked_by": [reference],
2080 "missing_dependencies": [reference],
2081 "question": question.then_some("ca5b is missing"),
2082 "recovery": (!question).then_some("hold"),
2083 "reason": "ca5b is missing"
2084 }]})
2085 .to_string();
2086 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2087 let (queued, finished) = if held {
2088 (vec![], vec![waiting.clone()])
2089 } else {
2090 (vec![waiting.clone()], vec![])
2091 };
2092 Conductor::new()
2093 .run_once(
2094 &cfg,
2095 dir.path(),
2096 &queue,
2097 &questions,
2098 dir.path(),
2099 &queued,
2100 &[],
2101 &finished,
2102 2,
2103 )
2104 .await
2105 .unwrap();
2106 let after = queue.get(&waiting.id).unwrap();
2107 assert_eq!(after.status, TaskStatus::Blocked);
2108 assert_eq!(after.blocked_by, [reference]);
2109 assert_eq!(after.blocked_from, Some(waiting.status));
2110 assert!(!after.block_reason.unwrap().contains("missing"));
2111 assert_eq!(after.instruction, waiting.instruction);
2112 if unknown {
2113 assert!(questions.root().is_file());
2114 } else {
2115 assert!(questions.list().is_empty());
2116 }
2117 }
2118 }
2119 }
2120 }
2121
2122 #[tokio::test]
2123 async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2124 for held in [false, true] {
2125 for unknown in [false, true] {
2126 for question in [false, true] {
2127 for short_blocker in [false, true] {
2128 let dir = tempdir().unwrap();
2129 let queue = Queue::at(dir.path().join("queue"));
2130 let questions = Questions::at(dir.path().join("questions"));
2131 let mut dep = task("running prerequisite");
2132 dep.id = "20261013-193611-ca5b".into();
2133 dep.start("existing-run".into());
2134 queue.put(&mut dep).unwrap();
2135 if unknown {
2136 std::fs::write(questions.root(), "not a directory").unwrap();
2137 } else {
2138 let mut other = task("another prerequisite");
2139 other.id = "20261014-193611-ca5b".into();
2140 queue.put(&mut other).unwrap();
2141 }
2142 let (blocker, missing) = if short_blocker {
2143 ("ca5b", dep.id.as_str())
2144 } else {
2145 (dep.id.as_str(), "ca5b")
2146 };
2147 let mut waiting = task("waiting");
2148 if held {
2149 waiting.hold_machine(Some("dependency could not be located".into()));
2150 }
2151 queue.put(&mut waiting).unwrap();
2152 let reply = serde_json::json!({"decisions": [{
2153 "id": waiting.id, "blocked_by": [blocker],
2154 "missing_dependencies": [missing],
2155 "question": question.then_some("ca5b is missing"),
2156 "recovery": (!question).then_some("hold"),
2157 "reason": "ca5b is missing"
2158 }]})
2159 .to_string();
2160 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2161 let (queued, finished) = if held {
2162 (vec![], vec![waiting.clone()])
2163 } else {
2164 (vec![waiting.clone()], vec![])
2165 };
2166 Conductor::new()
2167 .run_once(
2168 &cfg,
2169 dir.path(),
2170 &queue,
2171 &questions,
2172 dir.path(),
2173 &queued,
2174 &[],
2175 &finished,
2176 2,
2177 )
2178 .await
2179 .unwrap();
2180 assert_eq!(
2181 serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2182 serde_json::to_value(&waiting).unwrap(),
2183 "unverifiable aliases must leave the entire task unchanged"
2184 );
2185 if unknown {
2186 assert!(questions.root().is_file());
2187 } else {
2188 assert!(questions.list().is_empty());
2189 }
2190 }
2191 }
2192 }
2193 }
2194 }
2195
2196 #[tokio::test]
2197 async fn dependency_resolution_genuine_missing_reference_still_asks() {
2198 let dir = tempdir().unwrap();
2199 let queue = Queue::at(dir.path().join("queue"));
2200 let questions = Questions::at(dir.path().join("questions"));
2201 let mut waiting = task("missing prerequisite");
2202 queue.put(&mut waiting).unwrap();
2203 let reply = serde_json::json!({"decisions": [{
2204 "id": waiting.id, "blocked_by": ["not-present"],
2205 "missing_dependencies": ["not-present"],
2206 "question": "Which prerequisite replaces the missing task?"
2207 }]})
2208 .to_string();
2209 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2210 Conductor::new()
2211 .run_once(
2212 &cfg,
2213 dir.path(),
2214 &queue,
2215 &questions,
2216 dir.path(),
2217 &[waiting.clone()],
2218 &[],
2219 &[],
2220 2,
2221 )
2222 .await
2223 .unwrap();
2224 let after = queue.get(&waiting.id).unwrap();
2225 assert_eq!(after.status, TaskStatus::Blocked);
2226 assert_eq!(
2227 questions.get(&after.blocked_by[0]).unwrap().summary,
2228 "Which prerequisite replaces the missing task?"
2229 );
2230 }
2231
2232 #[tokio::test]
2233 async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2234 let dir = tempdir().unwrap();
2235 let queue = Queue::at(dir.path().join("queue"));
2236 let questions = Questions::at(dir.path().join("questions"));
2237 let mut dep = task("prerequisite");
2238 dep.id = "20261013-193611-ca5b".into();
2239 dep.start("existing-run".into());
2240 queue.put(&mut dep).unwrap();
2241 let mut unrelated = task("unrelated work");
2242 unrelated.start("other-run".into());
2243 queue.put(&mut unrelated).unwrap();
2244 std::fs::File::options()
2247 .write(true)
2248 .open(queue.path_of(&unrelated.id))
2249 .unwrap()
2250 .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2251 .unwrap();
2252 unrelated.succeed();
2253 let mut waiting = task("waiting");
2254 let mut asking = task("needs a product decision");
2255 let mut holding = task("must stay held");
2256 queue.put(&mut waiting).unwrap();
2257 queue.put(&mut asking).unwrap();
2258 queue.put(&mut holding).unwrap();
2259 let reply = serde_json::json!({"decisions": [
2260 {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2261 {"id": asking.id, "question": "Which behavior do you want?"},
2262 {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2263 ]})
2264 .to_string();
2265 let mut vars = env(&reply);
2266 vars.insert(
2267 "UNRELATED_PATH".into(),
2268 queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2269 );
2270 vars.insert(
2271 "UNRELATED_RECORD".into(),
2272 serde_json::to_string(&unrelated).unwrap(),
2273 );
2274 let script = r#"#!/bin/sh
2275cat >/dev/null
2276printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2277printf '%s\n' "$MOCK_REPLY"
2278"#;
2279 let cfg = config(mock_agent(dir.path(), script, vars));
2280 Conductor::new()
2281 .run_once(
2282 &cfg,
2283 dir.path(),
2284 &queue,
2285 &questions,
2286 dir.path(),
2287 &[waiting.clone(), asking.clone(), holding.clone()],
2288 &[],
2289 &[],
2290 2,
2291 )
2292 .await
2293 .unwrap();
2294 assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2295 let after = queue.get(&waiting.id).unwrap();
2296 assert_eq!(after.status, TaskStatus::Blocked);
2297 assert_eq!(after.blocked_by, [dep.id]);
2298 let asked = queue.get(&asking.id).unwrap();
2299 assert_eq!(asked.status, TaskStatus::Blocked);
2300 assert_eq!(
2301 questions.get(&asked.blocked_by[0]).unwrap().summary,
2302 "Which behavior do you want?"
2303 );
2304 assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2305 }
2306
2307 #[test]
2308 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2309 let dir = tempdir().unwrap();
2310 let queue = Queue::at(dir.path().join("queue"));
2311 let questions = Questions::at(dir.path().join("questions"));
2312 let mut a = task("a");
2313 a.priority = 9;
2314 queue.put(&mut a).unwrap();
2315
2316 let verdict = Verdict {
2317 decisions: vec![Decision {
2318 id: a.id.clone(),
2319 blocked_by: vec!["20260101-000000-dead".to_owned()],
2320 reason: Some("waits on the other task".to_owned()),
2321 recovery: None,
2322 question: None,
2323 choices: Vec::new(),
2324 actions: BTreeMap::new(),
2325 question_decision: None,
2326 evidence: Vec::new(),
2327 missing_dependencies: Vec::new(),
2328 }],
2329 };
2330 apply(&queue, &questions, &verdict).unwrap();
2331
2332 let back = queue.get(&a.id).unwrap();
2333 assert_eq!(back.status, TaskStatus::Blocked);
2334 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2335 assert_eq!(
2336 back.priority, 9,
2337 "the conductor's reply cannot carry priority"
2338 );
2339 }
2340
2341 #[test]
2342 fn a_question_decision_files_one_and_blocks_on_its_id() {
2343 let dir = tempdir().unwrap();
2344 let queue = Queue::at(dir.path().join("queue"));
2345 let questions = Questions::at(dir.path().join("questions"));
2346 let mut t = task("ambiguous");
2347 queue.put(&mut t).unwrap();
2348
2349 let verdict = Verdict {
2350 decisions: vec![Decision {
2351 id: t.id.clone(),
2352 blocked_by: Vec::new(),
2353 reason: Some("which backend?".to_owned()),
2354 recovery: None,
2355 question: Some("Which storage backend?".to_owned()),
2356 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2357 actions: BTreeMap::new(),
2358 question_decision: None,
2359 evidence: Vec::new(),
2360 missing_dependencies: Vec::new(),
2361 }],
2362 };
2363 apply(&queue, &questions, &verdict).unwrap();
2364
2365 let back = queue.get(&t.id).unwrap();
2366 assert_eq!(back.status, TaskStatus::Blocked);
2367 assert_eq!(back.blocked_by.len(), 1);
2368 let q = questions.get(&back.blocked_by[0]).unwrap();
2369 assert_eq!(q.summary, "Which storage backend?");
2370 assert_eq!(q.node, NODE);
2371 assert!(q.status.open());
2372 }
2373
2374 #[test]
2375 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2376 let dir = tempdir().unwrap();
2377 let queue = Queue::at(dir.path().join("queue"));
2378 let questions = Questions::at(dir.path().join("questions"));
2379 let mut t = task("asked once");
2380 queue.put(&mut t).unwrap();
2381
2382 let decision = Decision {
2383 id: t.id.clone(),
2384 reason: Some("still deciding".to_owned()),
2385 question: Some("Which backend?".to_owned()),
2386 ..Decision::default()
2387 };
2388 apply(
2389 &queue,
2390 &questions,
2391 &Verdict {
2392 decisions: vec![decision.clone()],
2393 },
2394 )
2395 .unwrap();
2396 assert_eq!(questions.list().len(), 1);
2397 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2398
2399 let mut released = queue.get(&t.id).unwrap();
2404 released.release();
2405 queue.put(&mut released).unwrap();
2406
2407 apply(
2408 &queue,
2409 &questions,
2410 &Verdict {
2411 decisions: vec![decision],
2412 },
2413 )
2414 .unwrap();
2415 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2416 let after = queue.get(&t.id).unwrap();
2417 assert_eq!(
2418 after.blocked_by,
2419 [first_question_id],
2420 "the existing open question is reused, not replaced"
2421 );
2422 }
2423
2424 #[test]
2425 fn a_same_id_question_from_another_node_is_not_reused() {
2426 let dir = tempdir().unwrap();
2427 let queue = Queue::at(dir.path().join("queue"));
2428 let questions = Questions::at(dir.path().join("questions"));
2429 let mut t = task("must ask the conductor");
2430 queue.put(&mut t).unwrap();
2431
2432 let mut unrelated = Question::new(
2433 t.id.clone(),
2434 "review".to_owned(),
2435 "reviewer-1".to_owned(),
2436 "An unrelated review question".to_owned(),
2437 String::new(),
2438 Vec::new(),
2439 );
2440 questions.put(&mut unrelated).unwrap();
2441
2442 apply(
2443 &queue,
2444 &questions,
2445 &Verdict {
2446 decisions: vec![Decision {
2447 id: t.id.clone(),
2448 question: Some("Which backend?".to_owned()),
2449 ..Decision::default()
2450 }],
2451 },
2452 )
2453 .unwrap();
2454
2455 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2456 assert_eq!(blocked_by.len(), 1);
2457 assert_ne!(blocked_by[0], unrelated.id);
2458 assert!(questions.get(&unrelated.id).unwrap().status.open());
2459 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2460 }
2461
2462 #[test]
2463 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2464 let dir = tempdir().unwrap();
2465 let queue = Queue::at(dir.path().join("queue"));
2466 let questions = Questions::at(dir.path().join("questions"));
2467 let mut t = task("waits on an answer");
2468 queue.put(&mut t).unwrap();
2469
2470 apply(
2471 &queue,
2472 &questions,
2473 &Verdict {
2474 decisions: vec![Decision {
2475 id: t.id.clone(),
2476 blocked_by: Vec::new(),
2477 reason: None,
2478 recovery: None,
2479 question: Some("Which backend?".to_owned()),
2480 choices: Vec::new(),
2481 actions: BTreeMap::new(),
2482 question_decision: None,
2483 evidence: Vec::new(),
2484 missing_dependencies: Vec::new(),
2485 }],
2486 },
2487 )
2488 .unwrap();
2489 let blocked = queue.get(&t.id).unwrap();
2490 let question_id = blocked.blocked_by[0].clone();
2491
2492 let mut q = questions.get(&question_id).unwrap();
2493 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2494 questions.put(&mut q).unwrap();
2495 assert_eq!(q.status, QuestionStatus::Answered);
2496
2497 let mut task_after = queue.get(&t.id).unwrap();
2501 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2502 task_after.unblock(&question_id);
2503 assert_eq!(task_after.status, TaskStatus::Queued);
2504 assert_eq!(task_after.answers[0].answer, "SQLite");
2505 }
2506
2507 #[test]
2508 fn a_stalled_task_can_be_requeued_or_held() {
2509 let dir = tempdir().unwrap();
2510 let queue = Queue::at(dir.path().join("queue"));
2511 let questions = Questions::at(dir.path().join("questions"));
2512
2513 let mut requeue_me = task("stuck a");
2514 requeue_me.start("run-1".to_owned());
2515 queue.put(&mut requeue_me).unwrap();
2516
2517 let mut hold_me = task("stuck b");
2518 hold_me.start("run-2".to_owned());
2519 queue.put(&mut hold_me).unwrap();
2520
2521 apply(
2522 &queue,
2523 &questions,
2524 &Verdict {
2525 decisions: vec![
2526 Decision {
2527 id: requeue_me.id.clone(),
2528 recovery: Some(Recovery::Requeue),
2529 ..Decision::default()
2530 },
2531 Decision {
2532 id: hold_me.id.clone(),
2533 recovery: Some(Recovery::Hold),
2534 reason: Some("looks broken".to_owned()),
2535 ..Decision::default()
2536 },
2537 ],
2538 },
2539 )
2540 .unwrap();
2541
2542 let requeued = queue.get(&requeue_me.id).unwrap();
2543 assert_eq!(requeued.status, TaskStatus::Queued);
2544 assert_eq!(requeued.attempts, 0);
2545
2546 let held = queue.get(&hold_me.id).unwrap();
2547 assert_eq!(held.status, TaskStatus::Held);
2548 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2549 }
2550
2551 #[test]
2552 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2553 let dir = tempdir().unwrap();
2558 let queue = Queue::at(dir.path().join("queue"));
2559 let questions = Questions::at(dir.path().join("questions"));
2560 let mut t = task("held out of attempts");
2561 t.hold_machine(Some("out of attempts".to_owned()));
2562 queue.put(&mut t).unwrap();
2563
2564 apply(
2565 &queue,
2566 &questions,
2567 &Verdict {
2568 decisions: vec![Decision {
2569 id: t.id.clone(),
2570 reason: Some("what should happen to this one?".to_owned()),
2571 question: Some("Hold it, or try again?".to_owned()),
2572 ..Decision::default()
2573 }],
2574 },
2575 )
2576 .unwrap();
2577 let blocked = queue.get(&t.id).unwrap();
2578 assert_eq!(blocked.status, TaskStatus::Blocked);
2579 let question_id = blocked.blocked_by[0].clone();
2580
2581 let mut q = questions.get(&question_id).unwrap();
2582 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2583 questions.put(&mut q).unwrap();
2584
2585 let mut after = queue.get(&t.id).unwrap();
2587 after.record_answer(q.summary.clone(), "leave it held".to_owned());
2588 after.unblock(&question_id);
2589 assert_eq!(
2590 after.status,
2591 TaskStatus::Held,
2592 "must not fall back to queued"
2593 );
2594 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2595 }
2596
2597 #[test]
2598 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2599 let dir = tempdir().unwrap();
2609 let queue = Queue::at(dir.path().join("queue"));
2610 let questions = Questions::at(dir.path().join("questions"));
2611 let mut t = task("disk pressure, then reconsidered");
2612 t.hold_machine(Some(
2613 "not enough free space to start a run: 10 bytes free, 100 required by \
2614 `[disk] min_free_bytes`"
2615 .to_owned(),
2616 ));
2617 t.record_answer(
2618 "How should this be handled?".to_owned(),
2619 "keep it held, a human will look at it later".to_owned(),
2620 );
2621 queue.put(&mut t).unwrap();
2622
2623 apply(
2624 &queue,
2625 &questions,
2626 &Verdict {
2627 decisions: vec![Decision {
2628 id: t.id.clone(),
2629 recovery: Some(Recovery::Hold),
2630 ..Decision::default()
2631 }],
2632 },
2633 )
2634 .unwrap();
2635
2636 let after = queue.get(&t.id).unwrap();
2637 assert_eq!(after.status, TaskStatus::Held);
2638 assert!(
2639 !after
2640 .hold_reason
2641 .as_deref()
2642 .unwrap_or_default()
2643 .starts_with("not enough free space"),
2644 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2645 after.hold_reason
2646 );
2647
2648 let cfg_dir = tempdir().unwrap();
2651 let config = cfg_dir.path().join("magi.toml");
2652 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2653 let report =
2654 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2655 assert!(report.resumed.is_empty(), "must not be auto-released");
2656 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2657 }
2658
2659 #[test]
2660 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2661 crate::run::pin_test_home();
2662 let dir = tempdir().unwrap();
2663 let queue = Queue::at(dir.path().join("queue"));
2664 let questions = Questions::at(dir.path().join("questions"));
2665 let config = dir.path().join("magi.toml");
2666 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2667 let now = jiff::Timestamp::now();
2668 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2669 let open = || {
2670 questions
2671 .list()
2672 .into_iter()
2673 .filter(|q| q.node == "triage" && q.status.open())
2674 .collect::<Vec<_>>()
2675 };
2676 let hold = Decision {
2677 recovery: Some(Recovery::Hold),
2678 reason: Some("waiting on manual worktree cleanup".to_owned()),
2679 ..Decision::default()
2680 };
2681
2682 let mut t = task("looping hold");
2683 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2684 queue.put(&mut t).unwrap();
2685 let hold = Decision {
2686 id: t.id.clone(),
2687 ..hold
2688 };
2689
2690 assert_eq!(triage().asked.len(), 1);
2692 let first = open().remove(0);
2693 let mut q = questions.get(&first.id).unwrap();
2694 let resume = q.choices[0].clone();
2695 q.answer(Answer::Choice(resume)).unwrap();
2696 questions.put(&mut q).unwrap();
2697 assert_eq!(triage().answered.len(), 1);
2698 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2699
2700 apply(
2702 &queue,
2703 &questions,
2704 &Verdict {
2705 decisions: vec![hold.clone()],
2706 },
2707 )
2708 .unwrap();
2709 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2710 assert!(triage().asked.is_empty());
2711
2712 let mut evidence = Question::new(
2714 t.id.clone(),
2715 "preflight".into(),
2716 "check".into(),
2717 "New permission failure".into(),
2718 String::new(),
2719 Vec::new(),
2720 );
2721 questions.put(&mut evidence).unwrap();
2722 let hold = Decision {
2723 evidence: vec![evidence.id.clone()],
2724 reason: Some("new permission check failed".into()),
2725 ..hold
2726 };
2727 apply(
2728 &queue,
2729 &questions,
2730 &Verdict {
2731 decisions: vec![hold.clone()],
2732 },
2733 )
2734 .unwrap();
2735 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2736
2737 assert_eq!(triage().asked.len(), 1);
2739 let second = open().remove(0);
2740 assert!(second.detail.contains("new permission check failed"));
2741 assert!(triage().asked.is_empty(), "no duplicate question");
2742 assert_eq!(open().len(), 1);
2743
2744 let mut q = questions.get(&second.id).unwrap();
2746 let force = q.choices[0].clone();
2747 q.answer(Answer::Choice(force)).unwrap();
2748 questions.put(&mut q).unwrap();
2749 assert_eq!(triage().answered.len(), 1);
2750 apply(
2751 &queue,
2752 &questions,
2753 &Verdict {
2754 decisions: vec![hold],
2755 },
2756 )
2757 .unwrap();
2758 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2759 }
2760
2761 #[test]
2762 fn conductor_resumption_notice_does_not_solicit_unregistered_questions() {
2763 crate::run::pin_test_home();
2764 let dir = tempdir().unwrap();
2765 let queue = Queue::at(dir.path().join("queue"));
2766 let questions = Questions::at(dir.path().join("questions"));
2767
2768 let mut t = task("task with explicit resumption");
2769 t.status = TaskStatus::Queued;
2770 t.resumption = Some(crate::queue::Resumption {
2771 generation: 1,
2772 source: "test".into(),
2773 question_id: Some("q1".into()),
2774 at: jiff::Timestamp::now(),
2775 instruction: "resume".into(),
2776 answers_through: 0,
2777 runs_through: 0,
2778 cleared_reason: None,
2779 blocking_evidence: Vec::new(),
2780 });
2781 queue.put(&mut t).unwrap();
2782
2783 let decision = Decision {
2785 id: t.id.clone(),
2786 question: Some("How should we proceed with this task?".into()),
2787 blocked_by: vec!["dep-x".into()],
2788 reason: Some("needs clarification".into()),
2789 recovery: Some(Recovery::Hold),
2790 ..Decision::default()
2791 };
2792
2793 apply(
2794 &queue,
2795 &questions,
2796 &Verdict {
2797 decisions: vec![decision],
2798 },
2799 )
2800 .unwrap();
2801
2802 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2804 assert!(questions.list().is_empty());
2806
2807 let notices = crate::notices::Notices::open();
2809 let notice = notices
2810 .get(&crate::notices::id_of(&format!(
2811 "conductor-resumption:{}",
2812 t.id
2813 )))
2814 .expect("conductor-resumption notice filed");
2815
2816 assert_eq!(
2817 notice.link,
2818 Some(crate::notices::Link::Task { id: t.id.clone() })
2819 );
2820 assert!(!notice.message.contains("How should we proceed"));
2822 assert!(
2824 notice
2825 .message
2826 .contains("No reply to this notice is needed.")
2827 );
2828 assert!(!notice.message.contains("Some("));
2830 assert!(!notice.message.contains("None"));
2831 assert!(!notice.message.contains("[]"));
2832 assert!(!notice.message.contains("{:?}"));
2833 assert!(notice.message.contains("Proposed dependencies: dep-x."));
2835 }
2836
2837 #[test]
2838 fn a_runnable_task_can_be_held_directly_without_a_question() {
2839 let dir = tempdir().unwrap();
2840 let queue = Queue::at(dir.path().join("queue"));
2841 let questions = Questions::at(dir.path().join("questions"));
2842 let mut t = task("already answered, should stay put");
2843 queue.put(&mut t).unwrap();
2844
2845 apply(
2846 &queue,
2847 &questions,
2848 &Verdict {
2849 decisions: vec![Decision {
2850 id: t.id.clone(),
2851 recovery: Some(Recovery::Hold),
2852 reason: Some("operator already said keep this held".to_owned()),
2853 ..Decision::default()
2854 }],
2855 },
2856 )
2857 .unwrap();
2858
2859 let after = queue.get(&t.id).unwrap();
2860 assert_eq!(after.status, TaskStatus::Held);
2861 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2862 }
2863
2864 #[test]
2865 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2866 crate::run::pin_test_home();
2867 let dir = tempdir().unwrap();
2873 let queue = Queue::at(dir.path().join("queue"));
2874 let questions = Questions::at(dir.path().join("questions"));
2875 let mut t = task("already merged by hand");
2876 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2877 t.record_answer(
2878 "Handle this one?".to_owned(),
2879 "already merged and cleaned up, close it".to_owned(),
2880 );
2881 queue.put(&mut t).unwrap();
2882
2883 apply(
2884 &queue,
2885 &questions,
2886 &Verdict {
2887 decisions: vec![Decision {
2888 id: t.id.clone(),
2889 recovery: Some(Recovery::Done),
2890 reason: Some("operator confirmed this already landed".to_owned()),
2891 ..Decision::default()
2892 }],
2893 },
2894 )
2895 .unwrap();
2896
2897 let after = queue.get(&t.id).unwrap();
2898 assert_eq!(after.status, TaskStatus::Done);
2899 assert!(after.hold_reason.is_none());
2900 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2901 }
2902
2903 #[test]
2904 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2905 let dir = tempdir().unwrap();
2906 let queue = Queue::at(dir.path().join("queue"));
2907 let questions = Questions::at(dir.path().join("questions"));
2908
2909 let mut queued = task("never ran yet");
2910 queue.put(&mut queued).unwrap();
2911
2912 let mut running = task("mid-run");
2913 running.start("run-1".to_owned());
2914 queue.put(&mut running).unwrap();
2915
2916 for id in [queued.id.clone(), running.id.clone()] {
2917 apply(
2918 &queue,
2919 &questions,
2920 &Verdict {
2921 decisions: vec![Decision {
2922 id,
2923 recovery: Some(Recovery::Done),
2924 ..Decision::default()
2925 }],
2926 },
2927 )
2928 .unwrap();
2929 }
2930
2931 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2932 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2933 }
2934
2935 #[test]
2936 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2937 let dir = tempdir().unwrap();
2940 let queue = Queue::at(dir.path().join("queue"));
2941 let questions = Questions::at(dir.path().join("questions"));
2942 let mut t = task("asked about repeatedly");
2943 t.hold_machine(Some("out of attempts".to_owned()));
2944 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2945 t.record_answer(
2946 "Handle this one? (2)".to_owned(),
2947 "still not yet".to_owned(),
2948 );
2949 queue.put(&mut t).unwrap();
2950 assert_eq!(questions.list().len(), 0);
2951
2952 let verdict = Verdict {
2953 decisions: vec![Decision {
2954 id: t.id.clone(),
2955 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2956 ..Decision::default()
2957 }],
2958 };
2959 apply(&queue, &questions, &verdict).unwrap();
2960
2961 let filed = questions.list();
2962 assert_eq!(filed.len(), 1, "the question was filed");
2963 assert_eq!(
2964 filed[0].summary,
2965 "Branch conflicts with origin/main, how do we proceed?"
2966 );
2967 let deputy = filed[0]
2968 .deputy
2969 .as_ref()
2970 .expect("the wait is handed to a deputy");
2971 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2972 assert_eq!(
2973 deputy.starts, 0,
2974 "filing never starts anything: the loop does not wait"
2975 );
2976 assert!(filed[0].status.open());
2977 assert_eq!(filed[0].node, NODE);
2978 let after = queue.get(&t.id).unwrap();
2979 assert_eq!(after.status, TaskStatus::Blocked);
2980 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2981 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2982 assert!(
2983 !after
2984 .hold_reason
2985 .clone()
2986 .unwrap_or_default()
2987 .contains("conduct tried to ask"),
2988 "no hold was applied"
2989 );
2990
2991 apply(&queue, &questions, &verdict).unwrap();
2992 let again = questions.list();
2993 assert_eq!(again.len(), 1, "the open question is reused");
2994 assert_eq!(again[0].id, filed[0].id);
2995 }
2996
2997 #[test]
2998 fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
2999 let dir = tempdir().unwrap();
3000 init_repo_with_branch(dir.path(), "source-a");
3001 let mut t = task("source decision");
3002 t.repo = dir.path().into();
3003 let decision = |reference: &str| QuestionDecision {
3004 operation: QuestionOperation::Recover,
3005 target: QuestionTarget::Source {
3006 reference: reference.into(),
3007 },
3008 };
3009 assert_eq!(
3010 decision("source-a").identity(&t).unwrap(),
3011 decision("refs/heads/source-a").identity(&t).unwrap()
3012 );
3013 assert_ne!(
3014 decision("source-a").identity(&t).unwrap(),
3015 decision("main").identity(&t).unwrap()
3016 );
3017 assert!(decision("invented-source-label").identity(&t).is_err());
3018 let unrelated = QuestionDecision {
3019 operation: QuestionOperation::Resume,
3020 target: QuestionTarget::Run {
3021 id: "unrelated-run".into(),
3022 },
3023 };
3024 assert!(unrelated.identity(&t).is_err());
3025 }
3026
3027 #[test]
3028 fn invalid_question_targets_still_file_and_block_queued_tasks() {
3029 let dir = tempdir().unwrap();
3030 init_repo_with_branch(dir.path(), "source-a");
3031 let queue = Queue::at(dir.path().join("queue"));
3032 let questions = Questions::at(dir.path().join("questions"));
3033 for target in [
3034 QuestionTarget::Source {
3035 reference: "missing-source".into(),
3036 },
3037 QuestionTarget::Run {
3038 id: "unrelated-run".into(),
3039 },
3040 ] {
3041 let mut t = task("decision with invalid metadata");
3042 t.repo = dir.path().into();
3043 queue.put(&mut t).unwrap();
3044 let d = Decision {
3045 id: t.id.clone(),
3046 question: Some("Which recovery should proceed?".into()),
3047 choices: vec!["inspect first".into(), "keep waiting".into()],
3048 question_decision: Some(QuestionDecision {
3049 operation: QuestionOperation::Recover,
3050 target,
3051 }),
3052 ..Decision::default()
3053 };
3054 apply_one(&queue, &questions, &d, None).unwrap();
3055 let q = questions
3056 .list()
3057 .into_iter()
3058 .find(|q| q.run == t.id)
3059 .unwrap();
3060 assert_eq!(q.summary, d.question.as_deref().unwrap());
3061 assert!(
3062 q.choices.iter().all(|c| q.actions.contains_key(c)),
3063 "manual-only choices must become an executable hold"
3064 );
3065 assert!(q.status.open());
3066 let context: serde_json::Value =
3067 serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
3068 assert!(context[3].is_null(), "invalid metadata was discarded");
3069 let after = queue.get(&t.id).unwrap();
3070 assert_eq!(after.status, TaskStatus::Blocked);
3071 assert_eq!(after.blocked_by, vec![q.id.clone()]);
3072 apply_one(&queue, &questions, &d, None).unwrap();
3073 assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
3074 }
3075 }
3076
3077 #[test]
3078 fn answered_conductor_question_needs_changed_facts_before_reasking() {
3079 let dir = tempdir().unwrap();
3080 let queue = Queue::at(dir.path().join("queue"));
3081 let questions = Questions::at(dir.path().join("questions"));
3082 let mut t = task("recovery decision");
3083 t.hold_machine(Some("out of attempts".into()));
3084 t.runs = vec!["source-run-a".into(), "source-run-b".into()];
3085 queue.put(&mut t).unwrap();
3086 let mut d = Decision {
3087 id: t.id.clone(),
3088 question: Some("Recover from source A?".into()),
3089 choices: vec!["approve source A".into(), "keep held".into()],
3090 question_decision: Some(QuestionDecision {
3091 operation: QuestionOperation::Recover,
3092 target: QuestionTarget::Run {
3093 id: "source-run-a".into(),
3094 },
3095 }),
3096 ..Decision::default()
3097 };
3098 apply_one(&queue, &questions, &d, None).unwrap();
3099 let mut q = questions.list().pop().unwrap();
3100 q.answer(crate::ask::Answer::Choice(q.choices[0].clone()))
3101 .unwrap();
3102 questions.put(&mut q).unwrap();
3103 queue
3104 .modify(&t.id, |task| {
3105 task.unblock(&q.id);
3106 task.hold_machine(Some("recovery remains under consideration".into()));
3107 true
3108 })
3109 .unwrap();
3110 d.question = Some("Should we keep waiting on recovery?".into());
3111 d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3112 apply_one(&queue, &questions, &d, None).unwrap();
3113 assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3114 d.question = Some("Recover from source B?".into());
3115 d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3117 id: "source-run-b".into(),
3118 };
3119 apply_one(&queue, &questions, &d, None).unwrap();
3120 assert_eq!(
3121 questions.list().len(),
3122 2,
3123 "new decision with unchanged task facts"
3124 );
3125 let after = queue.get(&t.id).unwrap();
3126 assert_eq!(after.instruction, t.instruction);
3127 assert_eq!(after.runs, t.runs);
3128 assert_eq!(after.status, TaskStatus::Blocked);
3129 }
3130
3131 #[test]
3132 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3133 let dir = tempdir().unwrap();
3134 let queue = Queue::at(dir.path().join("queue"));
3135 let questions = Questions::at(dir.path().join("questions"));
3136 let mut t = task("asked about once already");
3137 t.hold_machine(Some("out of attempts".to_owned()));
3138 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3139 queue.put(&mut t).unwrap();
3140
3141 apply(
3142 &queue,
3143 &questions,
3144 &Verdict {
3145 decisions: vec![Decision {
3146 id: t.id.clone(),
3147 question: Some("Still not sure - now what?".to_owned()),
3148 ..Decision::default()
3149 }],
3150 },
3151 )
3152 .unwrap();
3153
3154 assert_eq!(questions.list().len(), 1, "the second question was filed");
3155 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3156 }
3157
3158 #[test]
3159 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3160 let dir = tempdir().unwrap();
3166 let queue = Queue::at(dir.path().join("queue"));
3167 let questions = Questions::at(dir.path().join("questions"));
3168 let mut t = task("held, triage already asking about it");
3169 t.hold_machine(Some("cause unclear".to_owned()));
3170 queue.put(&mut t).unwrap();
3171
3172 let mut triage_q = Question::new(
3173 t.id.clone(),
3174 crate::triage::NODE.to_owned(),
3175 "triage".to_owned(),
3176 "Still needed?".to_owned(),
3177 String::new(),
3178 vec![
3179 "resume".to_owned(),
3180 "not yet".to_owned(),
3181 "discard".to_owned(),
3182 ],
3183 );
3184 questions.put(&mut triage_q).unwrap();
3185
3186 for decision in [
3187 Decision {
3188 id: t.id.clone(),
3189 question: Some("what now?".to_owned()),
3190 ..Decision::default()
3191 },
3192 Decision {
3193 id: t.id.clone(),
3194 recovery: Some(Recovery::Requeue),
3195 ..Decision::default()
3196 },
3197 ] {
3198 apply(
3199 &queue,
3200 &questions,
3201 &Verdict {
3202 decisions: vec![decision],
3203 },
3204 )
3205 .unwrap();
3206 }
3207
3208 let after = queue.get(&t.id).unwrap();
3209 assert_eq!(
3210 after.status,
3211 TaskStatus::Held,
3212 "triage still owns this hold"
3213 );
3214 assert!(after.blocked_by.is_empty());
3215 assert_eq!(
3216 questions.list().len(),
3217 1,
3218 "no second, conductor-owned question was filed"
3219 );
3220 }
3221
3222 #[test]
3223 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3224 let dir = tempdir().unwrap();
3232 let queue = Queue::at(dir.path().join("queue"));
3233 let questions = Questions::at(dir.path().join("questions"));
3234 let mut t = task("held, triage question answered but not yet applied");
3235 t.hold_machine(Some("cause unclear".to_owned()));
3236 queue.put(&mut t).unwrap();
3237
3238 let mut triage_q = Question::new(
3239 t.id.clone(),
3240 crate::triage::NODE.to_owned(),
3241 "triage".to_owned(),
3242 "Still needed?".to_owned(),
3243 String::new(),
3244 vec![
3245 "resume".to_owned(),
3246 "not yet".to_owned(),
3247 "discard".to_owned(),
3248 ],
3249 );
3250 questions.put(&mut triage_q).unwrap();
3251 triage_q
3252 .answer(Answer::Choice("not yet".to_owned()))
3253 .unwrap();
3254 questions.put(&mut triage_q).unwrap();
3255 assert!(!triage_q.status.open());
3256
3257 apply(
3258 &queue,
3259 &questions,
3260 &Verdict {
3261 decisions: vec![Decision {
3262 id: t.id.clone(),
3263 question: Some("what now?".to_owned()),
3264 ..Decision::default()
3265 }],
3266 },
3267 )
3268 .unwrap();
3269
3270 let after = queue.get(&t.id).unwrap();
3271 assert_eq!(
3272 after.status,
3273 TaskStatus::Held,
3274 "triage's own answer is not yet applied - conduct must wait"
3275 );
3276 assert_eq!(
3277 questions.list().len(),
3278 1,
3279 "no conductor question was filed over the pending triage answer"
3280 );
3281 }
3282
3283 #[test]
3284 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3285 let dir = tempdir().unwrap();
3286 let queue = Queue::at(dir.path().join("queue"));
3287 let questions = Questions::at(dir.path().join("questions"));
3288 let mut held = task("manual recovery");
3289 held.priority = 300;
3290 held.runs.push("run20260912-224242-daf5".to_owned());
3291 held.hold_manual(Some(
3292 "active manual recovery run20260912-224242-daf5".to_owned(),
3293 ));
3294 queue.put(&mut held).unwrap();
3295
3296 for decision in [
3300 Decision {
3301 id: held.id.clone(),
3302 recovery: Some(Recovery::Requeue),
3303 ..Decision::default()
3304 },
3305 Decision {
3306 id: held.id.clone(),
3307 recovery: Some(Recovery::Hold),
3308 reason: Some("stale replacement reason".to_owned()),
3309 ..Decision::default()
3310 },
3311 Decision {
3312 id: held.id.clone(),
3313 recovery: Some(Recovery::Review),
3314 ..Decision::default()
3315 },
3316 Decision {
3317 id: held.id.clone(),
3318 blocked_by: vec!["other-task".to_owned()],
3319 question: Some("retry now?".to_owned()),
3320 ..Decision::default()
3321 },
3322 ] {
3323 apply(
3324 &queue,
3325 &questions,
3326 &Verdict {
3327 decisions: vec![decision],
3328 },
3329 )
3330 .unwrap();
3331 }
3332
3333 let after = queue.get(&held.id).unwrap();
3334 assert_eq!(after.status, TaskStatus::Held);
3335 assert!(after.operator_held());
3336 assert_eq!(after.priority, 300);
3337 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3338 assert_eq!(
3339 after.hold_reason.as_deref(),
3340 Some("active manual recovery run20260912-224242-daf5")
3341 );
3342 assert!(after.blocked_by.is_empty());
3343 assert!(questions.list().is_empty());
3344 assert!(
3345 queue.next_runnable().is_none(),
3346 "must not dispatch a duplicate"
3347 );
3348 }
3349
3350 #[test]
3351 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3352 let dir = tempdir().unwrap();
3353 let queue = Queue::at(dir.path().join("queue"));
3354 let questions = Questions::at(dir.path().join("questions"));
3355
3356 let mut automatic = task("disk gate");
3357 automatic.hold_machine(Some("disk full".to_owned()));
3358 queue.put(&mut automatic).unwrap();
3359 let requeue = || Verdict {
3360 decisions: vec![Decision {
3361 id: automatic.id.clone(),
3362 recovery: Some(Recovery::Requeue),
3363 ..Decision::default()
3364 }],
3365 };
3366 apply(&queue, &questions, &requeue()).unwrap();
3367 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3368
3369 let mut manual = task("operator gate");
3370 manual.hold_manual(Some("wait for operator".to_owned()));
3371 queue.put(&mut manual).unwrap();
3372 apply(
3373 &queue,
3374 &questions,
3375 &Verdict {
3376 decisions: vec![Decision {
3377 id: manual.id.clone(),
3378 recovery: Some(Recovery::Requeue),
3379 ..Decision::default()
3380 }],
3381 },
3382 )
3383 .unwrap();
3384 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3385
3386 let mut released = queue.get(&manual.id).unwrap();
3389 released.release();
3390 queue.put(&mut released).unwrap();
3391 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3392 }
3393
3394 #[test]
3395 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3396 let dir = tempdir().unwrap();
3397 let queue = Queue::at(dir.path().join("queue"));
3398 let questions = Questions::at(dir.path().join("questions"));
3399 let mut legacy = task("old explicit hold");
3400 legacy.status = TaskStatus::Held;
3401 legacy.hold_reason = Some("manual recovery already active".to_owned());
3402 legacy.hold_source = None;
3403 legacy.blocked_by = vec!["dependency".to_owned()];
3404 queue.put(&mut legacy).unwrap();
3405
3406 apply(
3407 &queue,
3408 &questions,
3409 &Verdict {
3410 decisions: vec![Decision {
3411 id: legacy.id.clone(),
3412 recovery: Some(Recovery::Requeue),
3413 ..Decision::default()
3414 }],
3415 },
3416 )
3417 .unwrap();
3418
3419 let after = queue.get(&legacy.id).unwrap();
3420 assert_eq!(after.status, TaskStatus::Held);
3421 assert_eq!(after.hold_source, None);
3422 assert_eq!(after.hold_reason, legacy.hold_reason);
3423 assert_eq!(after.blocked_by, legacy.blocked_by);
3424 }
3425
3426 #[test]
3427 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3428 crate::run::pin_test_home();
3434 let dir = tempdir().unwrap();
3435 let queue = Queue::at(dir.path().join("queue"));
3436 let questions = Questions::at(dir.path().join("questions"));
3437 let mut t = task("blocked with no readable run");
3438 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
3440 queue.put(&mut t).unwrap();
3441
3442 apply(
3443 &queue,
3444 &questions,
3445 &Verdict {
3446 decisions: vec![Decision {
3447 id: t.id.clone(),
3448 recovery: Some(Recovery::Review),
3449 ..Decision::default()
3450 }],
3451 },
3452 )
3453 .unwrap();
3454
3455 let after = queue.get(&t.id).unwrap();
3456 assert_eq!(
3457 after.status,
3458 TaskStatus::Failed,
3459 "with nothing to reopen, the decision is dropped rather than guessed at"
3460 );
3461 assert!(after.review_branch.is_none());
3462 }
3463
3464 #[test]
3465 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3466 let dir = tempdir().unwrap();
3470 let queue = Queue::at(dir.path().join("queue"));
3471 let questions = Questions::at(dir.path().join("questions"));
3472
3473 for recovery in [Recovery::Requeue, Recovery::Review] {
3474 let mut t = task("ordinary");
3475 queue.put(&mut t).unwrap();
3476
3477 apply(
3478 &queue,
3479 &questions,
3480 &Verdict {
3481 decisions: vec![Decision {
3482 id: t.id.clone(),
3483 recovery: Some(recovery),
3484 ..Decision::default()
3485 }],
3486 },
3487 )
3488 .unwrap();
3489
3490 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3491 }
3492 }
3493
3494 #[tokio::test]
3495 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3496 let dir = tempdir().unwrap();
3497 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3498 let queue = Queue::at(dir.path().join("queue"));
3499 let questions = Questions::at(dir.path().join("questions"));
3500 let mut t = task("normal");
3501 queue.put(&mut t).unwrap();
3502
3503 let mut conductor = Conductor::new();
3504 conductor
3505 .maybe_run(
3506 &cfg,
3507 dir.path(),
3508 &queue,
3509 &questions,
3510 dir.path(),
3511 &[t.clone()],
3512 &[],
3513 &[],
3514 2,
3515 )
3516 .await;
3517
3518 assert_eq!(
3519 queue.get(&t.id).unwrap().status,
3520 TaskStatus::Queued,
3521 "a failed invocation must change nothing"
3522 );
3523 assert!(
3524 queue.next_runnable().is_some(),
3525 "the loop must still be able to take the next task"
3526 );
3527 }
3528
3529 #[tokio::test]
3530 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3531 let dir = tempdir().unwrap();
3532 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3533 let queue = Queue::at(dir.path().join("queue"));
3534 let questions = Questions::at(dir.path().join("questions"));
3535 let mut t = task("normal");
3536 queue.put(&mut t).unwrap();
3537
3538 let mut conductor = Conductor::new();
3539 conductor
3540 .maybe_run(
3541 &cfg,
3542 dir.path(),
3543 &queue,
3544 &questions,
3545 dir.path(),
3546 &[t.clone()],
3547 &[],
3548 &[],
3549 2,
3550 )
3551 .await;
3552
3553 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3554 }
3555
3556 #[tokio::test]
3557 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3558 let dir = tempdir().unwrap();
3559 let mut t = task("chained");
3560 let reply = format!(
3561 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3562 t.id
3563 );
3564 let counted = |id: &str, tail: &str| {
3565 let calls = dir.path().join(format!("{id}.calls"));
3566 let path = dir.path().join(format!("{id}.sh"));
3567 std::fs::write(
3568 &path,
3569 format!(
3570 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3571 calls.display()
3572 ),
3573 )
3574 .unwrap();
3575 AgentSpec {
3576 id: id.to_owned(),
3577 kind: AgentKind::Command,
3578 model: None,
3579 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3580 extra_args: Vec::new(),
3581 env: BTreeMap::new(),
3582 prompt_delivery: None,
3583 }
3584 };
3585 let n_calls = |id: &str| {
3586 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3587 .map_or(0, |s| s.lines().count())
3588 };
3589 let mut cfg = config(counted("a", "exit 3"));
3590 cfg.agents = vec![
3591 counted("a", "exit 3"),
3592 counted("b", &format!("printf '%s' '{reply}'")),
3593 ];
3594 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3595 "a".into(),
3596 "b".into(),
3597 "a".into(),
3598 ]));
3599 let queue = Queue::at(dir.path().join("queue"));
3600 let questions = Questions::at(dir.path().join("questions"));
3601 queue.put(&mut t).unwrap();
3602
3603 let mut conductor = Conductor::new();
3604 conductor
3605 .maybe_run(
3606 &cfg,
3607 dir.path(),
3608 &queue,
3609 &questions,
3610 dir.path(),
3611 &[t.clone()],
3612 &[],
3613 &[],
3614 2,
3615 )
3616 .await;
3617
3618 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3619 assert_eq!(n_calls("a"), 1, "each id is tried once");
3620 assert_eq!(n_calls("b"), 1);
3621 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3622 assert!(
3623 saved.contains("\"b\""),
3624 "the seat on disk is the one that ran: {saved}"
3625 );
3626 }
3627
3628 #[tokio::test]
3629 async fn json_survives_code_fences_and_a_preamble() {
3630 let dir = tempdir().unwrap();
3631 let mut t = task("fenced");
3632 let reply = format!(
3633 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3634 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3635 t.id
3636 );
3637 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3638 let queue = Queue::at(dir.path().join("queue"));
3639 let questions = Questions::at(dir.path().join("questions"));
3640 queue.put(&mut t).unwrap();
3641
3642 let mut conductor = Conductor::new();
3643 conductor
3644 .maybe_run(
3645 &cfg,
3646 dir.path(),
3647 &queue,
3648 &questions,
3649 dir.path(),
3650 &[t.clone()],
3651 &[],
3652 &[],
3653 2,
3654 )
3655 .await;
3656
3657 let back = queue.get(&t.id).unwrap();
3658 assert_eq!(back.status, TaskStatus::Blocked);
3659 assert_eq!(back.blocked_by, ["x"]);
3660 }
3661
3662 #[tokio::test]
3663 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3664 let dir = tempdir().unwrap();
3667 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3668 let queue = Queue::at(dir.path().join("queue"));
3669 let questions = Questions::at(dir.path().join("questions"));
3670 let mut t = task("stable");
3671 queue.put(&mut t).unwrap();
3672 let artifacts = dir.path().join("conduct").join("artifacts");
3673 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3674
3675 let mut conductor = Conductor::new();
3676 conductor
3677 .maybe_run(
3678 &cfg,
3679 dir.path(),
3680 &queue,
3681 &questions,
3682 dir.path(),
3683 &[t.clone()],
3684 &[],
3685 &[],
3686 2,
3687 )
3688 .await;
3689 assert!(turn(1).is_file(), "the first cycle must call the conductor");
3690
3691 conductor
3692 .maybe_run(
3693 &cfg,
3694 dir.path(),
3695 &queue,
3696 &questions,
3697 dir.path(),
3698 &[t.clone()],
3699 &[],
3700 &[],
3701 2,
3702 )
3703 .await;
3704 assert!(
3705 !turn(2).is_file(),
3706 "an unchanged revision and an unchanged stalled/finished set must not call the \
3707 conductor twice"
3708 );
3709
3710 t.priority = 1;
3712 queue.put(&mut t).unwrap();
3713 conductor
3714 .maybe_run(
3715 &cfg,
3716 dir.path(),
3717 &queue,
3718 &questions,
3719 dir.path(),
3720 &[t.clone()],
3721 &[],
3722 &[],
3723 2,
3724 )
3725 .await;
3726 assert!(turn(2).is_file(), "a moved revision calls it again");
3727 }
3728
3729 #[tokio::test]
3730 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3731 let dir = tempdir().unwrap();
3737 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3738 let queue = Queue::at(dir.path().join("queue"));
3739 let questions = Questions::at(dir.path().join("questions"));
3740 let mut t = task("quiet");
3741 queue.put(&mut t).unwrap();
3742 let artifacts = dir.path().join("conduct").join("artifacts");
3743 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3744
3745 let mut conductor = Conductor::new();
3746 conductor
3747 .maybe_run(
3748 &cfg,
3749 dir.path(),
3750 &queue,
3751 &questions,
3752 dir.path(),
3753 &[t.clone()],
3754 &[],
3755 &[],
3756 2,
3757 )
3758 .await;
3759 assert!(turn(1).is_file());
3760
3761 conductor
3762 .maybe_run(
3763 &cfg,
3764 dir.path(),
3765 &queue,
3766 &questions,
3767 dir.path(),
3768 &[],
3769 &[t.clone()],
3770 &[],
3771 2,
3772 )
3773 .await;
3774 assert!(
3775 turn(2).is_file(),
3776 "a task turning stalled must call the conductor again"
3777 );
3778
3779 conductor
3782 .maybe_run(
3783 &cfg,
3784 dir.path(),
3785 &queue,
3786 &questions,
3787 dir.path(),
3788 &[],
3789 &[t.clone()],
3790 &[],
3791 2,
3792 )
3793 .await;
3794 assert!(
3795 !turn(3).is_file(),
3796 "the same stalled task lingering must not call the conductor every cycle"
3797 );
3798 }
3799
3800 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3801 apply(
3802 queue,
3803 questions,
3804 &Verdict {
3805 decisions: vec![Decision {
3806 id: id.to_owned(),
3807 reason: Some(note.to_owned()),
3808 recovery: Some(Recovery::Hold),
3809 ..Decision::default()
3810 }],
3811 },
3812 )
3813 .unwrap();
3814 }
3815
3816 #[test]
3817 fn reholding_with_an_identical_note_changes_nothing() {
3818 let dir = tempdir().unwrap();
3819 let queue = Queue::at(dir.path().join("queue"));
3820 let questions = Questions::at(dir.path().join("questions"));
3821 let mut t = task("held");
3822 t.hold_machine(Some("waiting on a human".to_owned()));
3823 queue.put(&mut t).unwrap();
3824
3825 rehold(&queue, &questions, &t.id, "owner said keep it held");
3826 let first = queue.get(&t.id).unwrap();
3827 let rev = queue.revision();
3828 for _ in 0..5 {
3829 rehold(&queue, &questions, &t.id, "owner said keep it held");
3830 }
3831 let after = queue.get(&t.id).unwrap();
3832 assert_eq!(after.hold_reason, first.hold_reason);
3833 assert_eq!(after.updated_at, first.updated_at);
3834 assert_eq!(queue.revision(), rev, "nothing was written");
3835 }
3836
3837 #[test]
3838 fn reholding_with_different_notes_keeps_one_previously_level() {
3839 let dir = tempdir().unwrap();
3840 let queue = Queue::at(dir.path().join("queue"));
3841 let questions = Questions::at(dir.path().join("questions"));
3842 let mut t = task("held");
3843 t.hold_machine(Some("original".to_owned()));
3844 queue.put(&mut t).unwrap();
3845
3846 for i in 0..20 {
3847 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3848 }
3849 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3850 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3851 assert!(reason.starts_with("note "), "the new note leads: {reason}");
3852 }
3853
3854 #[test]
3855 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3856 let mut t = task("deep");
3857 let nested = "same\n\n(previously: same\n\n(previously: same))";
3858 t.hold_machine(Some(nested.to_owned()));
3859 let d = Decision {
3860 id: t.id.clone(),
3861 reason: Some("same".to_owned()),
3862 ..Decision::default()
3863 };
3864 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3865 let d2 = Decision {
3866 reason: Some("other".to_owned()),
3867 ..d
3868 };
3869 assert_eq!(
3870 reaffirmed_hold_reason(&t, &d2).as_deref(),
3871 Some("other\n\n(previously: same)")
3872 );
3873 }
3874
3875 #[tokio::test]
3876 async fn a_repeated_hold_decision_settles_instead_of_looping() {
3877 let dir = tempdir().unwrap();
3878 let queue = Queue::at(dir.path().join("queue"));
3879 let questions = Questions::at(dir.path().join("questions"));
3880 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3881 let mut t = task("held");
3882 t.hold_machine(Some("first".to_owned()));
3883 queue.put(&mut t).unwrap();
3884 let cfg = config(mock_agent(
3885 dir.path(),
3886 REPLY,
3887 env(&reply.replace("ID", &t.id)),
3888 ));
3889 let mut conductor = Conductor::new();
3890 for _ in 0..2 {
3892 let held = queue.get(&t.id).unwrap();
3893 conductor
3894 .maybe_run(
3895 &cfg,
3896 dir.path(),
3897 &queue,
3898 &questions,
3899 dir.path(),
3900 &[],
3901 &[],
3902 &[held],
3903 2,
3904 )
3905 .await;
3906 }
3907 let held = queue.get(&t.id).unwrap();
3908 assert!(
3909 held.hold_reason
3910 .as_deref()
3911 .unwrap()
3912 .starts_with("keep held")
3913 );
3914 assert!(
3915 !conductor.worth_a_look(&queue, &[], &[held]),
3916 "an unchanged held task must stop being reconsidered"
3917 );
3918 }
3919
3920 #[tokio::test]
3921 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3922 let dir = tempdir().unwrap();
3923 let queue = Queue::at(dir.path().join("queue"));
3924 let questions = Questions::at(dir.path().join("questions"));
3925 let mut t = task("held");
3926 t.hold_machine(Some("first".to_owned()));
3927 queue.put(&mut t).unwrap();
3928 let reply = |note: &str| {
3929 format!(
3930 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3931 t.id
3932 )
3933 };
3934 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3935 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3936 let mut conductor = Conductor::new();
3937 let mut writes = 0;
3938 for i in 0..6 {
3939 let held = queue.get(&t.id).unwrap();
3940 let before = queue.revision();
3941 conductor
3942 .maybe_run(
3943 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3944 dir.path(),
3945 &queue,
3946 &questions,
3947 dir.path(),
3948 &[],
3949 &[],
3950 &[held],
3951 2,
3952 )
3953 .await;
3954 if queue.revision() != before {
3955 writes += 1;
3956 }
3957 }
3958 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3959 let mut held = queue.get(&t.id).unwrap();
3960 held.instruction.push_str(" (edited)");
3961 queue.put(&mut held).unwrap();
3962 let held = queue.get(&t.id).unwrap();
3963 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3964 }
3965
3966 #[test]
3967 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3968 let dir = tempdir().unwrap();
3969 let queue = Queue::at(dir.path().join("queue"));
3970 let mut t = task("t");
3971 queue.put(&mut t).unwrap();
3972
3973 let mut conductor = Conductor::new();
3974 assert!(
3975 conductor.worth_a_look(&queue, &[], &[]),
3976 "a conductor that has never run has something to look at"
3977 );
3978
3979 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3980 assert!(
3981 !conductor.worth_a_look(&queue, &[], &[]),
3982 "nothing changed and nothing is stalled or finished"
3983 );
3984 assert!(
3985 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3986 "a stalled task is worth a look even at the same revision"
3987 );
3988 assert!(
3989 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3990 "a finished task is worth a look even at the same revision"
3991 );
3992 }
3993
3994 #[tokio::test]
3995 async fn the_conduct_path_never_calls_ask_and_wait() {
3996 let dir = tempdir().unwrap();
4002 let queue = Queue::at(dir.path().join("queue"));
4003 let questions = Questions::at(dir.path().join("questions"));
4004 let mut t = task("asks without blocking");
4005 queue.put(&mut t).unwrap();
4006
4007 apply(
4008 &queue,
4009 &questions,
4010 &Verdict {
4011 decisions: vec![Decision {
4012 id: t.id.clone(),
4013 question: Some("ok?".to_owned()),
4014 ..Decision::default()
4015 }],
4016 },
4017 )
4018 .unwrap();
4019 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
4021 }
4022
4023 #[test]
4024 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
4025 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
4026 t.resume_override = Some(crate::queue::OperatorResume {
4027 question_id: "q".into(),
4028 at: jiff::Timestamp::now(),
4029 conductor_rehold: None,
4030 forced: false,
4031 pinned_run: Some("run-1".into()),
4032 });
4033 assert!(pinned_resume(&t));
4034 assert!(!may_hold(&mut t, "waiting for magi resume"));
4035 assert!(
4036 t.resume_override
4037 .as_ref()
4038 .unwrap()
4039 .conductor_rehold
4040 .is_none(),
4041 "a refused hold is not recorded as an override"
4042 );
4043 t.resume_override = None;
4044 assert!(!pinned_resume(&t));
4045 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
4046 }
4047
4048 #[test]
4049 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
4050 let dir = tempdir().unwrap();
4051 let queue = Queue::at(dir.path().join("queue"));
4052 let questions = Questions::at(dir.path().join("questions"));
4053 let mut t = task("resume me");
4054 t.resume_override = Some(crate::queue::OperatorResume {
4055 question_id: "q".into(),
4056 at: jiff::Timestamp::now(),
4057 conductor_rehold: None,
4058 forced: true,
4059 pinned_run: Some("run-1".into()),
4060 });
4061 queue.put(&mut t).unwrap();
4062
4063 apply(
4064 &queue,
4065 &questions,
4066 &Verdict {
4067 decisions: vec![
4068 Decision {
4069 id: t.id.clone(),
4070 question: Some("really?".to_owned()),
4071 ..Decision::default()
4072 },
4073 Decision {
4074 id: t.id.clone(),
4075 blocked_by: vec!["other".to_owned()],
4076 ..Decision::default()
4077 },
4078 ],
4079 },
4080 )
4081 .unwrap();
4082
4083 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
4084 assert!(questions.list().is_empty());
4085 }
4086}