1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104 Recover,
106 Resume,
108 Repair,
110 Complete,
112 Inspect,
114 Configure,
116}
117
118#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123 Task,
125 Run {
127 id: String,
129 },
130 Setting {
133 path: String,
135 keys: Vec<String>,
137 },
138 Source {
140 reference: String,
142 },
143}
144
145#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148 pub operation: QuestionOperation,
150 pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155 fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156 let target = match &self.target {
157 QuestionTarget::Task => format!("task:{}", task.id),
158 QuestionTarget::Run { id } => {
159 anyhow::ensure!(
160 task.runs.contains(id),
161 "question target is not this task's run"
162 );
163 format!("run:{id}")
164 }
165 QuestionTarget::Setting { path, keys } => {
166 use std::path::Component;
167 anyhow::ensure!(
168 !path.is_empty()
169 && Path::new(path)
170 .components()
171 .all(|c| matches!(c, Component::Normal(_)))
172 && !keys.is_empty()
173 && keys.iter().all(|key| !key.is_empty()),
174 "question setting must name a repository file and concrete keys"
175 );
176 let root = task.repo.canonicalize()?;
177 let file = root.join(path).canonicalize()?;
178 let relative = file
179 .strip_prefix(&root)
180 .context("question setting is outside the repository")?;
181 let content = std::fs::read_to_string(&file)?;
182 let value: toml::Value = toml::from_str(&content)?;
183 let mut setting = &value;
184 for key in keys {
185 setting = setting
186 .get(key)
187 .context("question setting key does not exist")?;
188 }
189 format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190 }
191 QuestionTarget::Source { reference } => {
192 use crate::proc::Quiet as _;
193 let output = std::process::Command::new("git")
194 .args([
195 "rev-parse",
196 "--verify",
197 "--end-of-options",
198 &format!("{reference}^{{commit}}"),
199 ])
200 .current_dir(&task.repo)
201 .quiet()
202 .output()
203 .context("resolve question source")?;
204 anyhow::ensure!(
205 output.status.success(),
206 "question source cannot be resolved"
207 );
208 let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209 anyhow::ensure!(
210 !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211 "invalid question source commit"
212 );
213 format!("source:{commit}")
214 }
215 };
216 Ok((self.operation, target))
217 }
218}
219
220#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224 pub id: String,
226 #[serde(default)]
230 pub blocked_by: Vec<String>,
231 #[serde(default)]
236 pub missing_dependencies: Vec<String>,
237 #[serde(default)]
239 pub reason: Option<String>,
240 #[serde(default)]
247 pub recovery: Option<Recovery>,
248 #[serde(default)]
254 pub question: Option<String>,
255 #[serde(default)]
257 pub choices: Vec<String>,
258 #[serde(default)]
261 pub question_decision: Option<QuestionDecision>,
262 #[serde(default)]
264 pub evidence: Vec<String>,
265}
266
267#[derive(Debug, Clone, Default, Deserialize)]
280pub struct Verdict {
281 pub decisions: Vec<Decision>,
284}
285
286fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
289 prompt::ConductTask {
290 id: t.id.clone(),
291 title: t.title.clone(),
292 instruction: t.instruction.clone(),
293 repo: t.repo.display().to_string(),
294 priority: t.priority,
295 status: t.status.as_str().to_owned(),
296 attempts: t.attempts,
297 max_attempts: t.effective_attempt_limit(max_attempts),
298 last_error: t.last_error.clone(),
299 hold_reason: t.hold_reason.clone(),
300 hold_source: t.hold_source.map(|source| source.label().to_owned()),
301 blocked_by: t.blocked_by.clone(),
302 answers: t
303 .answers
304 .iter()
305 .map(|a| prompt::ConductAnswer {
306 question: a.question.clone(),
307 answer: a.answer.clone(),
308 })
309 .collect(),
310 operator_resume: t.resumption.as_ref().map(|r| format!(
311 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
312 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
313 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
314 wording alone is insufficient.",
315 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
316 )).or_else(|| t.resume_override.as_ref().map(|o| {
317 format!(
318 "the operator explicitly answered \"resume\" at {}; do not hold this \
319 task again for the same reason unless there is new information",
320 o.at
321 )
322 })),
323 }
324}
325
326fn pinned_resume(task: &Task) -> bool {
330 task.resume_override
331 .as_ref()
332 .is_some_and(|o| o.pinned_run.is_some())
333}
334
335fn new_blocking_evidence(
338 queue: &Queue,
339 questions: &Questions,
340 task: &Task,
341 d: &Decision,
342) -> Vec<String> {
343 let Some(r) = &task.resumption else {
344 return Vec::new();
345 };
346 d.evidence
347 .iter()
348 .filter_map(|id| {
349 let version = if let Ok(q) = questions.get(id) {
350 (q.asked_at > r.at
351 && (q.run == task.id || task.runs.contains(&q.run))
352 && q.node != NODE
353 && q.status != crate::ask::QuestionStatus::Abandoned)
354 .then(|| {
355 format!(
356 "question:{id}:{}:{:?}:{}",
357 q.asked_at,
358 q.status,
359 q.resolution().unwrap_or_default()
360 )
361 })
362 } else if d.blocked_by.contains(id) {
363 queue
364 .get(id)
365 .ok()
366 .filter(|dep| {
367 dep.id != task.id
368 && dep.status != TaskStatus::Done
369 && (dep.created_at > r.at
370 || dep.held_at.is_some_and(|at| at > r.at)
371 || dep.operator_decided_at.is_some_and(|at| at > r.at)
372 || dep.passes.iter().any(|p| p.settled_at > r.at)
373 || (dep.status == TaskStatus::Blocked
374 && dep.blocked_at.is_some_and(|at| at > r.at)))
375 })
376 .map(|dep| {
377 format!(
378 "dependency:{id}:{:?}:{:?}:{:?}",
379 dep.status,
380 dep.chat_notice_key(),
381 dep.hold_notice_cause
382 )
383 })
384 } else if task.runs.iter().any(|run| run == id) {
385 RunState::load(id)
386 .ok()
387 .filter(|run| {
388 run.updated_at > r.at
391 && run
392 .passes
393 .last()
394 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
395 && matches!(
396 run.status,
397 crate::run::RunStatus::Failed
398 | crate::run::RunStatus::Blocked
399 | crate::run::RunStatus::Stalled
400 )
401 })
402 .map(|run| {
403 format!(
404 "run:{id}:{:?}:{:?}",
405 run.status,
406 run.passes.last().map(|p| p.index)
407 )
408 })
409 } else {
410 None
411 };
412 version.filter(|v| !r.blocking_evidence.contains(v))
413 })
414 .collect()
415}
416
417fn may_hold(task: &mut Task, reason: &str) -> bool {
421 if pinned_resume(task) {
422 return false;
423 }
424 if task.resumption.is_some() {
425 return true; }
427 let Some(o) = task.resume_override.as_mut() else {
428 return true;
429 };
430 if o.forced || o.pinned_run.is_some() {
431 tracing::warn!(
432 "conductor tried to hold task {} after the operator forced a resume: {reason}",
433 task.id
434 );
435 return false;
436 }
437 if o.conductor_rehold.is_some() {
438 return false;
439 }
440 o.conductor_rehold = Some(reason.to_owned());
441 true
442}
443
444fn severity_str(s: crate::verdict::Severity) -> &'static str {
447 match s {
448 crate::verdict::Severity::Nit => "nit",
449 crate::verdict::Severity::Minor => "minor",
450 crate::verdict::Severity::Major => "major",
451 crate::verdict::Severity::Blocker => "blocker",
452 }
453}
454
455fn surviving_branch(task: &Task) -> Option<String> {
460 let last = task.runs.last()?;
461 let state = RunState::load(last).ok()?;
462 if !crate::daemon::compatible_run_instruction(&state, task) {
463 return None;
464 }
465 state.winner().map(|c| c.branch.clone())
466}
467
468fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
495 let note = match &d.reason {
496 Some(reason) => reason.clone(),
497 None => match task.answers.last() {
498 Some(a) => format!(
499 "conduct held this again with no new reason given; last operator \
500 answer on record: {}",
501 a.answer
502 ),
503 None => "conduct held this again with no reason given".to_owned(),
504 },
505 };
506 match task.hold_reason.as_deref() {
507 Some(prior) if !prior.is_empty() => {
508 let outer = outermost_hold_note(prior);
509 if note.trim() == prior.trim() || note.trim() == outer {
510 None
511 } else {
512 Some(format!("{note}\n\n(previously: {outer})"))
513 }
514 }
515 _ => Some(note),
516 }
517}
518
519const PREVIOUSLY: &str = "\n\n(previously: ";
522
523fn outermost_hold_note(reason: &str) -> &str {
526 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
527}
528
529fn hold_note(d: &Decision) -> String {
531 d.reason
532 .clone()
533 .unwrap_or_else(|| "(no reason given)".to_owned())
534}
535
536async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
538 let Some(run_id) = task.runs.last().cloned() else {
539 return prompt::ConductOutcome {
540 run_id: "(none)".to_owned(),
541 unreadable: Some("this task has not produced a run yet".to_owned()),
542 run_status: None,
543 open_findings: Vec::new(),
544 rounds_used: 0,
545 rounds_max: 0,
546 rounds: Vec::new(),
547 branch: None,
548 branch_head: None,
549 references: None,
550 empty_candidate: false,
551 };
552 };
553 let state = match RunState::load(&run_id) {
554 Ok(s) => s,
555 Err(e) => {
556 tracing::warn!(
561 "conductor: could not read run {run_id} for task {}: {e:#}",
562 task.short()
563 );
564 return prompt::ConductOutcome {
565 run_id,
566 unreadable: Some(format!("{e:#}")),
567 run_status: None,
568 open_findings: Vec::new(),
569 rounds_used: 0,
570 rounds_max: 0,
571 rounds: Vec::new(),
572 branch: None,
573 branch_head: None,
574 references: None,
575 empty_candidate: false,
576 };
577 }
578 };
579
580 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
581 id: f.id.clone(),
582 title: f.title.clone(),
583 severity: severity_str(f.severity).to_owned(),
584 };
585 let open_findings = state
586 .open_findings()
587 .into_iter()
588 .map(finding_view)
589 .collect();
590 let rounds = state
591 .reviews
592 .iter()
593 .map(|r| prompt::ConductRound {
594 round: r.round,
595 findings: r
596 .reviews
597 .iter()
598 .flat_map(|rec| rec.findings.iter())
599 .map(finding_view)
600 .collect(),
601 addressed: r
602 .fix
603 .as_ref()
604 .map(|fx| fx.addressed.clone())
605 .unwrap_or_default(),
606 rejected: r
607 .fix
608 .as_ref()
609 .map(|fx| {
610 fx.rejected
611 .iter()
612 .map(|rej| prompt::ConductRejection {
613 id: rej.id.clone(),
614 why: rej.why.clone(),
615 })
616 .collect()
617 })
618 .unwrap_or_default(),
619 })
620 .collect();
621 let branch = state.winner().map(|c| c.branch.clone());
622 let branch_head = match &branch {
623 Some(b) => crate::git::rev_parse(repo, b)
624 .await
625 .ok()
626 .map(|h| h.chars().take(8).collect()),
627 None => None,
628 };
629
630 prompt::ConductOutcome {
631 run_id,
632 unreadable: None,
633 run_status: Some(state.status.as_str().to_owned()),
634 open_findings,
635 rounds_used: state.reviews.len(),
636 rounds_max: state.config.graph.review_rounds,
637 rounds,
638 branch,
639 branch_head,
640 references: crate::refs::describe(&state.seeds),
641 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
642 }
643}
644
645async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
649 let id = task.runs.last()?;
650 let state = match RunState::load(id) {
651 Ok(s) => s,
652 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
653 };
654 let branch = state.winner()?.branch.clone();
655 let base = state.base_branch.clone();
656 match crate::land::find_open_pr(repo, &branch, &base).await {
657 Ok(crate::land::OpenPr::None) => None,
658 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
659 "pull request {url} is already open for {branch} into {base}"
660 )),
661 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
662 "several pull requests are already open for {branch} into {base}: {}",
663 urls.join(" ")
664 )),
665 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
666 }
667}
668
669async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
676 for d in verdict
677 .decisions
678 .iter_mut()
679 .filter(|d| d.question.is_some())
680 {
681 let Ok(task) = queue.get(&d.id) else {
682 continue;
683 };
684 let text = format!("{}\n{}", task.title, task.instruction);
685 let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
686 if crate::refs::scan(&text).is_empty() {
687 if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
688 q.push_str(&format!(
689 "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
690 ));
691 }
692 continue;
693 }
694 let repo = repo_for(&task, repo);
695 let remote = &cfg.merge.remote;
696 let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
697 .await
698 .ok();
699 let facts = match base {
700 Some(base) => {
701 let base_name = base.clone();
702 let tracking = format!("{remote}/{base}");
703 let refreshed = crate::git::fetch(&repo, remote, &base)
706 .await
707 .is_ok_and(|o| o.ok());
708 let against = tracking;
711 match crate::git::rev_parse(&repo, &against).await {
712 Ok(tip) => crate::refs::describe(
713 &crate::refs::resolve(&repo, &tip, remote, &text).await,
714 )
715 .map(|facts| {
716 if refreshed {
717 facts
718 } else {
719 format!(
720 "{facts}\n(could not fetch {remote}/{base_name}: this is \
721 against the last fetched `{against}`, which may be behind \
722 the remote)"
723 )
724 }
725 }),
726 Err(_) => Some(format!(
727 "could NOT be checked against the repository: {against} is not \
728 available here, so whether the work is in the base is unknown"
729 )),
730 }
731 }
732 None => Some("could not check the repository: no base branch known".to_owned()),
733 };
734 let facts = match (facts, pr_fact) {
735 (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
736 (f, p) => f.or(p),
737 };
738 if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
739 q.push_str(&format!(
740 "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
741 ));
742 }
743 }
744}
745
746async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
748 prompt::ConductFinished {
749 task: view(t, max_attempts),
750 outcome: outcome_for(t, &repo_for(t, repo)).await,
751 }
752}
753
754fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
757 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
758 fallback.to_path_buf()
759 } else {
760 task.repo.clone()
761 }
762}
763
764fn apply_one(
781 queue: &Queue,
782 questions: &Questions,
783 d: &Decision,
784 snapshot: Option<&Task>,
785) -> Result<()> {
786 let _claim = queue
787 .claim(&d.id)
788 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
789 let mut task = queue.get(&d.id).context("no such task")?;
790
791 if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
794 return Ok(());
795 }
796 let blocking =
797 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
798 let evidence = new_blocking_evidence(queue, questions, &task, d);
799 if blocking && task.resumption.is_some() && evidence.is_empty() {
800 crate::notices::raise(crate::notices::conductor_resumption(
801 &task.id,
802 task.short(),
803 d.reason.as_deref(),
804 &d.blocked_by,
805 &d.evidence,
806 d.question.is_some(),
807 ));
808 return Ok(());
809 }
810
811 if blocking && !evidence.is_empty() {
812 let mut cause = evidence.clone();
813 cause.sort();
814 cause.dedup();
815 task.notice_cause(
816 &format!("conductor-evidence:{}", cause.join(",")),
817 None,
818 None,
819 );
820 }
821 if blocking {
822 if let Some(r) = task.resumption.as_mut() {
823 r.blocking_evidence.extend(evidence);
824 }
825 }
826
827 if task.operator_held() {
831 return Ok(());
832 }
833
834 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
838 return Ok(());
839 }
840
841 let pinned = pinned_resume(&task);
844 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
845 return Ok(());
846 }
847
848 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
852 && !d.missing_dependencies.is_empty()
853 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
854 && d.missing_dependencies
855 .iter()
856 .any(|id| !d.blocked_by.contains(id))
857 && d.missing_dependencies
858 .iter()
859 .chain(&d.blocked_by)
860 .any(|id| {
861 matches!(
862 crate::queue::dependency(queue, questions, id),
863 crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
864 )
865 })
866 {
867 return Ok(());
868 }
869
870 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
876 && !d.missing_dependencies.is_empty()
877 && d.missing_dependencies.iter().all(|id| {
878 let claim = crate::queue::dependency(queue, questions, id);
879 claim != crate::queue::Dependency::Missing
880 && d.blocked_by.iter().any(|blocker| {
881 blocker == id
882 || (matches!(
883 &claim,
884 crate::queue::Dependency::Task(_)
885 | crate::queue::Dependency::Question(_)
886 | crate::queue::Dependency::Deleted(_)
887 ) && crate::queue::dependency(queue, questions, blocker) == claim)
888 })
889 })
890 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
891 && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
892 {
893 let mut ids = d.blocked_by.clone();
894 crate::queue::normalize_blockers(queue, questions, &mut ids);
895 let reason = format!("Waiting on dependencies: {}", ids.join(", "));
896 task.block(ids, Some(reason));
897 return queue.put(&mut task);
898 }
899
900 if let Some(text) = &d.question {
901 if task.status == TaskStatus::Done {
902 return Ok(());
903 }
904 let mut held = task.clone();
905 held.status = TaskStatus::Held;
906 let targets: BTreeSet<_> = d
907 .blocked_by
908 .iter()
909 .filter_map(|id| {
910 if let Ok(dep) = queue.get(id) {
911 Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
912 } else if questions.get(id).is_ok() {
913 Some(format!("question:{id}"))
914 } else {
915 None
916 }
917 })
918 .collect();
919 let decision = d
920 .question_decision
921 .as_ref()
922 .and_then(|q| match q.identity(&task) {
923 Ok(identity) => Some(identity),
924 Err(error) => {
925 tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
928 None
929 }
930 });
931 let context_value =
932 serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
933 let context = serde_json::to_string(&context_value)?;
934 if questions.list().iter().any(|q| {
938 q.node == NODE
939 && q.run == task.id
940 && q.status == crate::ask::QuestionStatus::Answered
941 && q.decision_context.as_ref().is_some_and(|stored| {
942 if stored == &context {
943 return true;
944 }
945 decision.is_none()
948 && serde_json::from_str::<serde_json::Value>(stored)
949 .ok()
950 .and_then(|v| {
951 v.as_array().map(|items| {
952 items.len() == 4
953 && items[..3]
954 == context_value.as_array().expect("context")[..3]
955 })
956 })
957 == Some(true)
958 })
959 }) {
960 return Ok(());
961 }
962 let question_id = match questions
966 .list()
967 .into_iter()
968 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
969 {
970 Some(existing) => existing.id,
971 None => {
972 let mut q = Question::new(
973 task.id.clone(),
974 NODE.to_owned(),
975 SEAT.to_owned(),
976 text.clone(),
977 d.reason.clone().unwrap_or_default(),
978 d.choices.clone(),
979 );
980 q.decision_context = Some(context);
981 q.deputy = Some(Deputy::new(crate::deputy::brief(
986 &task.id,
987 d.reason.as_deref().unwrap_or_default(),
988 &q.choices,
989 &q.actions,
990 )));
991 if task.repo.is_dir() && task.repo != Path::new(".") {
992 q.cwd = Some(task.repo.to_string_lossy().into_owned());
993 }
994 questions.put(&mut q)?;
995 q.id
996 }
997 };
998 task.block(vec![question_id], d.reason.clone());
999 return queue.put(&mut task);
1000 }
1001
1002 match task.status {
1003 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
1007 if may_hold(&mut task, &hold_note(d)) {
1008 task.hold_machine(d.reason.clone());
1009 queue.put(&mut task)?;
1010 }
1011 }
1012 TaskStatus::Queued if !d.blocked_by.is_empty() => {
1013 let mut ids = d.blocked_by.clone();
1014 crate::queue::normalize_blockers(queue, questions, &mut ids);
1015 task.block(ids, d.reason.clone());
1016 queue.put(&mut task)?;
1017 }
1018 TaskStatus::Running => match d.recovery {
1019 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1020 Some(Recovery::Requeue) => {
1021 task.requeue();
1022 queue.put(&mut task)?;
1023 }
1024 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
1025 task.hold_machine(d.reason.clone());
1026 queue.put(&mut task)?;
1027 }
1028 _ => {}
1032 },
1033 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
1034 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1037 Some(Recovery::Requeue) => {
1038 task.requeue();
1039 queue.put(&mut task)?;
1040 }
1041 Some(Recovery::Hold) => {
1042 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1046 if may_hold(&mut task, &hold_note(d)) {
1047 task.hold_machine(Some(reason));
1048 queue.put(&mut task)?;
1049 }
1050 }
1051 }
1052 Some(Recovery::Review) => {
1053 if let Some(branch) = surviving_branch(&task) {
1054 task.request_review(branch);
1055 queue.put(&mut task)?;
1056 }
1057 }
1063 Some(Recovery::Done) => {
1064 task.succeed();
1065 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1069 queue.put(&mut task)?;
1070 }
1071 None => {}
1072 },
1073 _ => {}
1076 }
1077 Ok(())
1078}
1079
1080pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1084 for d in &verdict.decisions {
1085 if let Err(e) = apply_one(queue, questions, d, None) {
1086 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1087 }
1088 }
1089 Ok(())
1090}
1091
1092pub fn seat_path(home: &Path) -> PathBuf {
1098 home.join("conduct").join("seat.json")
1099}
1100
1101pub fn load_seat(home: &Path) -> Option<SeatState> {
1103 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1104}
1105
1106fn busy_path(home: &Path) -> PathBuf {
1107 home.join("conduct").join("busy")
1108}
1109
1110pub fn busy(home: &Path) -> bool {
1116 std::fs::metadata(busy_path(home))
1117 .and_then(|m| m.modified())
1118 .ok()
1119 .and_then(|t| t.elapsed().ok())
1120 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1121}
1122
1123struct Busy(PathBuf);
1125
1126impl Busy {
1127 fn mark(home: &Path) -> Self {
1128 let path = busy_path(home);
1129 if let Some(dir) = path.parent() {
1130 let _ = std::fs::create_dir_all(dir);
1131 }
1132 let _ = std::fs::write(&path, std::process::id().to_string());
1133 Self(path)
1134 }
1135}
1136
1137impl Drop for Busy {
1138 fn drop(&mut self) {
1139 let _ = std::fs::remove_file(&self.0);
1140 }
1141}
1142
1143#[derive(Debug, Default)]
1146pub struct Conductor {
1147 seat: Option<SeatState>,
1148 last_seen: Option<(u64, BTreeSet<String>)>,
1149 considered: BTreeMap<String, String>,
1153}
1154
1155fn decision_input(task: &Task) -> serde_json::Value {
1160 let mut v = serde_json::to_value(task).unwrap_or_default();
1161 if let Some(o) = v.as_object_mut() {
1162 o.remove("chat_notice");
1163 o.remove("chat_notices");
1164 o.remove("updated_at");
1166 }
1167 v
1168}
1169
1170fn input_fingerprint(task: &Task) -> String {
1171 let mut v = decision_input(task);
1172 if let Some(o) = v.as_object_mut() {
1173 o.remove("hold_reason");
1174 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1175 ro.remove("conductor_rehold");
1176 }
1177 }
1178 v.to_string()
1179}
1180
1181impl Conductor {
1182 #[must_use]
1184 pub fn new() -> Self {
1185 Self::default()
1186 }
1187
1188 fn settled(
1193 considered: &BTreeMap<String, String>,
1194 stalled: &[Task],
1195 finished: &[Task],
1196 ) -> BTreeSet<String> {
1197 stalled
1198 .iter()
1199 .chain(finished)
1200 .filter(|t| t.status == TaskStatus::Held)
1201 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1202 .map(|t| t.id.clone())
1203 .collect()
1204 }
1205
1206 fn snapshot_with(
1207 considered: &BTreeMap<String, String>,
1208 queue: &Queue,
1209 stalled: &[Task],
1210 finished: &[Task],
1211 ) -> (u64, BTreeSet<String>) {
1212 let ids = stalled
1213 .iter()
1214 .chain(finished)
1215 .map(|t| t.id.clone())
1216 .collect();
1217 let skip = Self::settled(considered, stalled, finished);
1218 (queue.revision_excluding(&skip), ids)
1219 }
1220
1221 fn snapshot(
1222 &self,
1223 queue: &Queue,
1224 stalled: &[Task],
1225 finished: &[Task],
1226 ) -> (u64, BTreeSet<String>) {
1227 Self::snapshot_with(&self.considered, queue, stalled, finished)
1228 }
1229
1230 #[must_use]
1247 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1248 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1249 }
1250
1251 #[allow(clippy::too_many_arguments)]
1255 pub async fn maybe_run(
1256 &mut self,
1257 cfg: &Config,
1258 repo: &Path,
1259 queue: &Queue,
1260 questions: &Questions,
1261 home: &Path,
1262 queued: &[Task],
1263 stalled: &[Task],
1264 finished: &[Task],
1265 max_attempts: usize,
1266 ) {
1267 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1268 return;
1269 }
1270 self.considered = stalled
1273 .iter()
1274 .chain(finished)
1275 .filter(|t| t.status == TaskStatus::Held)
1276 .map(|t| (t.id.clone(), input_fingerprint(t)))
1277 .collect();
1278 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1279 if let Err(e) = self
1280 .run_once(
1281 cfg,
1282 repo,
1283 queue,
1284 questions,
1285 home,
1286 queued,
1287 stalled,
1288 finished,
1289 max_attempts,
1290 )
1291 .await
1292 {
1293 tracing::warn!("conductor: {e:#}");
1299 }
1300 }
1301
1302 #[allow(clippy::too_many_arguments)]
1303 async fn run_once(
1304 &mut self,
1305 cfg: &Config,
1306 repo: &Path,
1307 queue: &Queue,
1308 questions: &Questions,
1309 home: &Path,
1310 queued: &[Task],
1311 stalled: &[Task],
1312 finished: &[Task],
1313 max_attempts: usize,
1314 ) -> Result<()> {
1315 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1316 return Ok(());
1317 }
1318
1319 let primary = cfg
1320 .resolve_roles()
1321 .context("resolving the conductor seat")?
1322 .conductor;
1323 let chain = match cfg.roles.conductor.as_ref() {
1327 Some(c) if c.ids().len() > 1 => {
1328 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1329 }
1330 _ => vec![primary],
1331 };
1332
1333 let runnable_views: Vec<prompt::ConductTask> =
1334 queued.iter().map(|t| view(t, max_attempts)).collect();
1335 let stalled_views: Vec<prompt::ConductTask> =
1336 stalled.iter().map(|t| view(t, max_attempts)).collect();
1337 let mut finished_views = Vec::with_capacity(finished.len());
1338 for t in finished {
1339 finished_views.push(finished_view(t, repo, max_attempts).await);
1340 }
1341
1342 let body = prompt::with_overlay(
1343 prompt::conduct(
1344 &runnable_views,
1345 &stalled_views,
1346 &finished_views,
1347 &cfg.graph.language,
1348 ),
1349 cfg.prompts.overlay(NODE),
1350 );
1351
1352 let body = format!(
1353 "{body}\n\n{}",
1354 crate::queue::dependency_facts(queue, questions)
1355 );
1356 let artifacts = home.join("conduct").join("artifacts");
1357 let cache_dir = cfg.cache_dir();
1360
1361 let busy = Busy::mark(home);
1363 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1364 for spec in &chain {
1365 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1368 if needs_new_seat {
1369 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1370 }
1371 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1372 let stem = if std::ptr::eq(spec, &chain[0]) {
1375 format!("turn-{}", seat.turns + 1)
1376 } else {
1377 format!("turn-{}-{}", seat.turns + 1, spec.id)
1378 };
1379 let inv = Invocation {
1380 cwd: repo,
1381 prompt: &body,
1382 timeout: TURN_TIMEOUT,
1383 allow_write: false,
1386 unsandboxed: false,
1387 sessions: cfg.graph.sessions,
1388 artifacts: &artifacts,
1389 stem: &stem,
1390 run: NODE,
1391 node: NODE,
1392 cache_dir: cache_dir.as_deref(),
1393 attachments: &[],
1394 writable: &[],
1395 };
1396 let out = agent::invoke(spec, seat, &inv).await;
1397 if let Ok(body) = serde_json::to_string(&*seat) {
1400 let path = seat_path(home);
1401 if let Some(dir) = path.parent() {
1402 let _ = std::fs::create_dir_all(dir);
1403 }
1404 let _ = std::fs::write(path, body);
1405 }
1406 let advance = agent::chain_advances(&out);
1407 result = match out {
1408 Err(e) => Err(e.context("invoking the conductor")),
1409 Ok(out) if advance => Err(anyhow::anyhow!(
1410 "no usable reply (exit {:?}, timed out {})",
1411 out.exit_code,
1412 out.timed_out
1413 )),
1414 Ok(out) => verdict::extract_json(&out.text)
1415 .context("the conductor's reply could not be parsed"),
1416 };
1417 if result.is_ok() {
1418 break;
1419 }
1420 if chain.len() > 1 {
1421 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1422 }
1423 }
1424 drop(busy);
1425 let mut verdict = result?;
1426 attach_facts(cfg, repo, queue, &mut verdict).await;
1427 for d in &verdict.decisions {
1428 let snapshot = queued
1429 .iter()
1430 .chain(stalled)
1431 .chain(finished)
1432 .find(|t| t.id == d.id);
1433 if let Some(snapshot) = snapshot {
1434 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1435 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1436 }
1437 }
1438 }
1439 Ok(())
1440 }
1441}
1442
1443#[cfg(test)]
1444mod tests {
1445 use std::collections::BTreeMap;
1446
1447 use tempfile::tempdir;
1448
1449 use super::*;
1450 use crate::ask::{Answer, QuestionStatus};
1451 use crate::config::{AgentKind, AgentSpec, Graph};
1452 use crate::queue::Source;
1453
1454 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1455 let path = dir.join("mock-conduct-agent.sh");
1456 std::fs::write(&path, script).expect("write mock");
1457 AgentSpec {
1458 id: "mock".to_owned(),
1459 kind: AgentKind::Command,
1460 model: None,
1461 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1462 extra_args: Vec::new(),
1463 env,
1464 prompt_delivery: None,
1465 }
1466 }
1467
1468 fn config(spec: AgentSpec) -> Config {
1469 Config {
1470 agents: vec![spec],
1471 graph: Graph {
1472 language: "en".to_owned(),
1473 ..Graph::default()
1474 },
1475 ..Config::default()
1476 }
1477 }
1478
1479 fn task(title: &str) -> Task {
1480 Task::new(
1481 title.to_owned(),
1482 format!("do {title}"),
1483 std::path::PathBuf::from("."),
1484 Source::Human,
1485 )
1486 }
1487
1488 #[test]
1489 fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1490 let mut t = task("waiting");
1491 t.hold_machine(Some("waiting for recovery".into()));
1492 let before = input_fingerprint(&t);
1493 let snapshot = decision_input(&t);
1494 t.chat_notice = Some(crate::queue::ChatNotice {
1495 key: t.chat_notice_key().unwrap(),
1496 at: jiff::Timestamp::now(),
1497 outcome: crate::queue::ChatNoticeOutcome::Posted,
1498 });
1499 t.chat_notices.push(t.chat_notice_key().unwrap());
1500 t.updated_at = jiff::Timestamp::now();
1501 assert_eq!(before, input_fingerprint(&t));
1502 assert_eq!(snapshot, decision_input(&t));
1503 t.hold_reason = Some("recovery remains under consideration".into());
1504 assert_eq!(before, input_fingerprint(&t));
1505 t.record_answer("a different question".into(), "a new decision".into());
1506 assert_ne!(before, input_fingerprint(&t));
1507 }
1508
1509 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1510 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1511
1512 fn env(reply: &str) -> BTreeMap<String, String> {
1513 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1514 }
1515
1516 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1517
1518 fn init_repo_with_branch(dir: &Path, branch: &str) {
1522 use crate::proc::Quiet as _;
1523 let run = |args: &[&str]| {
1524 let out = std::process::Command::new("git")
1525 .args(args)
1526 .current_dir(dir)
1527 .quiet()
1528 .output()
1529 .expect("spawn git");
1530 assert!(
1531 out.status.success(),
1532 "git {args:?} failed: {}",
1533 String::from_utf8_lossy(&out.stderr)
1534 );
1535 };
1536 run(&["init", "-b", "main"]);
1537 run(&["config", "user.name", "magi test"]);
1538 run(&["config", "user.email", "magi@example.com"]);
1539 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1540 run(&["add", "-A"]);
1541 run(&["commit", "-m", "init"]);
1542 run(&["checkout", "-b", branch]);
1543 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1544 run(&["add", "-A"]);
1545 run(&["commit", "-m", "candidate work"]);
1546 }
1547
1548 fn review_round_with_finding(
1549 round: usize,
1550 finding_id: &str,
1551 title: &str,
1552 addressed: &[&str],
1553 rejected: &[(&str, &str)],
1554 ) -> crate::run::ReviewRound {
1555 crate::run::ReviewRound {
1556 round,
1557 head: "deadbeef".to_owned(),
1558 verified_head: None,
1559 verified_at: None,
1560 reviews: vec![crate::run::ReviewRecord {
1561 attempts: 0,
1562 reviewer: 1,
1563 agent: "mock".to_owned(),
1564 summary: String::new(),
1565 findings: vec![crate::verdict::Finding {
1566 id: finding_id.to_owned(),
1567 severity: crate::verdict::Severity::Major,
1568 file: None,
1569 line: None,
1570 title: title.to_owned(),
1571 detail: String::new(),
1572 }],
1573 vote: None,
1574 failed: None,
1575 duration_ms: 0,
1576 }],
1577 e2e: Vec::new(),
1578 verify_retried: false,
1579 e2e_deferred: false,
1580 e2e_defer_reason: None,
1581 fix: Some(crate::run::FixRecord {
1582 agent: "mock".to_owned(),
1583 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1584 rejected: rejected
1585 .iter()
1586 .map(|(id, why)| crate::verdict::Rejection {
1587 id: (*id).to_owned(),
1588 why: (*why).to_owned(),
1589 })
1590 .collect(),
1591 notes: String::new(),
1592 committed: false,
1593 failed: None,
1594 duration_ms: 0,
1595 continuation: None,
1596 }),
1597 blocking: 1,
1598 answered: 1,
1599 expected: 1,
1600 clean: false,
1601 progressed: true,
1602 vote_split: false,
1603 reconsideration: Vec::new(),
1604 verdict: None,
1605 }
1606 }
1607
1608 #[test]
1609 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1610 crate::run::pin_test_home();
1611 let dir = tempdir().unwrap();
1612 let queue = Queue::at(dir.path().join("queue"));
1613 let questions = Questions::at(dir.path().join("questions"));
1614 let mut t = task("recover");
1615 t.answers.push(crate::queue::AnsweredQuestion {
1616 question: "Inspect remote first?".into(),
1617 answer: "Inspect, then await resumption instructions".into(),
1618 });
1619 t.hold_machine(Some("waiting for remote inspection".into()));
1620 queue.put(&mut t).unwrap();
1621 let snapshot = queue.get(&t.id).unwrap();
1622 t.edit(
1623 "recover".into(),
1624 "Reuse the measured PR head; fix Clippy; review".into(),
1625 )
1626 .unwrap();
1627 t.release_operator("cli", None);
1628 queue.put(&mut t).unwrap();
1629 let approval = t.resumption.clone();
1630 let d = Decision {
1631 id: t.id.clone(),
1632 recovery: Some(Recovery::Hold),
1633 reason: Some("remote inspection still required".into()),
1634 ..Decision::default()
1635 };
1636 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1637 let stale_question = Decision {
1638 question: Some("Inspect remote?".into()),
1639 ..d.clone()
1640 };
1641 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1642 assert!(questions.list().is_empty());
1643 for reason in [
1644 "waiting for remote inspection",
1645 "inspection has not been confirmed",
1646 ] {
1647 let repeated = Decision {
1648 reason: Some(reason.into()),
1649 ..d.clone()
1650 };
1651 apply(
1652 &queue,
1653 &questions,
1654 &Verdict {
1655 decisions: vec![repeated],
1656 },
1657 )
1658 .unwrap();
1659 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1660 }
1661 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1662 assert_eq!(reloaded.resumption, approval);
1663 assert_eq!(reloaded.answers, t.answers);
1664 let v = view(&reloaded, 3);
1665 assert!(v.operator_resume.unwrap().contains("superseded"));
1666 let mut dep = task("new prerequisite");
1667 dep.hold_machine(Some("new authentication failure".into()));
1668 queue.put(&mut dep).unwrap();
1669 let new_blocker = Decision {
1670 blocked_by: vec![dep.id.clone()],
1671 evidence: vec![dep.id.clone()],
1672 recovery: None,
1674 reason: Some("new prerequisite failed".into()),
1675 ..d.clone()
1676 };
1677 apply(
1678 &queue,
1679 &questions,
1680 &Verdict {
1681 decisions: vec![new_blocker.clone()],
1682 },
1683 )
1684 .unwrap();
1685 let mut after = queue.get(&t.id).unwrap();
1686 assert_eq!(after.status, TaskStatus::Blocked);
1687 after.release();
1690 queue.put(&mut after).unwrap();
1691 apply(
1692 &queue,
1693 &questions,
1694 &Verdict {
1695 decisions: vec![new_blocker],
1696 },
1697 )
1698 .unwrap();
1699 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1700 after.start("new-run".into());
1701 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1702 after.hold_manual(Some("operator says wait".into()));
1703 queue.put(&mut after).unwrap();
1704 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1705 assert!(queue.get(&t.id).unwrap().operator_held());
1706 }
1707
1708 #[test]
1709 fn newly_blocked_dependency_is_material_evidence_after_approval() {
1710 let dir = tempdir().unwrap();
1711 let queue = Queue::at(dir.path().join("queue"));
1712 let questions = Questions::at(dir.path().join("questions"));
1713 let mut dep = task("old queued dependency");
1714 queue.put(&mut dep).unwrap();
1715 let mut t = task("approved task");
1716 t.release_operator("test", None);
1717 queue.put(&mut t).unwrap();
1718 let mut q = Question::new(
1719 dep.id.clone(),
1720 "check".into(),
1721 "check".into(),
1722 "New dependency approval".into(),
1723 String::new(),
1724 Vec::new(),
1725 );
1726 questions.put(&mut q).unwrap();
1727 dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1728 let changed = dep.blocked_at;
1729 queue.put(&mut dep).unwrap();
1730 dep.block(vec![q.id], Some("same decision paraphrased".into()));
1731 assert_eq!(
1732 dep.blocked_at, changed,
1733 "prose is not new blocking evidence"
1734 );
1735 queue.put(&mut dep).unwrap();
1736 let d = Decision {
1737 id: t.id.clone(),
1738 blocked_by: vec![dep.id.clone()],
1739 evidence: vec![dep.id],
1740 ..Decision::default()
1741 };
1742 assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1743 apply_one(&queue, &questions, &d, None).unwrap();
1744 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1745 }
1746
1747 #[test]
1748 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1749 crate::run::pin_test_home();
1750 let dir = tempdir().unwrap();
1751 let queue = Queue::at(dir.path().join("queue"));
1752 let questions = Questions::at(dir.path().join("questions"));
1753 let mut t = task("resume evidence");
1754 let mut run = RunState::new(
1755 t.repo.clone(),
1756 "main".into(),
1757 "base".into(),
1758 "work".into(),
1759 crate::config::Config::default(),
1760 );
1761 run.begin_pass();
1762 run.status = crate::run::RunStatus::Failed;
1763 run.save().unwrap();
1764 t.start(run.id.clone());
1765 t.runs.push(run.id.clone());
1767 t.release_operator("cli", None);
1768 let approval = t.resumption.as_ref().unwrap().at;
1769 let decision = Decision {
1770 evidence: vec![run.id.clone()],
1771 ..Decision::default()
1772 };
1773 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1774 t.start(run.id.clone());
1775 assert_eq!(t.runs.len(), 1);
1776 run.begin_pass();
1777 run.save().unwrap();
1778 assert!(run.passes.last().unwrap().started_at > approval);
1779 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1780 assert_eq!(evidence.len(), 1);
1781 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1782 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1783 run.begin_pass();
1784 run.save().unwrap();
1785 assert_eq!(
1786 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1787 1
1788 );
1789
1790 let mut q = Question::new(
1791 run.id.clone(),
1792 "review".into(),
1793 "check".into(),
1794 "New failure".into(),
1795 String::new(),
1796 Vec::new(),
1797 );
1798 questions.put(&mut q).unwrap();
1799 let decision = Decision {
1800 evidence: vec![q.id.clone()],
1801 ..Decision::default()
1802 };
1803 assert_eq!(
1804 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1805 1
1806 );
1807 q.asked_at = approval;
1808 questions.put(&mut q).unwrap();
1809 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1810 q.asked_at = jiff::Timestamp::now();
1811 q.run = "unrelated".into();
1812 questions.put(&mut q).unwrap();
1813 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1814
1815 let mut fresh = RunState::new(
1816 t.repo.clone(),
1817 "main".into(),
1818 "base".into(),
1819 "fresh work".into(),
1820 crate::config::Config::default(),
1821 );
1822 fresh.status = crate::run::RunStatus::Failed;
1823 fresh.save().unwrap();
1824 t.start(fresh.id.clone());
1825 let decision = Decision {
1826 evidence: vec![fresh.id.clone()],
1827 ..Decision::default()
1828 };
1829 assert_eq!(
1830 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1831 1
1832 );
1833 }
1834
1835 #[test]
1836 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1837 crate::run::pin_test_home();
1838 let dir = tempdir().unwrap();
1839 let default_repo = dir.path().join("default");
1840 let task_repo = dir.path().join("task");
1841 std::fs::create_dir_all(&default_repo).unwrap();
1842 std::fs::create_dir_all(&task_repo).unwrap();
1843 init_repo_with_branch(&default_repo, "other-branch");
1844 init_repo_with_branch(&task_repo, "magi/f00d/A");
1845
1846 let mut config = Config::default();
1847 config.graph.review_rounds = 6;
1848 let mut state = crate::run::RunState::new(
1849 task_repo.clone(),
1850 "main".to_owned(),
1851 "deadbeef".to_owned(),
1852 "task".to_owned(),
1853 config,
1854 );
1855 state.status = crate::run::RunStatus::Blocked;
1856 state.candidates.push(crate::run::Candidate {
1857 index: 0,
1858 label: 'A',
1859 agent: "mock".to_owned(),
1860 branch: "magi/f00d/A".to_owned(),
1861 worktree: task_repo.clone(),
1862 summary: String::new(),
1863 stat: String::new(),
1864 files: 1,
1865 commits: 1,
1866 empty: false,
1867 failed: None,
1868 verified_noop: None,
1869 duration_ms: 0,
1870 folded: false,
1871 });
1872 state.tally = Some(crate::run::Tally {
1873 first_choice: std::collections::BTreeMap::new(),
1874 borda: std::collections::BTreeMap::new(),
1875 winner: 'A',
1876 rankings: 0,
1877 unanimous_initial: false,
1878 deliberated: false,
1879 changed_votes: 0,
1880 unanimous_final: false,
1881 tie_break: None,
1882 judges: 0,
1883 present: 0,
1884 quorum: 0,
1885 met_quorum: true,
1886 uncontested: Some("solo".to_owned()),
1887 });
1888 state.reviews = vec![
1889 review_round_with_finding(
1890 1,
1891 "R1-1-2",
1892 "answer content is dropped",
1893 &[],
1894 &[("R1-1-2", "the id leaving blocked_by is enough")],
1895 ),
1896 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1897 ];
1898 state.save().unwrap();
1899
1900 let mut t = task("outcome test");
1901 t.repo = task_repo;
1902 t.runs.push(state.id.clone());
1903
1904 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1905 let outcome = finished.outcome;
1906
1907 assert!(outcome.unreadable.is_none());
1908 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1909 assert_eq!(outcome.rounds_used, 2);
1910 assert_eq!(outcome.rounds_max, 6);
1911 assert_eq!(outcome.rounds.len(), 2);
1912 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1913 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1914 assert!(outcome.rounds[1].addressed.is_empty());
1915 assert!(outcome.rounds[1].rejected.is_empty());
1916 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1917 assert!(
1918 outcome.branch_head.is_some(),
1919 "a real branch must resolve a head commit: {outcome:?}"
1920 );
1921 }
1922
1923 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1927 tokio::runtime::Builder::new_current_thread()
1928 .enable_all()
1929 .build()
1930 .unwrap()
1931 .block_on(f)
1932 }
1933
1934 #[test]
1935 fn view_resolves_task_attempt_limit_before_daemon_default() {
1936 let mut t = task("budget");
1937 assert_eq!(view(&t, 7).max_attempts, 7);
1938 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1939 .unwrap();
1940 assert_eq!(view(&t, 7).max_attempts, 3);
1941 t.change_attempt_limit(None).unwrap();
1942 assert_eq!(view(&t, 7).max_attempts, 7);
1943 }
1944
1945 #[test]
1946 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1947 let mut t = task("answered");
1948 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1949 let v = view(&t, 2);
1950 assert_eq!(v.answers.len(), 1);
1951 assert_eq!(v.answers[0].question, "Which backend?");
1952 assert_eq!(v.answers[0].answer, "SQLite");
1953 }
1954
1955 #[tokio::test]
1956 async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1957 for held in [false, true] {
1958 for reference in ["ca5b", "20261013-193611-ca5b"] {
1959 for question in [false, true] {
1960 let dir = tempdir().unwrap();
1961 let queue = Queue::at(dir.path().join("queue"));
1962 let questions = Questions::at(dir.path().join("questions"));
1963 let mut dep = task("prerequisite");
1964 dep.id = "20261013-193611-ca5b".into();
1965 dep.start("existing-run".into());
1966 queue.put(&mut dep).unwrap();
1967 let mut waiting = task("waiting");
1968 if held {
1969 waiting.hold_machine(Some("dependency could not be located".into()));
1970 }
1971 queue.put(&mut waiting).unwrap();
1972 let missing_reference = if reference == "ca5b" {
1973 dep.id.as_str()
1974 } else {
1975 "ca5b"
1976 };
1977 let reply = serde_json::json!({"decisions": [{
1978 "id": waiting.id, "blocked_by": [reference],
1979 "missing_dependencies": [missing_reference],
1980 "question": question.then_some("ca5b is missing; where is its replacement?"),
1981 "recovery": (!question).then_some("hold"),
1982 "reason": "ca5b is missing"
1983 }]}).to_string();
1984 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1985 let (queued, finished) = if held {
1986 (vec![], vec![waiting.clone()])
1987 } else {
1988 (vec![waiting.clone()], vec![])
1989 };
1990 Conductor::new()
1991 .run_once(
1992 &cfg,
1993 dir.path(),
1994 &queue,
1995 &questions,
1996 dir.path(),
1997 &queued,
1998 &[],
1999 &finished,
2000 2,
2001 )
2002 .await
2003 .unwrap();
2004 let after = queue.get(&waiting.id).unwrap();
2005 assert_eq!(after.status, TaskStatus::Blocked);
2006 assert_eq!(after.blocked_by, [dep.id]);
2007 assert_eq!(after.blocked_from, Some(waiting.status));
2008 assert!(!after.block_reason.unwrap().contains("missing"));
2009 assert!(questions.list().is_empty());
2010 assert_eq!(after.instruction, waiting.instruction);
2011 }
2012 }
2013 }
2014 }
2015
2016 #[tokio::test]
2017 async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2018 for held in [false, true] {
2019 for unknown in [false, true] {
2020 for question in [false, true] {
2021 let dir = tempdir().unwrap();
2022 let queue = Queue::at(dir.path().join("queue"));
2023 let questions = Questions::at(dir.path().join("questions"));
2024 let mut dep = task("running prerequisite");
2025 dep.id = "20261013-193611-ca5b".into();
2026 dep.start("existing-run".into());
2027 queue.put(&mut dep).unwrap();
2028 let reference = if unknown {
2029 std::fs::write(questions.root(), "not a directory").unwrap();
2032 dep.id.clone()
2033 } else {
2034 let mut other = task("another prerequisite");
2035 other.id = "20261014-193611-ca5b".into();
2036 queue.put(&mut other).unwrap();
2037 "ca5b".to_owned()
2038 };
2039 assert_eq!(
2040 crate::queue::dependency(&queue, &questions, &reference),
2041 if unknown {
2042 crate::queue::Dependency::Unknown
2043 } else {
2044 crate::queue::Dependency::Ambiguous
2045 }
2046 );
2047 let mut waiting = task("waiting");
2048 if held {
2049 waiting.hold_machine(Some("dependency could not be located".into()));
2050 }
2051 queue.put(&mut waiting).unwrap();
2052 let reply = serde_json::json!({"decisions": [{
2053 "id": waiting.id, "blocked_by": [reference],
2054 "missing_dependencies": [reference],
2055 "question": question.then_some("ca5b is missing"),
2056 "recovery": (!question).then_some("hold"),
2057 "reason": "ca5b is missing"
2058 }]})
2059 .to_string();
2060 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2061 let (queued, finished) = if held {
2062 (vec![], vec![waiting.clone()])
2063 } else {
2064 (vec![waiting.clone()], vec![])
2065 };
2066 Conductor::new()
2067 .run_once(
2068 &cfg,
2069 dir.path(),
2070 &queue,
2071 &questions,
2072 dir.path(),
2073 &queued,
2074 &[],
2075 &finished,
2076 2,
2077 )
2078 .await
2079 .unwrap();
2080 let after = queue.get(&waiting.id).unwrap();
2081 assert_eq!(after.status, TaskStatus::Blocked);
2082 assert_eq!(after.blocked_by, [reference]);
2083 assert_eq!(after.blocked_from, Some(waiting.status));
2084 assert!(!after.block_reason.unwrap().contains("missing"));
2085 assert_eq!(after.instruction, waiting.instruction);
2086 if unknown {
2087 assert!(questions.root().is_file());
2088 } else {
2089 assert!(questions.list().is_empty());
2090 }
2091 }
2092 }
2093 }
2094 }
2095
2096 #[tokio::test]
2097 async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2098 for held in [false, true] {
2099 for unknown in [false, true] {
2100 for question in [false, true] {
2101 for short_blocker in [false, true] {
2102 let dir = tempdir().unwrap();
2103 let queue = Queue::at(dir.path().join("queue"));
2104 let questions = Questions::at(dir.path().join("questions"));
2105 let mut dep = task("running prerequisite");
2106 dep.id = "20261013-193611-ca5b".into();
2107 dep.start("existing-run".into());
2108 queue.put(&mut dep).unwrap();
2109 if unknown {
2110 std::fs::write(questions.root(), "not a directory").unwrap();
2111 } else {
2112 let mut other = task("another prerequisite");
2113 other.id = "20261014-193611-ca5b".into();
2114 queue.put(&mut other).unwrap();
2115 }
2116 let (blocker, missing) = if short_blocker {
2117 ("ca5b", dep.id.as_str())
2118 } else {
2119 (dep.id.as_str(), "ca5b")
2120 };
2121 let mut waiting = task("waiting");
2122 if held {
2123 waiting.hold_machine(Some("dependency could not be located".into()));
2124 }
2125 queue.put(&mut waiting).unwrap();
2126 let reply = serde_json::json!({"decisions": [{
2127 "id": waiting.id, "blocked_by": [blocker],
2128 "missing_dependencies": [missing],
2129 "question": question.then_some("ca5b is missing"),
2130 "recovery": (!question).then_some("hold"),
2131 "reason": "ca5b is missing"
2132 }]})
2133 .to_string();
2134 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2135 let (queued, finished) = if held {
2136 (vec![], vec![waiting.clone()])
2137 } else {
2138 (vec![waiting.clone()], vec![])
2139 };
2140 Conductor::new()
2141 .run_once(
2142 &cfg,
2143 dir.path(),
2144 &queue,
2145 &questions,
2146 dir.path(),
2147 &queued,
2148 &[],
2149 &finished,
2150 2,
2151 )
2152 .await
2153 .unwrap();
2154 assert_eq!(
2155 serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2156 serde_json::to_value(&waiting).unwrap(),
2157 "unverifiable aliases must leave the entire task unchanged"
2158 );
2159 if unknown {
2160 assert!(questions.root().is_file());
2161 } else {
2162 assert!(questions.list().is_empty());
2163 }
2164 }
2165 }
2166 }
2167 }
2168 }
2169
2170 #[tokio::test]
2171 async fn dependency_resolution_genuine_missing_reference_still_asks() {
2172 let dir = tempdir().unwrap();
2173 let queue = Queue::at(dir.path().join("queue"));
2174 let questions = Questions::at(dir.path().join("questions"));
2175 let mut waiting = task("missing prerequisite");
2176 queue.put(&mut waiting).unwrap();
2177 let reply = serde_json::json!({"decisions": [{
2178 "id": waiting.id, "blocked_by": ["not-present"],
2179 "missing_dependencies": ["not-present"],
2180 "question": "Which prerequisite replaces the missing task?"
2181 }]})
2182 .to_string();
2183 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2184 Conductor::new()
2185 .run_once(
2186 &cfg,
2187 dir.path(),
2188 &queue,
2189 &questions,
2190 dir.path(),
2191 &[waiting.clone()],
2192 &[],
2193 &[],
2194 2,
2195 )
2196 .await
2197 .unwrap();
2198 let after = queue.get(&waiting.id).unwrap();
2199 assert_eq!(after.status, TaskStatus::Blocked);
2200 assert_eq!(
2201 questions.get(&after.blocked_by[0]).unwrap().summary,
2202 "Which prerequisite replaces the missing task?"
2203 );
2204 }
2205
2206 #[tokio::test]
2207 async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2208 let dir = tempdir().unwrap();
2209 let queue = Queue::at(dir.path().join("queue"));
2210 let questions = Questions::at(dir.path().join("questions"));
2211 let mut dep = task("prerequisite");
2212 dep.id = "20261013-193611-ca5b".into();
2213 dep.start("existing-run".into());
2214 queue.put(&mut dep).unwrap();
2215 let mut unrelated = task("unrelated work");
2216 unrelated.start("other-run".into());
2217 queue.put(&mut unrelated).unwrap();
2218 std::fs::File::options()
2221 .write(true)
2222 .open(queue.path_of(&unrelated.id))
2223 .unwrap()
2224 .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2225 .unwrap();
2226 unrelated.succeed();
2227 let mut waiting = task("waiting");
2228 let mut asking = task("needs a product decision");
2229 let mut holding = task("must stay held");
2230 queue.put(&mut waiting).unwrap();
2231 queue.put(&mut asking).unwrap();
2232 queue.put(&mut holding).unwrap();
2233 let reply = serde_json::json!({"decisions": [
2234 {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2235 {"id": asking.id, "question": "Which behavior do you want?"},
2236 {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2237 ]})
2238 .to_string();
2239 let mut vars = env(&reply);
2240 vars.insert(
2241 "UNRELATED_PATH".into(),
2242 queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2243 );
2244 vars.insert(
2245 "UNRELATED_RECORD".into(),
2246 serde_json::to_string(&unrelated).unwrap(),
2247 );
2248 let script = r#"#!/bin/sh
2249cat >/dev/null
2250printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2251printf '%s\n' "$MOCK_REPLY"
2252"#;
2253 let cfg = config(mock_agent(dir.path(), script, vars));
2254 Conductor::new()
2255 .run_once(
2256 &cfg,
2257 dir.path(),
2258 &queue,
2259 &questions,
2260 dir.path(),
2261 &[waiting.clone(), asking.clone(), holding.clone()],
2262 &[],
2263 &[],
2264 2,
2265 )
2266 .await
2267 .unwrap();
2268 assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2269 let after = queue.get(&waiting.id).unwrap();
2270 assert_eq!(after.status, TaskStatus::Blocked);
2271 assert_eq!(after.blocked_by, [dep.id]);
2272 let asked = queue.get(&asking.id).unwrap();
2273 assert_eq!(asked.status, TaskStatus::Blocked);
2274 assert_eq!(
2275 questions.get(&asked.blocked_by[0]).unwrap().summary,
2276 "Which behavior do you want?"
2277 );
2278 assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2279 }
2280
2281 #[test]
2282 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2283 let dir = tempdir().unwrap();
2284 let queue = Queue::at(dir.path().join("queue"));
2285 let questions = Questions::at(dir.path().join("questions"));
2286 let mut a = task("a");
2287 a.priority = 9;
2288 queue.put(&mut a).unwrap();
2289
2290 let verdict = Verdict {
2291 decisions: vec![Decision {
2292 id: a.id.clone(),
2293 blocked_by: vec!["20260101-000000-dead".to_owned()],
2294 reason: Some("waits on the other task".to_owned()),
2295 recovery: None,
2296 question: None,
2297 choices: Vec::new(),
2298 question_decision: None,
2299 evidence: Vec::new(),
2300 missing_dependencies: Vec::new(),
2301 }],
2302 };
2303 apply(&queue, &questions, &verdict).unwrap();
2304
2305 let back = queue.get(&a.id).unwrap();
2306 assert_eq!(back.status, TaskStatus::Blocked);
2307 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2308 assert_eq!(
2309 back.priority, 9,
2310 "the conductor's reply cannot carry priority"
2311 );
2312 }
2313
2314 #[test]
2315 fn a_question_decision_files_one_and_blocks_on_its_id() {
2316 let dir = tempdir().unwrap();
2317 let queue = Queue::at(dir.path().join("queue"));
2318 let questions = Questions::at(dir.path().join("questions"));
2319 let mut t = task("ambiguous");
2320 queue.put(&mut t).unwrap();
2321
2322 let verdict = Verdict {
2323 decisions: vec![Decision {
2324 id: t.id.clone(),
2325 blocked_by: Vec::new(),
2326 reason: Some("which backend?".to_owned()),
2327 recovery: None,
2328 question: Some("Which storage backend?".to_owned()),
2329 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2330 question_decision: None,
2331 evidence: Vec::new(),
2332 missing_dependencies: Vec::new(),
2333 }],
2334 };
2335 apply(&queue, &questions, &verdict).unwrap();
2336
2337 let back = queue.get(&t.id).unwrap();
2338 assert_eq!(back.status, TaskStatus::Blocked);
2339 assert_eq!(back.blocked_by.len(), 1);
2340 let q = questions.get(&back.blocked_by[0]).unwrap();
2341 assert_eq!(q.summary, "Which storage backend?");
2342 assert_eq!(q.node, NODE);
2343 assert!(q.status.open());
2344 }
2345
2346 #[test]
2347 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2348 let dir = tempdir().unwrap();
2349 let queue = Queue::at(dir.path().join("queue"));
2350 let questions = Questions::at(dir.path().join("questions"));
2351 let mut t = task("asked once");
2352 queue.put(&mut t).unwrap();
2353
2354 let decision = Decision {
2355 id: t.id.clone(),
2356 reason: Some("still deciding".to_owned()),
2357 question: Some("Which backend?".to_owned()),
2358 ..Decision::default()
2359 };
2360 apply(
2361 &queue,
2362 &questions,
2363 &Verdict {
2364 decisions: vec![decision.clone()],
2365 },
2366 )
2367 .unwrap();
2368 assert_eq!(questions.list().len(), 1);
2369 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2370
2371 let mut released = queue.get(&t.id).unwrap();
2376 released.release();
2377 queue.put(&mut released).unwrap();
2378
2379 apply(
2380 &queue,
2381 &questions,
2382 &Verdict {
2383 decisions: vec![decision],
2384 },
2385 )
2386 .unwrap();
2387 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2388 let after = queue.get(&t.id).unwrap();
2389 assert_eq!(
2390 after.blocked_by,
2391 [first_question_id],
2392 "the existing open question is reused, not replaced"
2393 );
2394 }
2395
2396 #[test]
2397 fn a_same_id_question_from_another_node_is_not_reused() {
2398 let dir = tempdir().unwrap();
2399 let queue = Queue::at(dir.path().join("queue"));
2400 let questions = Questions::at(dir.path().join("questions"));
2401 let mut t = task("must ask the conductor");
2402 queue.put(&mut t).unwrap();
2403
2404 let mut unrelated = Question::new(
2405 t.id.clone(),
2406 "review".to_owned(),
2407 "reviewer-1".to_owned(),
2408 "An unrelated review question".to_owned(),
2409 String::new(),
2410 Vec::new(),
2411 );
2412 questions.put(&mut unrelated).unwrap();
2413
2414 apply(
2415 &queue,
2416 &questions,
2417 &Verdict {
2418 decisions: vec![Decision {
2419 id: t.id.clone(),
2420 question: Some("Which backend?".to_owned()),
2421 ..Decision::default()
2422 }],
2423 },
2424 )
2425 .unwrap();
2426
2427 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2428 assert_eq!(blocked_by.len(), 1);
2429 assert_ne!(blocked_by[0], unrelated.id);
2430 assert!(questions.get(&unrelated.id).unwrap().status.open());
2431 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2432 }
2433
2434 #[test]
2435 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2436 let dir = tempdir().unwrap();
2437 let queue = Queue::at(dir.path().join("queue"));
2438 let questions = Questions::at(dir.path().join("questions"));
2439 let mut t = task("waits on an answer");
2440 queue.put(&mut t).unwrap();
2441
2442 apply(
2443 &queue,
2444 &questions,
2445 &Verdict {
2446 decisions: vec![Decision {
2447 id: t.id.clone(),
2448 blocked_by: Vec::new(),
2449 reason: None,
2450 recovery: None,
2451 question: Some("Which backend?".to_owned()),
2452 choices: Vec::new(),
2453 question_decision: None,
2454 evidence: Vec::new(),
2455 missing_dependencies: Vec::new(),
2456 }],
2457 },
2458 )
2459 .unwrap();
2460 let blocked = queue.get(&t.id).unwrap();
2461 let question_id = blocked.blocked_by[0].clone();
2462
2463 let mut q = questions.get(&question_id).unwrap();
2464 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2465 questions.put(&mut q).unwrap();
2466 assert_eq!(q.status, QuestionStatus::Answered);
2467
2468 let mut task_after = queue.get(&t.id).unwrap();
2472 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2473 task_after.unblock(&question_id);
2474 assert_eq!(task_after.status, TaskStatus::Queued);
2475 assert_eq!(task_after.answers[0].answer, "SQLite");
2476 }
2477
2478 #[test]
2479 fn a_stalled_task_can_be_requeued_or_held() {
2480 let dir = tempdir().unwrap();
2481 let queue = Queue::at(dir.path().join("queue"));
2482 let questions = Questions::at(dir.path().join("questions"));
2483
2484 let mut requeue_me = task("stuck a");
2485 requeue_me.start("run-1".to_owned());
2486 queue.put(&mut requeue_me).unwrap();
2487
2488 let mut hold_me = task("stuck b");
2489 hold_me.start("run-2".to_owned());
2490 queue.put(&mut hold_me).unwrap();
2491
2492 apply(
2493 &queue,
2494 &questions,
2495 &Verdict {
2496 decisions: vec![
2497 Decision {
2498 id: requeue_me.id.clone(),
2499 recovery: Some(Recovery::Requeue),
2500 ..Decision::default()
2501 },
2502 Decision {
2503 id: hold_me.id.clone(),
2504 recovery: Some(Recovery::Hold),
2505 reason: Some("looks broken".to_owned()),
2506 ..Decision::default()
2507 },
2508 ],
2509 },
2510 )
2511 .unwrap();
2512
2513 let requeued = queue.get(&requeue_me.id).unwrap();
2514 assert_eq!(requeued.status, TaskStatus::Queued);
2515 assert_eq!(requeued.attempts, 0);
2516
2517 let held = queue.get(&hold_me.id).unwrap();
2518 assert_eq!(held.status, TaskStatus::Held);
2519 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2520 }
2521
2522 #[test]
2523 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2524 let dir = tempdir().unwrap();
2529 let queue = Queue::at(dir.path().join("queue"));
2530 let questions = Questions::at(dir.path().join("questions"));
2531 let mut t = task("held out of attempts");
2532 t.hold_machine(Some("out of attempts".to_owned()));
2533 queue.put(&mut t).unwrap();
2534
2535 apply(
2536 &queue,
2537 &questions,
2538 &Verdict {
2539 decisions: vec![Decision {
2540 id: t.id.clone(),
2541 reason: Some("what should happen to this one?".to_owned()),
2542 question: Some("Hold it, or try again?".to_owned()),
2543 ..Decision::default()
2544 }],
2545 },
2546 )
2547 .unwrap();
2548 let blocked = queue.get(&t.id).unwrap();
2549 assert_eq!(blocked.status, TaskStatus::Blocked);
2550 let question_id = blocked.blocked_by[0].clone();
2551
2552 let mut q = questions.get(&question_id).unwrap();
2553 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2554 questions.put(&mut q).unwrap();
2555
2556 let mut after = queue.get(&t.id).unwrap();
2558 after.record_answer(q.summary.clone(), "leave it held".to_owned());
2559 after.unblock(&question_id);
2560 assert_eq!(
2561 after.status,
2562 TaskStatus::Held,
2563 "must not fall back to queued"
2564 );
2565 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2566 }
2567
2568 #[test]
2569 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2570 let dir = tempdir().unwrap();
2580 let queue = Queue::at(dir.path().join("queue"));
2581 let questions = Questions::at(dir.path().join("questions"));
2582 let mut t = task("disk pressure, then reconsidered");
2583 t.hold_machine(Some(
2584 "not enough free space to start a run: 10 bytes free, 100 required by \
2585 `[disk] min_free_bytes`"
2586 .to_owned(),
2587 ));
2588 t.record_answer(
2589 "How should this be handled?".to_owned(),
2590 "keep it held, a human will look at it later".to_owned(),
2591 );
2592 queue.put(&mut t).unwrap();
2593
2594 apply(
2595 &queue,
2596 &questions,
2597 &Verdict {
2598 decisions: vec![Decision {
2599 id: t.id.clone(),
2600 recovery: Some(Recovery::Hold),
2601 ..Decision::default()
2602 }],
2603 },
2604 )
2605 .unwrap();
2606
2607 let after = queue.get(&t.id).unwrap();
2608 assert_eq!(after.status, TaskStatus::Held);
2609 assert!(
2610 !after
2611 .hold_reason
2612 .as_deref()
2613 .unwrap_or_default()
2614 .starts_with("not enough free space"),
2615 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2616 after.hold_reason
2617 );
2618
2619 let cfg_dir = tempdir().unwrap();
2622 let config = cfg_dir.path().join("magi.toml");
2623 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2624 let report =
2625 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2626 assert!(report.resumed.is_empty(), "must not be auto-released");
2627 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2628 }
2629
2630 #[test]
2631 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2632 crate::run::pin_test_home();
2633 let dir = tempdir().unwrap();
2634 let queue = Queue::at(dir.path().join("queue"));
2635 let questions = Questions::at(dir.path().join("questions"));
2636 let config = dir.path().join("magi.toml");
2637 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2638 let now = jiff::Timestamp::now();
2639 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2640 let open = || {
2641 questions
2642 .list()
2643 .into_iter()
2644 .filter(|q| q.node == "triage" && q.status.open())
2645 .collect::<Vec<_>>()
2646 };
2647 let hold = Decision {
2648 recovery: Some(Recovery::Hold),
2649 reason: Some("waiting on manual worktree cleanup".to_owned()),
2650 ..Decision::default()
2651 };
2652
2653 let mut t = task("looping hold");
2654 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2655 queue.put(&mut t).unwrap();
2656 let hold = Decision {
2657 id: t.id.clone(),
2658 ..hold
2659 };
2660
2661 assert_eq!(triage().asked.len(), 1);
2663 let first = open().remove(0);
2664 let mut q = questions.get(&first.id).unwrap();
2665 let resume = q.choices[0].clone();
2666 q.answer(Answer::Choice(resume)).unwrap();
2667 questions.put(&mut q).unwrap();
2668 assert_eq!(triage().answered.len(), 1);
2669 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2670
2671 apply(
2673 &queue,
2674 &questions,
2675 &Verdict {
2676 decisions: vec![hold.clone()],
2677 },
2678 )
2679 .unwrap();
2680 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2681 assert!(triage().asked.is_empty());
2682
2683 let mut evidence = Question::new(
2685 t.id.clone(),
2686 "preflight".into(),
2687 "check".into(),
2688 "New permission failure".into(),
2689 String::new(),
2690 Vec::new(),
2691 );
2692 questions.put(&mut evidence).unwrap();
2693 let hold = Decision {
2694 evidence: vec![evidence.id.clone()],
2695 reason: Some("new permission check failed".into()),
2696 ..hold
2697 };
2698 apply(
2699 &queue,
2700 &questions,
2701 &Verdict {
2702 decisions: vec![hold.clone()],
2703 },
2704 )
2705 .unwrap();
2706 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2707
2708 assert_eq!(triage().asked.len(), 1);
2710 let second = open().remove(0);
2711 assert!(second.detail.contains("new permission check failed"));
2712 assert!(triage().asked.is_empty(), "no duplicate question");
2713 assert_eq!(open().len(), 1);
2714
2715 let mut q = questions.get(&second.id).unwrap();
2717 let force = q.choices[0].clone();
2718 q.answer(Answer::Choice(force)).unwrap();
2719 questions.put(&mut q).unwrap();
2720 assert_eq!(triage().answered.len(), 1);
2721 apply(
2722 &queue,
2723 &questions,
2724 &Verdict {
2725 decisions: vec![hold],
2726 },
2727 )
2728 .unwrap();
2729 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2730 }
2731
2732 #[test]
2733 fn conductor_resumption_notice_does_not_solicit_unregistered_questions() {
2734 crate::run::pin_test_home();
2735 let dir = tempdir().unwrap();
2736 let queue = Queue::at(dir.path().join("queue"));
2737 let questions = Questions::at(dir.path().join("questions"));
2738
2739 let mut t = task("task with explicit resumption");
2740 t.status = TaskStatus::Queued;
2741 t.resumption = Some(crate::queue::Resumption {
2742 generation: 1,
2743 source: "test".into(),
2744 question_id: Some("q1".into()),
2745 at: jiff::Timestamp::now(),
2746 instruction: "resume".into(),
2747 answers_through: 0,
2748 runs_through: 0,
2749 cleared_reason: None,
2750 blocking_evidence: Vec::new(),
2751 });
2752 queue.put(&mut t).unwrap();
2753
2754 let decision = Decision {
2756 id: t.id.clone(),
2757 question: Some("How should we proceed with this task?".into()),
2758 blocked_by: vec!["dep-x".into()],
2759 reason: Some("needs clarification".into()),
2760 recovery: Some(Recovery::Hold),
2761 ..Decision::default()
2762 };
2763
2764 apply(
2765 &queue,
2766 &questions,
2767 &Verdict {
2768 decisions: vec![decision],
2769 },
2770 )
2771 .unwrap();
2772
2773 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2775 assert!(questions.list().is_empty());
2777
2778 let notices = crate::notices::Notices::open();
2780 let notice = notices
2781 .get(&crate::notices::id_of(&format!(
2782 "conductor-resumption:{}",
2783 t.id
2784 )))
2785 .expect("conductor-resumption notice filed");
2786
2787 assert_eq!(
2788 notice.link,
2789 Some(crate::notices::Link::Task { id: t.id.clone() })
2790 );
2791 assert!(!notice.message.contains("How should we proceed"));
2793 assert!(
2795 notice
2796 .message
2797 .contains("No reply to this notice is needed.")
2798 );
2799 assert!(!notice.message.contains("Some("));
2801 assert!(!notice.message.contains("None"));
2802 assert!(!notice.message.contains("[]"));
2803 assert!(!notice.message.contains("{:?}"));
2804 assert!(notice.message.contains("Proposed dependencies: dep-x."));
2806 }
2807
2808 #[test]
2809 fn a_runnable_task_can_be_held_directly_without_a_question() {
2810 let dir = tempdir().unwrap();
2811 let queue = Queue::at(dir.path().join("queue"));
2812 let questions = Questions::at(dir.path().join("questions"));
2813 let mut t = task("already answered, should stay put");
2814 queue.put(&mut t).unwrap();
2815
2816 apply(
2817 &queue,
2818 &questions,
2819 &Verdict {
2820 decisions: vec![Decision {
2821 id: t.id.clone(),
2822 recovery: Some(Recovery::Hold),
2823 reason: Some("operator already said keep this held".to_owned()),
2824 ..Decision::default()
2825 }],
2826 },
2827 )
2828 .unwrap();
2829
2830 let after = queue.get(&t.id).unwrap();
2831 assert_eq!(after.status, TaskStatus::Held);
2832 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2833 }
2834
2835 #[test]
2836 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2837 crate::run::pin_test_home();
2838 let dir = tempdir().unwrap();
2844 let queue = Queue::at(dir.path().join("queue"));
2845 let questions = Questions::at(dir.path().join("questions"));
2846 let mut t = task("already merged by hand");
2847 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2848 t.record_answer(
2849 "Handle this one?".to_owned(),
2850 "already merged and cleaned up, close it".to_owned(),
2851 );
2852 queue.put(&mut t).unwrap();
2853
2854 apply(
2855 &queue,
2856 &questions,
2857 &Verdict {
2858 decisions: vec![Decision {
2859 id: t.id.clone(),
2860 recovery: Some(Recovery::Done),
2861 reason: Some("operator confirmed this already landed".to_owned()),
2862 ..Decision::default()
2863 }],
2864 },
2865 )
2866 .unwrap();
2867
2868 let after = queue.get(&t.id).unwrap();
2869 assert_eq!(after.status, TaskStatus::Done);
2870 assert!(after.hold_reason.is_none());
2871 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2872 }
2873
2874 #[test]
2875 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2876 let dir = tempdir().unwrap();
2877 let queue = Queue::at(dir.path().join("queue"));
2878 let questions = Questions::at(dir.path().join("questions"));
2879
2880 let mut queued = task("never ran yet");
2881 queue.put(&mut queued).unwrap();
2882
2883 let mut running = task("mid-run");
2884 running.start("run-1".to_owned());
2885 queue.put(&mut running).unwrap();
2886
2887 for id in [queued.id.clone(), running.id.clone()] {
2888 apply(
2889 &queue,
2890 &questions,
2891 &Verdict {
2892 decisions: vec![Decision {
2893 id,
2894 recovery: Some(Recovery::Done),
2895 ..Decision::default()
2896 }],
2897 },
2898 )
2899 .unwrap();
2900 }
2901
2902 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2903 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2904 }
2905
2906 #[test]
2907 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2908 let dir = tempdir().unwrap();
2911 let queue = Queue::at(dir.path().join("queue"));
2912 let questions = Questions::at(dir.path().join("questions"));
2913 let mut t = task("asked about repeatedly");
2914 t.hold_machine(Some("out of attempts".to_owned()));
2915 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2916 t.record_answer(
2917 "Handle this one? (2)".to_owned(),
2918 "still not yet".to_owned(),
2919 );
2920 queue.put(&mut t).unwrap();
2921 assert_eq!(questions.list().len(), 0);
2922
2923 let verdict = Verdict {
2924 decisions: vec![Decision {
2925 id: t.id.clone(),
2926 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2927 ..Decision::default()
2928 }],
2929 };
2930 apply(&queue, &questions, &verdict).unwrap();
2931
2932 let filed = questions.list();
2933 assert_eq!(filed.len(), 1, "the question was filed");
2934 assert_eq!(
2935 filed[0].summary,
2936 "Branch conflicts with origin/main, how do we proceed?"
2937 );
2938 let deputy = filed[0]
2939 .deputy
2940 .as_ref()
2941 .expect("the wait is handed to a deputy");
2942 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2943 assert_eq!(
2944 deputy.starts, 0,
2945 "filing never starts anything: the loop does not wait"
2946 );
2947 assert!(filed[0].status.open());
2948 assert_eq!(filed[0].node, NODE);
2949 let after = queue.get(&t.id).unwrap();
2950 assert_eq!(after.status, TaskStatus::Blocked);
2951 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2952 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2953 assert!(
2954 !after
2955 .hold_reason
2956 .clone()
2957 .unwrap_or_default()
2958 .contains("conduct tried to ask"),
2959 "no hold was applied"
2960 );
2961
2962 apply(&queue, &questions, &verdict).unwrap();
2963 let again = questions.list();
2964 assert_eq!(again.len(), 1, "the open question is reused");
2965 assert_eq!(again[0].id, filed[0].id);
2966 }
2967
2968 #[test]
2969 fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
2970 let dir = tempdir().unwrap();
2971 init_repo_with_branch(dir.path(), "source-a");
2972 let mut t = task("source decision");
2973 t.repo = dir.path().into();
2974 let decision = |reference: &str| QuestionDecision {
2975 operation: QuestionOperation::Recover,
2976 target: QuestionTarget::Source {
2977 reference: reference.into(),
2978 },
2979 };
2980 assert_eq!(
2981 decision("source-a").identity(&t).unwrap(),
2982 decision("refs/heads/source-a").identity(&t).unwrap()
2983 );
2984 assert_ne!(
2985 decision("source-a").identity(&t).unwrap(),
2986 decision("main").identity(&t).unwrap()
2987 );
2988 assert!(decision("invented-source-label").identity(&t).is_err());
2989 let unrelated = QuestionDecision {
2990 operation: QuestionOperation::Resume,
2991 target: QuestionTarget::Run {
2992 id: "unrelated-run".into(),
2993 },
2994 };
2995 assert!(unrelated.identity(&t).is_err());
2996 }
2997
2998 #[test]
2999 fn invalid_question_targets_still_file_and_block_queued_tasks() {
3000 let dir = tempdir().unwrap();
3001 init_repo_with_branch(dir.path(), "source-a");
3002 let queue = Queue::at(dir.path().join("queue"));
3003 let questions = Questions::at(dir.path().join("questions"));
3004 for target in [
3005 QuestionTarget::Source {
3006 reference: "missing-source".into(),
3007 },
3008 QuestionTarget::Run {
3009 id: "unrelated-run".into(),
3010 },
3011 ] {
3012 let mut t = task("decision with invalid metadata");
3013 t.repo = dir.path().into();
3014 queue.put(&mut t).unwrap();
3015 let d = Decision {
3016 id: t.id.clone(),
3017 question: Some("Which recovery should proceed?".into()),
3018 choices: vec!["inspect first".into(), "keep waiting".into()],
3019 question_decision: Some(QuestionDecision {
3020 operation: QuestionOperation::Recover,
3021 target,
3022 }),
3023 ..Decision::default()
3024 };
3025 apply_one(&queue, &questions, &d, None).unwrap();
3026 let q = questions
3027 .list()
3028 .into_iter()
3029 .find(|q| q.run == t.id)
3030 .unwrap();
3031 assert_eq!(q.summary, d.question.as_deref().unwrap());
3032 assert_eq!(q.choices, d.choices);
3033 assert!(q.status.open());
3034 let context: serde_json::Value =
3035 serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
3036 assert!(context[3].is_null(), "invalid metadata was discarded");
3037 let after = queue.get(&t.id).unwrap();
3038 assert_eq!(after.status, TaskStatus::Blocked);
3039 assert_eq!(after.blocked_by, vec![q.id.clone()]);
3040 apply_one(&queue, &questions, &d, None).unwrap();
3041 assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
3042 }
3043 }
3044
3045 #[test]
3046 fn answered_conductor_question_needs_changed_facts_before_reasking() {
3047 let dir = tempdir().unwrap();
3048 let queue = Queue::at(dir.path().join("queue"));
3049 let questions = Questions::at(dir.path().join("questions"));
3050 let mut t = task("recovery decision");
3051 t.hold_machine(Some("out of attempts".into()));
3052 t.runs = vec!["source-run-a".into(), "source-run-b".into()];
3053 queue.put(&mut t).unwrap();
3054 let mut d = Decision {
3055 id: t.id.clone(),
3056 question: Some("Recover from source A?".into()),
3057 choices: vec!["approve source A".into(), "keep held".into()],
3058 question_decision: Some(QuestionDecision {
3059 operation: QuestionOperation::Recover,
3060 target: QuestionTarget::Run {
3061 id: "source-run-a".into(),
3062 },
3063 }),
3064 ..Decision::default()
3065 };
3066 apply_one(&queue, &questions, &d, None).unwrap();
3067 let mut q = questions.list().pop().unwrap();
3068 q.answer(crate::ask::Answer::Choice("keep held".into()))
3069 .unwrap();
3070 questions.put(&mut q).unwrap();
3071 queue
3072 .modify(&t.id, |task| {
3073 task.unblock(&q.id);
3074 task.hold_machine(Some("recovery remains under consideration".into()));
3075 true
3076 })
3077 .unwrap();
3078 d.question = Some("Should we keep waiting on recovery?".into());
3079 d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3080 apply_one(&queue, &questions, &d, None).unwrap();
3081 assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3082 d.question = Some("Recover from source B?".into());
3083 d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3085 id: "source-run-b".into(),
3086 };
3087 apply_one(&queue, &questions, &d, None).unwrap();
3088 assert_eq!(
3089 questions.list().len(),
3090 2,
3091 "new decision with unchanged task facts"
3092 );
3093 let after = queue.get(&t.id).unwrap();
3094 assert_eq!(after.instruction, t.instruction);
3095 assert_eq!(after.runs, t.runs);
3096 assert_eq!(after.status, TaskStatus::Blocked);
3097 }
3098
3099 #[test]
3100 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3101 let dir = tempdir().unwrap();
3102 let queue = Queue::at(dir.path().join("queue"));
3103 let questions = Questions::at(dir.path().join("questions"));
3104 let mut t = task("asked about once already");
3105 t.hold_machine(Some("out of attempts".to_owned()));
3106 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3107 queue.put(&mut t).unwrap();
3108
3109 apply(
3110 &queue,
3111 &questions,
3112 &Verdict {
3113 decisions: vec![Decision {
3114 id: t.id.clone(),
3115 question: Some("Still not sure - now what?".to_owned()),
3116 ..Decision::default()
3117 }],
3118 },
3119 )
3120 .unwrap();
3121
3122 assert_eq!(questions.list().len(), 1, "the second question was filed");
3123 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3124 }
3125
3126 #[test]
3127 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3128 let dir = tempdir().unwrap();
3134 let queue = Queue::at(dir.path().join("queue"));
3135 let questions = Questions::at(dir.path().join("questions"));
3136 let mut t = task("held, triage already asking about it");
3137 t.hold_machine(Some("cause unclear".to_owned()));
3138 queue.put(&mut t).unwrap();
3139
3140 let mut triage_q = Question::new(
3141 t.id.clone(),
3142 crate::triage::NODE.to_owned(),
3143 "triage".to_owned(),
3144 "Still needed?".to_owned(),
3145 String::new(),
3146 vec![
3147 "resume".to_owned(),
3148 "not yet".to_owned(),
3149 "discard".to_owned(),
3150 ],
3151 );
3152 questions.put(&mut triage_q).unwrap();
3153
3154 for decision in [
3155 Decision {
3156 id: t.id.clone(),
3157 question: Some("what now?".to_owned()),
3158 ..Decision::default()
3159 },
3160 Decision {
3161 id: t.id.clone(),
3162 recovery: Some(Recovery::Requeue),
3163 ..Decision::default()
3164 },
3165 ] {
3166 apply(
3167 &queue,
3168 &questions,
3169 &Verdict {
3170 decisions: vec![decision],
3171 },
3172 )
3173 .unwrap();
3174 }
3175
3176 let after = queue.get(&t.id).unwrap();
3177 assert_eq!(
3178 after.status,
3179 TaskStatus::Held,
3180 "triage still owns this hold"
3181 );
3182 assert!(after.blocked_by.is_empty());
3183 assert_eq!(
3184 questions.list().len(),
3185 1,
3186 "no second, conductor-owned question was filed"
3187 );
3188 }
3189
3190 #[test]
3191 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3192 let dir = tempdir().unwrap();
3200 let queue = Queue::at(dir.path().join("queue"));
3201 let questions = Questions::at(dir.path().join("questions"));
3202 let mut t = task("held, triage question answered but not yet applied");
3203 t.hold_machine(Some("cause unclear".to_owned()));
3204 queue.put(&mut t).unwrap();
3205
3206 let mut triage_q = Question::new(
3207 t.id.clone(),
3208 crate::triage::NODE.to_owned(),
3209 "triage".to_owned(),
3210 "Still needed?".to_owned(),
3211 String::new(),
3212 vec![
3213 "resume".to_owned(),
3214 "not yet".to_owned(),
3215 "discard".to_owned(),
3216 ],
3217 );
3218 questions.put(&mut triage_q).unwrap();
3219 triage_q
3220 .answer(Answer::Choice("not yet".to_owned()))
3221 .unwrap();
3222 questions.put(&mut triage_q).unwrap();
3223 assert!(!triage_q.status.open());
3224
3225 apply(
3226 &queue,
3227 &questions,
3228 &Verdict {
3229 decisions: vec![Decision {
3230 id: t.id.clone(),
3231 question: Some("what now?".to_owned()),
3232 ..Decision::default()
3233 }],
3234 },
3235 )
3236 .unwrap();
3237
3238 let after = queue.get(&t.id).unwrap();
3239 assert_eq!(
3240 after.status,
3241 TaskStatus::Held,
3242 "triage's own answer is not yet applied - conduct must wait"
3243 );
3244 assert_eq!(
3245 questions.list().len(),
3246 1,
3247 "no conductor question was filed over the pending triage answer"
3248 );
3249 }
3250
3251 #[test]
3252 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3253 let dir = tempdir().unwrap();
3254 let queue = Queue::at(dir.path().join("queue"));
3255 let questions = Questions::at(dir.path().join("questions"));
3256 let mut held = task("manual recovery");
3257 held.priority = 300;
3258 held.runs.push("run20260912-224242-daf5".to_owned());
3259 held.hold_manual(Some(
3260 "active manual recovery run20260912-224242-daf5".to_owned(),
3261 ));
3262 queue.put(&mut held).unwrap();
3263
3264 for decision in [
3268 Decision {
3269 id: held.id.clone(),
3270 recovery: Some(Recovery::Requeue),
3271 ..Decision::default()
3272 },
3273 Decision {
3274 id: held.id.clone(),
3275 recovery: Some(Recovery::Hold),
3276 reason: Some("stale replacement reason".to_owned()),
3277 ..Decision::default()
3278 },
3279 Decision {
3280 id: held.id.clone(),
3281 recovery: Some(Recovery::Review),
3282 ..Decision::default()
3283 },
3284 Decision {
3285 id: held.id.clone(),
3286 blocked_by: vec!["other-task".to_owned()],
3287 question: Some("retry now?".to_owned()),
3288 ..Decision::default()
3289 },
3290 ] {
3291 apply(
3292 &queue,
3293 &questions,
3294 &Verdict {
3295 decisions: vec![decision],
3296 },
3297 )
3298 .unwrap();
3299 }
3300
3301 let after = queue.get(&held.id).unwrap();
3302 assert_eq!(after.status, TaskStatus::Held);
3303 assert!(after.operator_held());
3304 assert_eq!(after.priority, 300);
3305 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3306 assert_eq!(
3307 after.hold_reason.as_deref(),
3308 Some("active manual recovery run20260912-224242-daf5")
3309 );
3310 assert!(after.blocked_by.is_empty());
3311 assert!(questions.list().is_empty());
3312 assert!(
3313 queue.next_runnable().is_none(),
3314 "must not dispatch a duplicate"
3315 );
3316 }
3317
3318 #[test]
3319 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3320 let dir = tempdir().unwrap();
3321 let queue = Queue::at(dir.path().join("queue"));
3322 let questions = Questions::at(dir.path().join("questions"));
3323
3324 let mut automatic = task("disk gate");
3325 automatic.hold_machine(Some("disk full".to_owned()));
3326 queue.put(&mut automatic).unwrap();
3327 let requeue = || Verdict {
3328 decisions: vec![Decision {
3329 id: automatic.id.clone(),
3330 recovery: Some(Recovery::Requeue),
3331 ..Decision::default()
3332 }],
3333 };
3334 apply(&queue, &questions, &requeue()).unwrap();
3335 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3336
3337 let mut manual = task("operator gate");
3338 manual.hold_manual(Some("wait for operator".to_owned()));
3339 queue.put(&mut manual).unwrap();
3340 apply(
3341 &queue,
3342 &questions,
3343 &Verdict {
3344 decisions: vec![Decision {
3345 id: manual.id.clone(),
3346 recovery: Some(Recovery::Requeue),
3347 ..Decision::default()
3348 }],
3349 },
3350 )
3351 .unwrap();
3352 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3353
3354 let mut released = queue.get(&manual.id).unwrap();
3357 released.release();
3358 queue.put(&mut released).unwrap();
3359 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3360 }
3361
3362 #[test]
3363 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3364 let dir = tempdir().unwrap();
3365 let queue = Queue::at(dir.path().join("queue"));
3366 let questions = Questions::at(dir.path().join("questions"));
3367 let mut legacy = task("old explicit hold");
3368 legacy.status = TaskStatus::Held;
3369 legacy.hold_reason = Some("manual recovery already active".to_owned());
3370 legacy.hold_source = None;
3371 legacy.blocked_by = vec!["dependency".to_owned()];
3372 queue.put(&mut legacy).unwrap();
3373
3374 apply(
3375 &queue,
3376 &questions,
3377 &Verdict {
3378 decisions: vec![Decision {
3379 id: legacy.id.clone(),
3380 recovery: Some(Recovery::Requeue),
3381 ..Decision::default()
3382 }],
3383 },
3384 )
3385 .unwrap();
3386
3387 let after = queue.get(&legacy.id).unwrap();
3388 assert_eq!(after.status, TaskStatus::Held);
3389 assert_eq!(after.hold_source, None);
3390 assert_eq!(after.hold_reason, legacy.hold_reason);
3391 assert_eq!(after.blocked_by, legacy.blocked_by);
3392 }
3393
3394 #[test]
3395 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3396 crate::run::pin_test_home();
3402 let dir = tempdir().unwrap();
3403 let queue = Queue::at(dir.path().join("queue"));
3404 let questions = Questions::at(dir.path().join("questions"));
3405 let mut t = task("blocked with no readable run");
3406 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
3408 queue.put(&mut t).unwrap();
3409
3410 apply(
3411 &queue,
3412 &questions,
3413 &Verdict {
3414 decisions: vec![Decision {
3415 id: t.id.clone(),
3416 recovery: Some(Recovery::Review),
3417 ..Decision::default()
3418 }],
3419 },
3420 )
3421 .unwrap();
3422
3423 let after = queue.get(&t.id).unwrap();
3424 assert_eq!(
3425 after.status,
3426 TaskStatus::Failed,
3427 "with nothing to reopen, the decision is dropped rather than guessed at"
3428 );
3429 assert!(after.review_branch.is_none());
3430 }
3431
3432 #[test]
3433 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3434 let dir = tempdir().unwrap();
3438 let queue = Queue::at(dir.path().join("queue"));
3439 let questions = Questions::at(dir.path().join("questions"));
3440
3441 for recovery in [Recovery::Requeue, Recovery::Review] {
3442 let mut t = task("ordinary");
3443 queue.put(&mut t).unwrap();
3444
3445 apply(
3446 &queue,
3447 &questions,
3448 &Verdict {
3449 decisions: vec![Decision {
3450 id: t.id.clone(),
3451 recovery: Some(recovery),
3452 ..Decision::default()
3453 }],
3454 },
3455 )
3456 .unwrap();
3457
3458 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3459 }
3460 }
3461
3462 #[tokio::test]
3463 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3464 let dir = tempdir().unwrap();
3465 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3466 let queue = Queue::at(dir.path().join("queue"));
3467 let questions = Questions::at(dir.path().join("questions"));
3468 let mut t = task("normal");
3469 queue.put(&mut t).unwrap();
3470
3471 let mut conductor = Conductor::new();
3472 conductor
3473 .maybe_run(
3474 &cfg,
3475 dir.path(),
3476 &queue,
3477 &questions,
3478 dir.path(),
3479 &[t.clone()],
3480 &[],
3481 &[],
3482 2,
3483 )
3484 .await;
3485
3486 assert_eq!(
3487 queue.get(&t.id).unwrap().status,
3488 TaskStatus::Queued,
3489 "a failed invocation must change nothing"
3490 );
3491 assert!(
3492 queue.next_runnable().is_some(),
3493 "the loop must still be able to take the next task"
3494 );
3495 }
3496
3497 #[tokio::test]
3498 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3499 let dir = tempdir().unwrap();
3500 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3501 let queue = Queue::at(dir.path().join("queue"));
3502 let questions = Questions::at(dir.path().join("questions"));
3503 let mut t = task("normal");
3504 queue.put(&mut t).unwrap();
3505
3506 let mut conductor = Conductor::new();
3507 conductor
3508 .maybe_run(
3509 &cfg,
3510 dir.path(),
3511 &queue,
3512 &questions,
3513 dir.path(),
3514 &[t.clone()],
3515 &[],
3516 &[],
3517 2,
3518 )
3519 .await;
3520
3521 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3522 }
3523
3524 #[tokio::test]
3525 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3526 let dir = tempdir().unwrap();
3527 let mut t = task("chained");
3528 let reply = format!(
3529 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3530 t.id
3531 );
3532 let counted = |id: &str, tail: &str| {
3533 let calls = dir.path().join(format!("{id}.calls"));
3534 let path = dir.path().join(format!("{id}.sh"));
3535 std::fs::write(
3536 &path,
3537 format!(
3538 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3539 calls.display()
3540 ),
3541 )
3542 .unwrap();
3543 AgentSpec {
3544 id: id.to_owned(),
3545 kind: AgentKind::Command,
3546 model: None,
3547 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3548 extra_args: Vec::new(),
3549 env: BTreeMap::new(),
3550 prompt_delivery: None,
3551 }
3552 };
3553 let n_calls = |id: &str| {
3554 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3555 .map_or(0, |s| s.lines().count())
3556 };
3557 let mut cfg = config(counted("a", "exit 3"));
3558 cfg.agents = vec![
3559 counted("a", "exit 3"),
3560 counted("b", &format!("printf '%s' '{reply}'")),
3561 ];
3562 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3563 "a".into(),
3564 "b".into(),
3565 "a".into(),
3566 ]));
3567 let queue = Queue::at(dir.path().join("queue"));
3568 let questions = Questions::at(dir.path().join("questions"));
3569 queue.put(&mut t).unwrap();
3570
3571 let mut conductor = Conductor::new();
3572 conductor
3573 .maybe_run(
3574 &cfg,
3575 dir.path(),
3576 &queue,
3577 &questions,
3578 dir.path(),
3579 &[t.clone()],
3580 &[],
3581 &[],
3582 2,
3583 )
3584 .await;
3585
3586 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3587 assert_eq!(n_calls("a"), 1, "each id is tried once");
3588 assert_eq!(n_calls("b"), 1);
3589 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3590 assert!(
3591 saved.contains("\"b\""),
3592 "the seat on disk is the one that ran: {saved}"
3593 );
3594 }
3595
3596 #[tokio::test]
3597 async fn json_survives_code_fences_and_a_preamble() {
3598 let dir = tempdir().unwrap();
3599 let mut t = task("fenced");
3600 let reply = format!(
3601 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3602 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3603 t.id
3604 );
3605 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3606 let queue = Queue::at(dir.path().join("queue"));
3607 let questions = Questions::at(dir.path().join("questions"));
3608 queue.put(&mut t).unwrap();
3609
3610 let mut conductor = Conductor::new();
3611 conductor
3612 .maybe_run(
3613 &cfg,
3614 dir.path(),
3615 &queue,
3616 &questions,
3617 dir.path(),
3618 &[t.clone()],
3619 &[],
3620 &[],
3621 2,
3622 )
3623 .await;
3624
3625 let back = queue.get(&t.id).unwrap();
3626 assert_eq!(back.status, TaskStatus::Blocked);
3627 assert_eq!(back.blocked_by, ["x"]);
3628 }
3629
3630 #[tokio::test]
3631 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3632 let dir = tempdir().unwrap();
3635 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3636 let queue = Queue::at(dir.path().join("queue"));
3637 let questions = Questions::at(dir.path().join("questions"));
3638 let mut t = task("stable");
3639 queue.put(&mut t).unwrap();
3640 let artifacts = dir.path().join("conduct").join("artifacts");
3641 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3642
3643 let mut conductor = Conductor::new();
3644 conductor
3645 .maybe_run(
3646 &cfg,
3647 dir.path(),
3648 &queue,
3649 &questions,
3650 dir.path(),
3651 &[t.clone()],
3652 &[],
3653 &[],
3654 2,
3655 )
3656 .await;
3657 assert!(turn(1).is_file(), "the first cycle must call the conductor");
3658
3659 conductor
3660 .maybe_run(
3661 &cfg,
3662 dir.path(),
3663 &queue,
3664 &questions,
3665 dir.path(),
3666 &[t.clone()],
3667 &[],
3668 &[],
3669 2,
3670 )
3671 .await;
3672 assert!(
3673 !turn(2).is_file(),
3674 "an unchanged revision and an unchanged stalled/finished set must not call the \
3675 conductor twice"
3676 );
3677
3678 t.priority = 1;
3680 queue.put(&mut t).unwrap();
3681 conductor
3682 .maybe_run(
3683 &cfg,
3684 dir.path(),
3685 &queue,
3686 &questions,
3687 dir.path(),
3688 &[t.clone()],
3689 &[],
3690 &[],
3691 2,
3692 )
3693 .await;
3694 assert!(turn(2).is_file(), "a moved revision calls it again");
3695 }
3696
3697 #[tokio::test]
3698 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3699 let dir = tempdir().unwrap();
3705 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3706 let queue = Queue::at(dir.path().join("queue"));
3707 let questions = Questions::at(dir.path().join("questions"));
3708 let mut t = task("quiet");
3709 queue.put(&mut t).unwrap();
3710 let artifacts = dir.path().join("conduct").join("artifacts");
3711 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3712
3713 let mut conductor = Conductor::new();
3714 conductor
3715 .maybe_run(
3716 &cfg,
3717 dir.path(),
3718 &queue,
3719 &questions,
3720 dir.path(),
3721 &[t.clone()],
3722 &[],
3723 &[],
3724 2,
3725 )
3726 .await;
3727 assert!(turn(1).is_file());
3728
3729 conductor
3730 .maybe_run(
3731 &cfg,
3732 dir.path(),
3733 &queue,
3734 &questions,
3735 dir.path(),
3736 &[],
3737 &[t.clone()],
3738 &[],
3739 2,
3740 )
3741 .await;
3742 assert!(
3743 turn(2).is_file(),
3744 "a task turning stalled must call the conductor again"
3745 );
3746
3747 conductor
3750 .maybe_run(
3751 &cfg,
3752 dir.path(),
3753 &queue,
3754 &questions,
3755 dir.path(),
3756 &[],
3757 &[t.clone()],
3758 &[],
3759 2,
3760 )
3761 .await;
3762 assert!(
3763 !turn(3).is_file(),
3764 "the same stalled task lingering must not call the conductor every cycle"
3765 );
3766 }
3767
3768 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3769 apply(
3770 queue,
3771 questions,
3772 &Verdict {
3773 decisions: vec![Decision {
3774 id: id.to_owned(),
3775 reason: Some(note.to_owned()),
3776 recovery: Some(Recovery::Hold),
3777 ..Decision::default()
3778 }],
3779 },
3780 )
3781 .unwrap();
3782 }
3783
3784 #[test]
3785 fn reholding_with_an_identical_note_changes_nothing() {
3786 let dir = tempdir().unwrap();
3787 let queue = Queue::at(dir.path().join("queue"));
3788 let questions = Questions::at(dir.path().join("questions"));
3789 let mut t = task("held");
3790 t.hold_machine(Some("waiting on a human".to_owned()));
3791 queue.put(&mut t).unwrap();
3792
3793 rehold(&queue, &questions, &t.id, "owner said keep it held");
3794 let first = queue.get(&t.id).unwrap();
3795 let rev = queue.revision();
3796 for _ in 0..5 {
3797 rehold(&queue, &questions, &t.id, "owner said keep it held");
3798 }
3799 let after = queue.get(&t.id).unwrap();
3800 assert_eq!(after.hold_reason, first.hold_reason);
3801 assert_eq!(after.updated_at, first.updated_at);
3802 assert_eq!(queue.revision(), rev, "nothing was written");
3803 }
3804
3805 #[test]
3806 fn reholding_with_different_notes_keeps_one_previously_level() {
3807 let dir = tempdir().unwrap();
3808 let queue = Queue::at(dir.path().join("queue"));
3809 let questions = Questions::at(dir.path().join("questions"));
3810 let mut t = task("held");
3811 t.hold_machine(Some("original".to_owned()));
3812 queue.put(&mut t).unwrap();
3813
3814 for i in 0..20 {
3815 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3816 }
3817 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3818 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3819 assert!(reason.starts_with("note "), "the new note leads: {reason}");
3820 }
3821
3822 #[test]
3823 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3824 let mut t = task("deep");
3825 let nested = "same\n\n(previously: same\n\n(previously: same))";
3826 t.hold_machine(Some(nested.to_owned()));
3827 let d = Decision {
3828 id: t.id.clone(),
3829 reason: Some("same".to_owned()),
3830 ..Decision::default()
3831 };
3832 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3833 let d2 = Decision {
3834 reason: Some("other".to_owned()),
3835 ..d
3836 };
3837 assert_eq!(
3838 reaffirmed_hold_reason(&t, &d2).as_deref(),
3839 Some("other\n\n(previously: same)")
3840 );
3841 }
3842
3843 #[tokio::test]
3844 async fn a_repeated_hold_decision_settles_instead_of_looping() {
3845 let dir = tempdir().unwrap();
3846 let queue = Queue::at(dir.path().join("queue"));
3847 let questions = Questions::at(dir.path().join("questions"));
3848 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3849 let mut t = task("held");
3850 t.hold_machine(Some("first".to_owned()));
3851 queue.put(&mut t).unwrap();
3852 let cfg = config(mock_agent(
3853 dir.path(),
3854 REPLY,
3855 env(&reply.replace("ID", &t.id)),
3856 ));
3857 let mut conductor = Conductor::new();
3858 for _ in 0..2 {
3860 let held = queue.get(&t.id).unwrap();
3861 conductor
3862 .maybe_run(
3863 &cfg,
3864 dir.path(),
3865 &queue,
3866 &questions,
3867 dir.path(),
3868 &[],
3869 &[],
3870 &[held],
3871 2,
3872 )
3873 .await;
3874 }
3875 let held = queue.get(&t.id).unwrap();
3876 assert!(
3877 held.hold_reason
3878 .as_deref()
3879 .unwrap()
3880 .starts_with("keep held")
3881 );
3882 assert!(
3883 !conductor.worth_a_look(&queue, &[], &[held]),
3884 "an unchanged held task must stop being reconsidered"
3885 );
3886 }
3887
3888 #[tokio::test]
3889 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3890 let dir = tempdir().unwrap();
3891 let queue = Queue::at(dir.path().join("queue"));
3892 let questions = Questions::at(dir.path().join("questions"));
3893 let mut t = task("held");
3894 t.hold_machine(Some("first".to_owned()));
3895 queue.put(&mut t).unwrap();
3896 let reply = |note: &str| {
3897 format!(
3898 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3899 t.id
3900 )
3901 };
3902 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3903 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3904 let mut conductor = Conductor::new();
3905 let mut writes = 0;
3906 for i in 0..6 {
3907 let held = queue.get(&t.id).unwrap();
3908 let before = queue.revision();
3909 conductor
3910 .maybe_run(
3911 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3912 dir.path(),
3913 &queue,
3914 &questions,
3915 dir.path(),
3916 &[],
3917 &[],
3918 &[held],
3919 2,
3920 )
3921 .await;
3922 if queue.revision() != before {
3923 writes += 1;
3924 }
3925 }
3926 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3927 let mut held = queue.get(&t.id).unwrap();
3928 held.instruction.push_str(" (edited)");
3929 queue.put(&mut held).unwrap();
3930 let held = queue.get(&t.id).unwrap();
3931 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3932 }
3933
3934 #[test]
3935 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3936 let dir = tempdir().unwrap();
3937 let queue = Queue::at(dir.path().join("queue"));
3938 let mut t = task("t");
3939 queue.put(&mut t).unwrap();
3940
3941 let mut conductor = Conductor::new();
3942 assert!(
3943 conductor.worth_a_look(&queue, &[], &[]),
3944 "a conductor that has never run has something to look at"
3945 );
3946
3947 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3948 assert!(
3949 !conductor.worth_a_look(&queue, &[], &[]),
3950 "nothing changed and nothing is stalled or finished"
3951 );
3952 assert!(
3953 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3954 "a stalled task is worth a look even at the same revision"
3955 );
3956 assert!(
3957 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3958 "a finished task is worth a look even at the same revision"
3959 );
3960 }
3961
3962 #[tokio::test]
3963 async fn the_conduct_path_never_calls_ask_and_wait() {
3964 let dir = tempdir().unwrap();
3970 let queue = Queue::at(dir.path().join("queue"));
3971 let questions = Questions::at(dir.path().join("questions"));
3972 let mut t = task("asks without blocking");
3973 queue.put(&mut t).unwrap();
3974
3975 apply(
3976 &queue,
3977 &questions,
3978 &Verdict {
3979 decisions: vec![Decision {
3980 id: t.id.clone(),
3981 question: Some("ok?".to_owned()),
3982 ..Decision::default()
3983 }],
3984 },
3985 )
3986 .unwrap();
3987 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3989 }
3990
3991 #[test]
3992 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3993 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3994 t.resume_override = Some(crate::queue::OperatorResume {
3995 question_id: "q".into(),
3996 at: jiff::Timestamp::now(),
3997 conductor_rehold: None,
3998 forced: false,
3999 pinned_run: Some("run-1".into()),
4000 });
4001 assert!(pinned_resume(&t));
4002 assert!(!may_hold(&mut t, "waiting for magi resume"));
4003 assert!(
4004 t.resume_override
4005 .as_ref()
4006 .unwrap()
4007 .conductor_rehold
4008 .is_none(),
4009 "a refused hold is not recorded as an override"
4010 );
4011 t.resume_override = None;
4012 assert!(!pinned_resume(&t));
4013 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
4014 }
4015
4016 #[test]
4017 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
4018 let dir = tempdir().unwrap();
4019 let queue = Queue::at(dir.path().join("queue"));
4020 let questions = Questions::at(dir.path().join("questions"));
4021 let mut t = task("resume me");
4022 t.resume_override = Some(crate::queue::OperatorResume {
4023 question_id: "q".into(),
4024 at: jiff::Timestamp::now(),
4025 conductor_rehold: None,
4026 forced: true,
4027 pinned_run: Some("run-1".into()),
4028 });
4029 queue.put(&mut t).unwrap();
4030
4031 apply(
4032 &queue,
4033 &questions,
4034 &Verdict {
4035 decisions: vec![
4036 Decision {
4037 id: t.id.clone(),
4038 question: Some("really?".to_owned()),
4039 ..Decision::default()
4040 },
4041 Decision {
4042 id: t.id.clone(),
4043 blocked_by: vec!["other".to_owned()],
4044 ..Decision::default()
4045 },
4046 ],
4047 },
4048 )
4049 .unwrap();
4050
4051 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
4052 assert!(questions.list().is_empty());
4053 }
4054}