1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{ChoiceAction, Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104 Recover,
106 Resume,
108 Repair,
110 Complete,
112 Inspect,
114 Configure,
116}
117
118#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123 Task,
125 Run {
127 id: String,
129 },
130 Setting {
133 path: String,
135 keys: Vec<String>,
137 },
138 Source {
140 reference: String,
142 },
143}
144
145#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148 pub operation: QuestionOperation,
150 pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155 fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156 let target = match &self.target {
157 QuestionTarget::Task => format!("task:{}", task.id),
158 QuestionTarget::Run { id } => {
159 anyhow::ensure!(
160 task.runs.contains(id),
161 "question target is not this task's run"
162 );
163 format!("run:{id}")
164 }
165 QuestionTarget::Setting { path, keys } => {
166 use std::path::Component;
167 anyhow::ensure!(
168 !path.is_empty()
169 && Path::new(path)
170 .components()
171 .all(|c| matches!(c, Component::Normal(_)))
172 && !keys.is_empty()
173 && keys.iter().all(|key| !key.is_empty()),
174 "question setting must name a repository file and concrete keys"
175 );
176 let root = task.repo.canonicalize()?;
177 let file = root.join(path).canonicalize()?;
178 let relative = file
179 .strip_prefix(&root)
180 .context("question setting is outside the repository")?;
181 let content = std::fs::read_to_string(&file)?;
182 let value: toml::Value = toml::from_str(&content)?;
183 let mut setting = &value;
184 for key in keys {
185 setting = setting
186 .get(key)
187 .context("question setting key does not exist")?;
188 }
189 format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190 }
191 QuestionTarget::Source { reference } => {
192 use crate::proc::Quiet as _;
193 let output = std::process::Command::new("git")
194 .args([
195 "rev-parse",
196 "--verify",
197 "--end-of-options",
198 &format!("{reference}^{{commit}}"),
199 ])
200 .current_dir(&task.repo)
201 .quiet()
202 .output()
203 .context("resolve question source")?;
204 anyhow::ensure!(
205 output.status.success(),
206 "question source cannot be resolved"
207 );
208 let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209 anyhow::ensure!(
210 !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211 "invalid question source commit"
212 );
213 format!("source:{commit}")
214 }
215 };
216 Ok((self.operation, target))
217 }
218}
219
220#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224 pub id: String,
226 #[serde(default)]
230 pub blocked_by: Vec<String>,
231 #[serde(default)]
236 pub missing_dependencies: Vec<String>,
237 #[serde(default)]
239 pub reason: Option<String>,
240 #[serde(default)]
247 pub recovery: Option<Recovery>,
248 #[serde(default)]
254 pub question: Option<String>,
255 #[serde(default)]
257 pub choices: Vec<String>,
258 #[serde(default)]
260 pub actions: BTreeMap<String, ChoiceAction>,
261 #[serde(default)]
264 pub question_decision: Option<QuestionDecision>,
265 #[serde(default)]
267 pub evidence: Vec<String>,
268}
269
270#[derive(Debug, Clone, Default, Deserialize)]
283pub struct Verdict {
284 pub decisions: Vec<Decision>,
287}
288
289fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
292 prompt::ConductTask {
293 id: t.id.clone(),
294 title: t.title.clone(),
295 instruction: t.instruction.clone(),
296 repo: t.repo.display().to_string(),
297 priority: t.priority,
298 status: t.status.as_str().to_owned(),
299 attempts: t.attempts,
300 max_attempts: t.effective_attempt_limit(max_attempts),
301 last_error: t.last_error.clone(),
302 hold_reason: t.hold_reason.clone(),
303 hold_source: t.hold_source.map(|source| source.label().to_owned()),
304 blocked_by: t.blocked_by.clone(),
305 answers: t
306 .answers
307 .iter()
308 .map(|a| prompt::ConductAnswer {
309 question: a.question.clone(),
310 answer: a.answer.clone(),
311 })
312 .collect(),
313 operator_resume: t.resumption.as_ref().map(|r| format!(
314 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
315 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
316 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
317 wording alone is insufficient.",
318 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
319 )).or_else(|| t.resume_override.as_ref().map(|o| {
320 format!(
321 "the operator explicitly answered \"resume\" at {}; do not hold this \
322 task again for the same reason unless there is new information",
323 o.at
324 )
325 })),
326 }
327}
328
329fn pinned_resume(task: &Task) -> bool {
333 task.resume_override
334 .as_ref()
335 .is_some_and(|o| o.pinned_run.is_some())
336}
337
338fn new_blocking_evidence(
341 queue: &Queue,
342 questions: &Questions,
343 task: &Task,
344 d: &Decision,
345) -> Vec<String> {
346 let Some(r) = &task.resumption else {
347 return Vec::new();
348 };
349 d.evidence
350 .iter()
351 .filter_map(|id| {
352 let version = if let Ok(q) = questions.get(id) {
353 (q.asked_at > r.at
354 && (q.run == task.id || task.runs.contains(&q.run))
355 && q.node != NODE
356 && q.status != crate::ask::QuestionStatus::Abandoned)
357 .then(|| {
358 format!(
359 "question:{id}:{}:{:?}:{}",
360 q.asked_at,
361 q.status,
362 q.resolution().unwrap_or_default()
363 )
364 })
365 } else if d.blocked_by.contains(id) {
366 queue
367 .get(id)
368 .ok()
369 .filter(|dep| {
370 dep.id != task.id
371 && dep.status != TaskStatus::Done
372 && (dep.created_at > r.at
373 || dep.held_at.is_some_and(|at| at > r.at)
374 || dep.operator_decided_at.is_some_and(|at| at > r.at)
375 || dep.passes.iter().any(|p| p.settled_at > r.at)
376 || (dep.status == TaskStatus::Blocked
377 && dep.blocked_at.is_some_and(|at| at > r.at)))
378 })
379 .map(|dep| {
380 format!(
381 "dependency:{id}:{:?}:{:?}:{:?}",
382 dep.status,
383 dep.chat_notice_key(),
384 dep.hold_notice_cause
385 )
386 })
387 } else if task.runs.iter().any(|run| run == id) {
388 RunState::load(id)
389 .ok()
390 .filter(|run| {
391 run.updated_at > r.at
394 && run
395 .passes
396 .last()
397 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
398 && matches!(
399 run.status,
400 crate::run::RunStatus::Failed
401 | crate::run::RunStatus::Blocked
402 | crate::run::RunStatus::Stalled
403 )
404 })
405 .map(|run| {
406 format!(
407 "run:{id}:{:?}:{:?}",
408 run.status,
409 run.passes.last().map(|p| p.index)
410 )
411 })
412 } else {
413 None
414 };
415 version.filter(|v| !r.blocking_evidence.contains(v))
416 })
417 .collect()
418}
419
420fn may_hold(task: &mut Task, reason: &str) -> bool {
424 if pinned_resume(task) {
425 return false;
426 }
427 if task.resumption.is_some() {
428 return true; }
430 let Some(o) = task.resume_override.as_mut() else {
431 return true;
432 };
433 if o.forced || o.pinned_run.is_some() {
434 tracing::warn!(
435 "conductor tried to hold task {} after the operator forced a resume: {reason}",
436 task.id
437 );
438 return false;
439 }
440 if o.conductor_rehold.is_some() {
441 return false;
442 }
443 o.conductor_rehold = Some(reason.to_owned());
444 true
445}
446
447fn severity_str(s: crate::verdict::Severity) -> &'static str {
450 match s {
451 crate::verdict::Severity::Nit => "nit",
452 crate::verdict::Severity::Minor => "minor",
453 crate::verdict::Severity::Major => "major",
454 crate::verdict::Severity::Blocker => "blocker",
455 }
456}
457
458fn surviving_branch(task: &Task) -> Option<String> {
463 let last = task.runs.last()?;
464 let state = RunState::load(last).ok()?;
465 if !crate::daemon::compatible_run_instruction(&state, task) {
466 return None;
467 }
468 state.winner().map(|c| c.branch.clone())
469}
470
471fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
498 let note = match &d.reason {
499 Some(reason) => reason.clone(),
500 None => match task.answers.last() {
501 Some(a) => format!(
502 "conduct held this again with no new reason given; last operator \
503 answer on record: {}",
504 a.answer
505 ),
506 None => "conduct held this again with no reason given".to_owned(),
507 },
508 };
509 match task.hold_reason.as_deref() {
510 Some(prior) if !prior.is_empty() => {
511 let outer = outermost_hold_note(prior);
512 if note.trim() == prior.trim() || note.trim() == outer {
513 None
514 } else {
515 Some(format!("{note}\n\n(previously: {outer})"))
516 }
517 }
518 _ => Some(note),
519 }
520}
521
522const PREVIOUSLY: &str = "\n\n(previously: ";
525
526fn outermost_hold_note(reason: &str) -> &str {
529 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
530}
531
532fn hold_note(d: &Decision) -> String {
534 d.reason
535 .clone()
536 .unwrap_or_else(|| "(no reason given)".to_owned())
537}
538
539async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
541 let Some(run_id) = task.runs.last().cloned() else {
542 return prompt::ConductOutcome {
543 run_id: "(none)".to_owned(),
544 unreadable: Some("this task has not produced a run yet".to_owned()),
545 run_status: None,
546 open_findings: Vec::new(),
547 rounds_used: 0,
548 rounds_max: 0,
549 rounds: Vec::new(),
550 branch: None,
551 branch_head: None,
552 references: None,
553 empty_candidate: false,
554 };
555 };
556 let state = match RunState::load(&run_id) {
557 Ok(s) => s,
558 Err(e) => {
559 tracing::warn!(
564 "conductor: could not read run {run_id} for task {}: {e:#}",
565 task.short()
566 );
567 return prompt::ConductOutcome {
568 run_id,
569 unreadable: Some(format!("{e:#}")),
570 run_status: None,
571 open_findings: Vec::new(),
572 rounds_used: 0,
573 rounds_max: 0,
574 rounds: Vec::new(),
575 branch: None,
576 branch_head: None,
577 references: None,
578 empty_candidate: false,
579 };
580 }
581 };
582
583 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
584 id: f.id.clone(),
585 title: f.title.clone(),
586 severity: severity_str(f.severity).to_owned(),
587 };
588 let open_findings = state
589 .open_findings()
590 .into_iter()
591 .map(finding_view)
592 .collect();
593 let rounds = state
594 .reviews
595 .iter()
596 .map(|r| prompt::ConductRound {
597 round: r.round,
598 findings: r
599 .reviews
600 .iter()
601 .flat_map(|rec| rec.findings.iter())
602 .map(finding_view)
603 .collect(),
604 addressed: r
605 .fix
606 .as_ref()
607 .map(|fx| fx.addressed.clone())
608 .unwrap_or_default(),
609 rejected: r
610 .fix
611 .as_ref()
612 .map(|fx| {
613 fx.rejected
614 .iter()
615 .map(|rej| prompt::ConductRejection {
616 id: rej.id.clone(),
617 why: rej.why.clone(),
618 })
619 .collect()
620 })
621 .unwrap_or_default(),
622 })
623 .collect();
624 let branch = state.winner().map(|c| c.branch.clone());
625 let branch_head = match &branch {
626 Some(b) => crate::git::rev_parse(repo, b)
627 .await
628 .ok()
629 .map(|h| h.chars().take(8).collect()),
630 None => None,
631 };
632
633 prompt::ConductOutcome {
634 run_id,
635 unreadable: None,
636 run_status: Some(state.status.as_str().to_owned()),
637 open_findings,
638 rounds_used: state.reviews.len(),
639 rounds_max: state.config.graph.review_rounds,
640 rounds,
641 branch,
642 branch_head,
643 references: crate::refs::describe(&state.seeds),
644 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
645 }
646}
647
648async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
652 let id = task.runs.last()?;
653 let state = match RunState::load(id) {
654 Ok(s) => s,
655 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
656 };
657 let branch = state.winner()?.branch.clone();
658 let base = state.base_branch.clone();
659 match crate::land::find_open_pr(repo, &branch, &base).await {
660 Ok(crate::land::OpenPr::None) => None,
661 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
662 "pull request {url} is already open for {branch} into {base}"
663 )),
664 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
665 "several pull requests are already open for {branch} into {base}: {}",
666 urls.join(" ")
667 )),
668 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
669 }
670}
671
672async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
679 for d in verdict
680 .decisions
681 .iter_mut()
682 .filter(|d| d.question.is_some())
683 {
684 let Ok(task) = queue.get(&d.id) else {
685 continue;
686 };
687 let text = format!("{}\n{}", task.title, task.instruction);
688 let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
689 if crate::refs::scan(&text).is_empty() {
690 if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
691 q.push_str(&format!(
692 "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
693 ));
694 }
695 continue;
696 }
697 let repo = repo_for(&task, repo);
698 let remote = &cfg.merge.remote;
699 let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
700 .await
701 .ok();
702 let facts = match base {
703 Some(base) => {
704 let base_name = base.clone();
705 let tracking = format!("{remote}/{base}");
706 let refreshed = crate::git::fetch(&repo, remote, &base)
709 .await
710 .is_ok_and(|o| o.ok());
711 let against = tracking;
714 match crate::git::rev_parse(&repo, &against).await {
715 Ok(tip) => crate::refs::describe(
716 &crate::refs::resolve(&repo, &tip, remote, &text).await,
717 )
718 .map(|facts| {
719 if refreshed {
720 facts
721 } else {
722 format!(
723 "{facts}\n(could not fetch {remote}/{base_name}: this is \
724 against the last fetched `{against}`, which may be behind \
725 the remote)"
726 )
727 }
728 }),
729 Err(_) => Some(format!(
730 "could NOT be checked against the repository: {against} is not \
731 available here, so whether the work is in the base is unknown"
732 )),
733 }
734 }
735 None => Some("could not check the repository: no base branch known".to_owned()),
736 };
737 let facts = match (facts, pr_fact) {
738 (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
739 (f, p) => f.or(p),
740 };
741 if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
742 q.push_str(&format!(
743 "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
744 ));
745 }
746 }
747}
748
749async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
751 prompt::ConductFinished {
752 task: view(t, max_attempts),
753 outcome: outcome_for(t, &repo_for(t, repo)).await,
754 }
755}
756
757fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
760 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
761 fallback.to_path_buf()
762 } else {
763 task.repo.clone()
764 }
765}
766
767fn apply_one(
784 queue: &Queue,
785 questions: &Questions,
786 d: &Decision,
787 snapshot: Option<&Task>,
788) -> Result<()> {
789 let _claim = queue
790 .claim(&d.id)
791 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
792 let mut task = queue.get(&d.id).context("no such task")?;
793
794 if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
797 return Ok(());
798 }
799 let blocking =
800 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
801 let evidence = new_blocking_evidence(queue, questions, &task, d);
802 if blocking && task.resumption.is_some() && evidence.is_empty() {
803 crate::notices::raise(crate::notices::conductor_resumption(
804 &task.id,
805 task.short(),
806 d.reason.as_deref(),
807 &d.blocked_by,
808 &d.evidence,
809 d.question.is_some(),
810 ));
811 return Ok(());
812 }
813
814 if blocking && !evidence.is_empty() {
815 let mut cause = evidence.clone();
816 cause.sort();
817 cause.dedup();
818 task.notice_cause(
819 &format!("conductor-evidence:{}", cause.join(",")),
820 None,
821 None,
822 );
823 }
824 if blocking {
825 if let Some(r) = task.resumption.as_mut() {
826 r.blocking_evidence.extend(evidence);
827 }
828 }
829
830 if task.operator_held() {
834 return Ok(());
835 }
836
837 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
841 return Ok(());
842 }
843
844 let pinned = pinned_resume(&task);
847 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
848 return Ok(());
849 }
850
851 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
855 && !d.missing_dependencies.is_empty()
856 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
857 && d.missing_dependencies
858 .iter()
859 .any(|id| !d.blocked_by.contains(id))
860 && d.missing_dependencies
861 .iter()
862 .chain(&d.blocked_by)
863 .any(|id| {
864 matches!(
865 crate::queue::dependency(queue, questions, id),
866 crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
867 )
868 })
869 {
870 return Ok(());
871 }
872
873 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
879 && !d.missing_dependencies.is_empty()
880 && d.missing_dependencies.iter().all(|id| {
881 let claim = crate::queue::dependency(queue, questions, id);
882 claim != crate::queue::Dependency::Missing
883 && d.blocked_by.iter().any(|blocker| {
884 blocker == id
885 || (matches!(
886 &claim,
887 crate::queue::Dependency::Task(_)
888 | crate::queue::Dependency::Question(_)
889 | crate::queue::Dependency::Deleted(_)
890 ) && crate::queue::dependency(queue, questions, blocker) == claim)
891 })
892 })
893 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
894 && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
895 {
896 let mut ids = d.blocked_by.clone();
897 crate::queue::normalize_blockers(queue, questions, &mut ids);
898 let reason = format!("Waiting on dependencies: {}", ids.join(", "));
899 task.block(ids, Some(reason));
900 return queue.put(&mut task);
901 }
902
903 if let Some(text) = &d.question {
904 if task.status == TaskStatus::Done {
905 return Ok(());
906 }
907 let mut held = task.clone();
908 held.status = TaskStatus::Held;
909 let targets: BTreeSet<_> = d
910 .blocked_by
911 .iter()
912 .filter_map(|id| {
913 if let Ok(dep) = queue.get(id) {
914 Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
915 } else if questions.get(id).is_ok() {
916 Some(format!("question:{id}"))
917 } else {
918 None
919 }
920 })
921 .collect();
922 let decision = d
923 .question_decision
924 .as_ref()
925 .and_then(|q| match q.identity(&task) {
926 Ok(identity) => Some(identity),
927 Err(error) => {
928 tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
931 None
932 }
933 });
934 let context_value =
935 serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
936 let context = serde_json::to_string(&context_value)?;
937 if questions.list().iter().any(|q| {
941 q.node == NODE
942 && q.run == task.id
943 && q.status == crate::ask::QuestionStatus::Answered
944 && q.decision_context.as_ref().is_some_and(|stored| {
945 if stored == &context {
946 return true;
947 }
948 decision.is_none()
951 && serde_json::from_str::<serde_json::Value>(stored)
952 .ok()
953 .and_then(|v| {
954 v.as_array().map(|items| {
955 items.len() == 4
956 && items[..3]
957 == context_value.as_array().expect("context")[..3]
958 })
959 })
960 == Some(true)
961 })
962 }) {
963 return Ok(());
964 }
965 let question_id = match questions
969 .list()
970 .into_iter()
971 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
972 {
973 Some(existing) => existing.id,
974 None => {
975 let mut q = Question::new(
976 task.id.clone(),
977 NODE.to_owned(),
978 SEAT.to_owned(),
979 text.clone(),
980 d.reason.clone().unwrap_or_default(),
981 d.choices.clone(),
982 );
983 q.actions = d.actions.clone();
984 anyhow::ensure!(
985 q.actions.keys().all(|label| q.choices.contains(label)),
986 "question action must name an offered choice"
987 );
988 q.decision_context = Some(context);
989 q.deputy = Some(Deputy::new(crate::deputy::brief(
994 &task.id,
995 d.reason.as_deref().unwrap_or_default(),
996 &q.choices,
997 &q.actions,
998 )));
999 if task.repo.is_dir() && task.repo != Path::new(".") {
1000 q.cwd = Some(task.repo.to_string_lossy().into_owned());
1001 }
1002 questions.put(&mut q)?;
1003 q.id
1004 }
1005 };
1006 task.block(vec![question_id], d.reason.clone());
1007 return queue.put(&mut task);
1008 }
1009
1010 match task.status {
1011 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
1015 if may_hold(&mut task, &hold_note(d)) {
1016 task.hold_machine(d.reason.clone());
1017 queue.put(&mut task)?;
1018 }
1019 }
1020 TaskStatus::Queued if !d.blocked_by.is_empty() => {
1021 let mut ids = d.blocked_by.clone();
1022 crate::queue::normalize_blockers(queue, questions, &mut ids);
1023 task.block(ids, d.reason.clone());
1024 queue.put(&mut task)?;
1025 }
1026 TaskStatus::Running => match d.recovery {
1027 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1028 Some(Recovery::Requeue) => {
1029 task.requeue();
1030 queue.put(&mut task)?;
1031 }
1032 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
1033 task.hold_machine(d.reason.clone());
1034 queue.put(&mut task)?;
1035 }
1036 _ => {}
1040 },
1041 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
1042 Some(Recovery::Requeue) if pinned_resume(&task) => {}
1045 Some(Recovery::Requeue) => {
1046 task.requeue();
1047 queue.put(&mut task)?;
1048 }
1049 Some(Recovery::Hold) => {
1050 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1054 if may_hold(&mut task, &hold_note(d)) {
1055 task.hold_machine(Some(reason));
1056 queue.put(&mut task)?;
1057 }
1058 }
1059 }
1060 Some(Recovery::Review) => {
1061 if let Some(branch) = surviving_branch(&task) {
1062 task.request_review(branch);
1063 queue.put(&mut task)?;
1064 }
1065 }
1071 Some(Recovery::Done) => {
1072 task.succeed();
1073 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1077 queue.put(&mut task)?;
1078 }
1079 None => {}
1080 },
1081 _ => {}
1084 }
1085 Ok(())
1086}
1087
1088pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1092 for d in &verdict.decisions {
1093 if let Err(e) = apply_one(queue, questions, d, None) {
1094 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1095 }
1096 }
1097 Ok(())
1098}
1099
1100pub fn seat_path(home: &Path) -> PathBuf {
1106 home.join("conduct").join("seat.json")
1107}
1108
1109pub fn load_seat(home: &Path) -> Option<SeatState> {
1111 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1112}
1113
1114fn busy_path(home: &Path) -> PathBuf {
1115 home.join("conduct").join("busy")
1116}
1117
1118pub fn busy(home: &Path) -> bool {
1124 std::fs::metadata(busy_path(home))
1125 .and_then(|m| m.modified())
1126 .ok()
1127 .and_then(|t| t.elapsed().ok())
1128 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1129}
1130
1131struct Busy(PathBuf);
1133
1134impl Busy {
1135 fn mark(home: &Path) -> Self {
1136 let path = busy_path(home);
1137 if let Some(dir) = path.parent() {
1138 let _ = std::fs::create_dir_all(dir);
1139 }
1140 let _ = std::fs::write(&path, std::process::id().to_string());
1141 Self(path)
1142 }
1143}
1144
1145impl Drop for Busy {
1146 fn drop(&mut self) {
1147 let _ = std::fs::remove_file(&self.0);
1148 }
1149}
1150
1151#[derive(Debug, Default)]
1154pub struct Conductor {
1155 seat: Option<SeatState>,
1156 last_seen: Option<(u64, BTreeSet<String>)>,
1157 considered: BTreeMap<String, String>,
1161}
1162
1163fn decision_input(task: &Task) -> serde_json::Value {
1168 let mut v = serde_json::to_value(task).unwrap_or_default();
1169 if let Some(o) = v.as_object_mut() {
1170 o.remove("chat_notice");
1171 o.remove("chat_notices");
1172 o.remove("updated_at");
1174 }
1175 v
1176}
1177
1178fn input_fingerprint(task: &Task) -> String {
1179 let mut v = decision_input(task);
1180 if let Some(o) = v.as_object_mut() {
1181 o.remove("hold_reason");
1182 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1183 ro.remove("conductor_rehold");
1184 }
1185 }
1186 v.to_string()
1187}
1188
1189impl Conductor {
1190 #[must_use]
1192 pub fn new() -> Self {
1193 Self::default()
1194 }
1195
1196 fn settled(
1201 considered: &BTreeMap<String, String>,
1202 stalled: &[Task],
1203 finished: &[Task],
1204 ) -> BTreeSet<String> {
1205 stalled
1206 .iter()
1207 .chain(finished)
1208 .filter(|t| t.status == TaskStatus::Held)
1209 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1210 .map(|t| t.id.clone())
1211 .collect()
1212 }
1213
1214 fn snapshot_with(
1215 considered: &BTreeMap<String, String>,
1216 queue: &Queue,
1217 stalled: &[Task],
1218 finished: &[Task],
1219 ) -> (u64, BTreeSet<String>) {
1220 let ids = stalled
1221 .iter()
1222 .chain(finished)
1223 .map(|t| t.id.clone())
1224 .collect();
1225 let skip = Self::settled(considered, stalled, finished);
1226 (queue.revision_excluding(&skip), ids)
1227 }
1228
1229 fn snapshot(
1230 &self,
1231 queue: &Queue,
1232 stalled: &[Task],
1233 finished: &[Task],
1234 ) -> (u64, BTreeSet<String>) {
1235 Self::snapshot_with(&self.considered, queue, stalled, finished)
1236 }
1237
1238 #[must_use]
1255 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1256 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1257 }
1258
1259 #[allow(clippy::too_many_arguments)]
1263 pub async fn maybe_run(
1264 &mut self,
1265 cfg: &Config,
1266 repo: &Path,
1267 queue: &Queue,
1268 questions: &Questions,
1269 home: &Path,
1270 queued: &[Task],
1271 stalled: &[Task],
1272 finished: &[Task],
1273 max_attempts: usize,
1274 ) {
1275 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1276 return;
1277 }
1278 self.considered = stalled
1281 .iter()
1282 .chain(finished)
1283 .filter(|t| t.status == TaskStatus::Held)
1284 .map(|t| (t.id.clone(), input_fingerprint(t)))
1285 .collect();
1286 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1287 if let Err(e) = self
1288 .run_once(
1289 cfg,
1290 repo,
1291 queue,
1292 questions,
1293 home,
1294 queued,
1295 stalled,
1296 finished,
1297 max_attempts,
1298 )
1299 .await
1300 {
1301 tracing::warn!("conductor: {e:#}");
1307 }
1308 }
1309
1310 #[allow(clippy::too_many_arguments)]
1311 async fn run_once(
1312 &mut self,
1313 cfg: &Config,
1314 repo: &Path,
1315 queue: &Queue,
1316 questions: &Questions,
1317 home: &Path,
1318 queued: &[Task],
1319 stalled: &[Task],
1320 finished: &[Task],
1321 max_attempts: usize,
1322 ) -> Result<()> {
1323 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1324 return Ok(());
1325 }
1326
1327 let primary = cfg
1328 .resolve_roles()
1329 .context("resolving the conductor seat")?
1330 .conductor;
1331 let chain = match cfg.roles.conductor.as_ref() {
1335 Some(c) if c.ids().len() > 1 => {
1336 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1337 }
1338 _ => vec![primary],
1339 };
1340
1341 let runnable_views: Vec<prompt::ConductTask> =
1342 queued.iter().map(|t| view(t, max_attempts)).collect();
1343 let stalled_views: Vec<prompt::ConductTask> =
1344 stalled.iter().map(|t| view(t, max_attempts)).collect();
1345 let mut finished_views = Vec::with_capacity(finished.len());
1346 for t in finished {
1347 finished_views.push(finished_view(t, repo, max_attempts).await);
1348 }
1349
1350 let body = prompt::with_overlay(
1351 prompt::conduct(
1352 &runnable_views,
1353 &stalled_views,
1354 &finished_views,
1355 &cfg.graph.language,
1356 ),
1357 cfg.prompts.overlay(NODE),
1358 );
1359
1360 let body = format!(
1361 "{body}\n\n{}",
1362 crate::queue::dependency_facts(queue, questions)
1363 );
1364 let artifacts = home.join("conduct").join("artifacts");
1365 let cache_dir = cfg.cache_dir();
1368
1369 let busy = Busy::mark(home);
1371 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1372 for spec in &chain {
1373 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1376 if needs_new_seat {
1377 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1378 }
1379 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1380 let stem = if std::ptr::eq(spec, &chain[0]) {
1383 format!("turn-{}", seat.turns + 1)
1384 } else {
1385 format!("turn-{}-{}", seat.turns + 1, spec.id)
1386 };
1387 let inv = Invocation {
1388 scope: crate::agent::InvocationScope::Repository,
1389 coordination: cfg.coordination.as_ref(),
1390 fetched_base: None,
1391 cwd: repo,
1392 prompt: &body,
1393 timeout: TURN_TIMEOUT,
1394 allow_write: false,
1397 unsandboxed: false,
1398 sessions: cfg.graph.sessions,
1399 artifacts: &artifacts,
1400 stem: &stem,
1401 run: NODE,
1402 node: NODE,
1403 cache_dir: cache_dir.as_deref(),
1404 attachments: &[],
1405 writable: &[],
1406 };
1407 let out = agent::invoke(spec, seat, &inv).await;
1408 if let Ok(body) = serde_json::to_string(&*seat) {
1411 let path = seat_path(home);
1412 if let Some(dir) = path.parent() {
1413 let _ = std::fs::create_dir_all(dir);
1414 }
1415 let _ = std::fs::write(path, body);
1416 }
1417 let advance = agent::chain_advances(&out);
1418 result = match out {
1419 Err(e) => Err(e.context("invoking the conductor")),
1420 Ok(out) if advance => Err(anyhow::anyhow!(
1421 "no usable reply ({})",
1422 crate::notices::turn_status(
1423 out.exit_code,
1424 out.timed_out,
1425 out.quota_exhausted()
1426 )
1427 )),
1428 Ok(out) => verdict::extract_json(&out.text)
1429 .context("the conductor's reply could not be parsed"),
1430 };
1431 if result.is_ok() {
1432 break;
1433 }
1434 if chain.len() > 1 {
1435 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1436 }
1437 }
1438 drop(busy);
1439 let mut verdict = result?;
1440 attach_facts(cfg, repo, queue, &mut verdict).await;
1441 for d in &verdict.decisions {
1442 let snapshot = queued
1443 .iter()
1444 .chain(stalled)
1445 .chain(finished)
1446 .find(|t| t.id == d.id);
1447 if let Some(snapshot) = snapshot {
1448 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1449 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1450 }
1451 }
1452 }
1453 Ok(())
1454 }
1455}
1456
1457#[cfg(test)]
1458mod tests {
1459 use std::collections::BTreeMap;
1460
1461 use tempfile::tempdir;
1462
1463 use super::*;
1464 use crate::ask::{Answer, QuestionStatus};
1465 use crate::config::{AgentKind, AgentSpec, Graph};
1466 use crate::queue::Source;
1467
1468 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1469 let path = dir.join("mock-conduct-agent.sh");
1470 std::fs::write(&path, script).expect("write mock");
1471 AgentSpec {
1472 id: "mock".to_owned(),
1473 kind: AgentKind::Command,
1474 model: None,
1475 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1476 extra_args: Vec::new(),
1477 env,
1478 prompt_delivery: None,
1479 }
1480 }
1481
1482 fn config(spec: AgentSpec) -> Config {
1483 Config {
1484 agents: vec![spec],
1485 graph: Graph {
1486 language: "en".to_owned(),
1487 ..Graph::default()
1488 },
1489 ..Config::default()
1490 }
1491 }
1492
1493 fn task(title: &str) -> Task {
1494 Task::new(
1495 title.to_owned(),
1496 format!("do {title}"),
1497 std::path::PathBuf::from("."),
1498 Source::Human,
1499 )
1500 }
1501
1502 #[test]
1503 fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1504 let mut t = task("waiting");
1505 t.hold_machine(Some("waiting for recovery".into()));
1506 let before = input_fingerprint(&t);
1507 let snapshot = decision_input(&t);
1508 t.chat_notice = Some(crate::queue::ChatNotice {
1509 key: t.chat_notice_key().unwrap(),
1510 at: jiff::Timestamp::now(),
1511 outcome: crate::queue::ChatNoticeOutcome::Posted,
1512 });
1513 t.chat_notices.push(t.chat_notice_key().unwrap());
1514 t.updated_at = jiff::Timestamp::now();
1515 assert_eq!(before, input_fingerprint(&t));
1516 assert_eq!(snapshot, decision_input(&t));
1517 t.hold_reason = Some("recovery remains under consideration".into());
1518 assert_eq!(before, input_fingerprint(&t));
1519 t.record_answer("a different question".into(), "a new decision".into());
1520 assert_ne!(before, input_fingerprint(&t));
1521 }
1522
1523 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1524 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1525
1526 fn env(reply: &str) -> BTreeMap<String, String> {
1527 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1528 }
1529
1530 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1531
1532 fn init_repo_with_branch(dir: &Path, branch: &str) {
1536 use crate::proc::Quiet as _;
1537 let run = |args: &[&str]| {
1538 let out = std::process::Command::new("git")
1539 .args(args)
1540 .current_dir(dir)
1541 .quiet()
1542 .output()
1543 .expect("spawn git");
1544 assert!(
1545 out.status.success(),
1546 "git {args:?} failed: {}",
1547 String::from_utf8_lossy(&out.stderr)
1548 );
1549 };
1550 run(&["init", "-b", "main"]);
1551 run(&["config", "user.name", "magi test"]);
1552 run(&["config", "user.email", "magi@example.com"]);
1553 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1554 run(&["add", "-A"]);
1555 run(&["commit", "-m", "init"]);
1556 run(&["checkout", "-b", branch]);
1557 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1558 run(&["add", "-A"]);
1559 run(&["commit", "-m", "candidate work"]);
1560 }
1561
1562 fn review_round_with_finding(
1563 round: usize,
1564 finding_id: &str,
1565 title: &str,
1566 addressed: &[&str],
1567 rejected: &[(&str, &str)],
1568 ) -> crate::run::ReviewRound {
1569 crate::run::ReviewRound {
1570 round,
1571 head: "deadbeef".to_owned(),
1572 verified_head: None,
1573 verified_at: None,
1574 reviews: vec![crate::run::ReviewRecord {
1575 attempts: 0,
1576 reviewer: 1,
1577 agent: "mock".to_owned(),
1578 summary: String::new(),
1579 findings: vec![crate::verdict::Finding {
1580 id: finding_id.to_owned(),
1581 severity: crate::verdict::Severity::Major,
1582 file: None,
1583 line: None,
1584 title: title.to_owned(),
1585 detail: String::new(),
1586 }],
1587 vote: None,
1588 failed: None,
1589 duration_ms: 0,
1590 }],
1591 e2e: Vec::new(),
1592 verify_retried: false,
1593 e2e_deferred: false,
1594 e2e_defer_reason: None,
1595 fix: Some(crate::run::FixRecord {
1596 agent: "mock".to_owned(),
1597 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1598 rejected: rejected
1599 .iter()
1600 .map(|(id, why)| crate::verdict::Rejection {
1601 id: (*id).to_owned(),
1602 why: (*why).to_owned(),
1603 })
1604 .collect(),
1605 notes: String::new(),
1606 committed: false,
1607 failed: None,
1608 duration_ms: 0,
1609 continuation: None,
1610 }),
1611 blocking: 1,
1612 answered: 1,
1613 expected: 1,
1614 clean: false,
1615 progressed: true,
1616 vote_split: false,
1617 reconsideration: Vec::new(),
1618 verdict: None,
1619 }
1620 }
1621
1622 #[test]
1623 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1624 crate::run::pin_test_home();
1625 let dir = tempdir().unwrap();
1626 let queue = Queue::at(dir.path().join("queue"));
1627 let questions = Questions::at(dir.path().join("questions"));
1628 let mut t = task("recover");
1629 t.answers.push(crate::queue::AnsweredQuestion {
1630 question: "Inspect remote first?".into(),
1631 answer: "Inspect, then await resumption instructions".into(),
1632 });
1633 t.hold_machine(Some("waiting for remote inspection".into()));
1634 queue.put(&mut t).unwrap();
1635 let snapshot = queue.get(&t.id).unwrap();
1636 t.edit(
1637 "recover".into(),
1638 "Reuse the measured PR head; fix Clippy; review".into(),
1639 )
1640 .unwrap();
1641 t.release_operator("cli", None);
1642 queue.put(&mut t).unwrap();
1643 let approval = t.resumption.clone();
1644 let d = Decision {
1645 id: t.id.clone(),
1646 recovery: Some(Recovery::Hold),
1647 reason: Some("remote inspection still required".into()),
1648 ..Decision::default()
1649 };
1650 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1651 let stale_question = Decision {
1652 question: Some("Inspect remote?".into()),
1653 ..d.clone()
1654 };
1655 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1656 assert!(questions.list().is_empty());
1657 for reason in [
1658 "waiting for remote inspection",
1659 "inspection has not been confirmed",
1660 ] {
1661 let repeated = Decision {
1662 reason: Some(reason.into()),
1663 question: Some("How should we proceed?".into()),
1664 ..d.clone()
1665 };
1666 apply(
1667 &queue,
1668 &questions,
1669 &Verdict {
1670 decisions: vec![repeated],
1671 },
1672 )
1673 .unwrap();
1674 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1675 }
1676 assert!(
1677 questions.list().is_empty(),
1678 "a rejected proposal registers no question"
1679 );
1680 let notice = crate::notices::Notices::open()
1681 .get(&crate::notices::id_of(&format!(
1682 "conductor-resumption:{}",
1683 t.id
1684 )))
1685 .unwrap();
1686 assert!(!notice.message.contains("How should we proceed?"));
1687 assert!(!notice.message.contains("Some("));
1688 assert!(notice.message.contains("No reply to this notice is needed"));
1689 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1690 assert_eq!(reloaded.resumption, approval);
1691 assert_eq!(reloaded.answers, t.answers);
1692 let v = view(&reloaded, 3);
1693 assert!(v.operator_resume.unwrap().contains("superseded"));
1694 let mut dep = task("new prerequisite");
1695 dep.hold_machine(Some("new authentication failure".into()));
1696 queue.put(&mut dep).unwrap();
1697 let new_blocker = Decision {
1698 blocked_by: vec![dep.id.clone()],
1699 evidence: vec![dep.id.clone()],
1700 recovery: None,
1702 reason: Some("new prerequisite failed".into()),
1703 ..d.clone()
1704 };
1705 apply(
1706 &queue,
1707 &questions,
1708 &Verdict {
1709 decisions: vec![new_blocker.clone()],
1710 },
1711 )
1712 .unwrap();
1713 let mut after = queue.get(&t.id).unwrap();
1714 assert_eq!(after.status, TaskStatus::Blocked);
1715 after.release();
1718 queue.put(&mut after).unwrap();
1719 apply(
1720 &queue,
1721 &questions,
1722 &Verdict {
1723 decisions: vec![new_blocker],
1724 },
1725 )
1726 .unwrap();
1727 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1728 after.start("new-run".into());
1729 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1730 after.hold_manual(Some("operator says wait".into()));
1731 queue.put(&mut after).unwrap();
1732 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1733 assert!(queue.get(&t.id).unwrap().operator_held());
1734 }
1735
1736 #[test]
1737 fn newly_blocked_dependency_is_material_evidence_after_approval() {
1738 let dir = tempdir().unwrap();
1739 let queue = Queue::at(dir.path().join("queue"));
1740 let questions = Questions::at(dir.path().join("questions"));
1741 let mut dep = task("old queued dependency");
1742 queue.put(&mut dep).unwrap();
1743 let mut t = task("approved task");
1744 t.release_operator("test", None);
1745 queue.put(&mut t).unwrap();
1746 let mut q = Question::new(
1747 dep.id.clone(),
1748 "check".into(),
1749 "check".into(),
1750 "New dependency approval".into(),
1751 String::new(),
1752 Vec::new(),
1753 );
1754 questions.put(&mut q).unwrap();
1755 dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1756 let changed = dep.blocked_at;
1757 queue.put(&mut dep).unwrap();
1758 dep.block(vec![q.id], Some("same decision paraphrased".into()));
1759 assert_eq!(
1760 dep.blocked_at, changed,
1761 "prose is not new blocking evidence"
1762 );
1763 queue.put(&mut dep).unwrap();
1764 let d = Decision {
1765 id: t.id.clone(),
1766 blocked_by: vec![dep.id.clone()],
1767 evidence: vec![dep.id],
1768 ..Decision::default()
1769 };
1770 assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1771 apply_one(&queue, &questions, &d, None).unwrap();
1772 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1773 }
1774
1775 #[test]
1776 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1777 crate::run::pin_test_home();
1778 let dir = tempdir().unwrap();
1779 let queue = Queue::at(dir.path().join("queue"));
1780 let questions = Questions::at(dir.path().join("questions"));
1781 let mut t = task("resume evidence");
1782 let mut run = RunState::new(
1783 t.repo.clone(),
1784 "main".into(),
1785 "base".into(),
1786 "work".into(),
1787 crate::config::Config::default(),
1788 );
1789 run.begin_pass();
1790 run.status = crate::run::RunStatus::Failed;
1791 run.save().unwrap();
1792 t.start(run.id.clone());
1793 t.runs.push(run.id.clone());
1795 t.release_operator("cli", None);
1796 let approval = t.resumption.as_ref().unwrap().at;
1797 let decision = Decision {
1798 evidence: vec![run.id.clone()],
1799 ..Decision::default()
1800 };
1801 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1802 t.start(run.id.clone());
1803 assert_eq!(t.runs.len(), 1);
1804 run.begin_pass();
1805 run.save().unwrap();
1806 assert!(run.passes.last().unwrap().started_at > approval);
1807 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1808 assert_eq!(evidence.len(), 1);
1809 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1810 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1811 run.begin_pass();
1812 run.save().unwrap();
1813 assert_eq!(
1814 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1815 1
1816 );
1817
1818 let mut q = Question::new(
1819 run.id.clone(),
1820 "review".into(),
1821 "check".into(),
1822 "New failure".into(),
1823 String::new(),
1824 Vec::new(),
1825 );
1826 questions.put(&mut q).unwrap();
1827 let decision = Decision {
1828 evidence: vec![q.id.clone()],
1829 ..Decision::default()
1830 };
1831 assert_eq!(
1832 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1833 1
1834 );
1835 q.asked_at = approval;
1836 questions.put(&mut q).unwrap();
1837 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1838 q.asked_at = jiff::Timestamp::now();
1839 q.run = "unrelated".into();
1840 questions.put(&mut q).unwrap();
1841 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1842
1843 let mut fresh = RunState::new(
1844 t.repo.clone(),
1845 "main".into(),
1846 "base".into(),
1847 "fresh work".into(),
1848 crate::config::Config::default(),
1849 );
1850 fresh.status = crate::run::RunStatus::Failed;
1851 fresh.save().unwrap();
1852 t.start(fresh.id.clone());
1853 let decision = Decision {
1854 evidence: vec![fresh.id.clone()],
1855 ..Decision::default()
1856 };
1857 assert_eq!(
1858 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1859 1
1860 );
1861 }
1862
1863 #[test]
1864 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1865 crate::run::pin_test_home();
1866 let dir = tempdir().unwrap();
1867 let default_repo = dir.path().join("default");
1868 let task_repo = dir.path().join("task");
1869 std::fs::create_dir_all(&default_repo).unwrap();
1870 std::fs::create_dir_all(&task_repo).unwrap();
1871 init_repo_with_branch(&default_repo, "other-branch");
1872 init_repo_with_branch(&task_repo, "magi/f00d/A");
1873
1874 let mut config = Config::default();
1875 config.graph.review_rounds = 6;
1876 let mut state = crate::run::RunState::new(
1877 task_repo.clone(),
1878 "main".to_owned(),
1879 "deadbeef".to_owned(),
1880 "task".to_owned(),
1881 config,
1882 );
1883 state.status = crate::run::RunStatus::Blocked;
1884 state.candidates.push(crate::run::Candidate {
1885 index: 0,
1886 label: 'A',
1887 agent: "mock".to_owned(),
1888 branch: "magi/f00d/A".to_owned(),
1889 worktree: task_repo.clone(),
1890 summary: String::new(),
1891 stat: String::new(),
1892 files: 1,
1893 commits: 1,
1894 empty: false,
1895 failed: None,
1896 verified_noop: None,
1897 duration_ms: 0,
1898 folded: false,
1899 });
1900 state.tally = Some(crate::run::Tally {
1901 first_choice: std::collections::BTreeMap::new(),
1902 borda: std::collections::BTreeMap::new(),
1903 winner: 'A',
1904 rankings: 0,
1905 unanimous_initial: false,
1906 deliberated: false,
1907 changed_votes: 0,
1908 unanimous_final: false,
1909 tie_break: None,
1910 judges: 0,
1911 present: 0,
1912 quorum: 0,
1913 met_quorum: true,
1914 uncontested: Some("solo".to_owned()),
1915 });
1916 state.reviews = vec![
1917 review_round_with_finding(
1918 1,
1919 "R1-1-2",
1920 "answer content is dropped",
1921 &[],
1922 &[("R1-1-2", "the id leaving blocked_by is enough")],
1923 ),
1924 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1925 ];
1926 state.save().unwrap();
1927
1928 let mut t = task("outcome test");
1929 t.repo = task_repo;
1930 t.runs.push(state.id.clone());
1931
1932 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1933 let outcome = finished.outcome;
1934
1935 assert!(outcome.unreadable.is_none());
1936 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1937 assert_eq!(outcome.rounds_used, 2);
1938 assert_eq!(outcome.rounds_max, 6);
1939 assert_eq!(outcome.rounds.len(), 2);
1940 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1941 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1942 assert!(outcome.rounds[1].addressed.is_empty());
1943 assert!(outcome.rounds[1].rejected.is_empty());
1944 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1945 assert!(
1946 outcome.branch_head.is_some(),
1947 "a real branch must resolve a head commit: {outcome:?}"
1948 );
1949 }
1950
1951 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1955 tokio::runtime::Builder::new_current_thread()
1956 .enable_all()
1957 .build()
1958 .unwrap()
1959 .block_on(f)
1960 }
1961
1962 #[test]
1963 fn view_resolves_task_attempt_limit_before_daemon_default() {
1964 let mut t = task("budget");
1965 assert_eq!(view(&t, 7).max_attempts, 7);
1966 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1967 .unwrap();
1968 assert_eq!(view(&t, 7).max_attempts, 3);
1969 t.change_attempt_limit(None).unwrap();
1970 assert_eq!(view(&t, 7).max_attempts, 7);
1971 }
1972
1973 #[test]
1974 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1975 let mut t = task("answered");
1976 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1977 let v = view(&t, 2);
1978 assert_eq!(v.answers.len(), 1);
1979 assert_eq!(v.answers[0].question, "Which backend?");
1980 assert_eq!(v.answers[0].answer, "SQLite");
1981 }
1982
1983 #[tokio::test]
1984 async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1985 for held in [false, true] {
1986 for reference in ["ca5b", "20261013-193611-ca5b"] {
1987 for question in [false, true] {
1988 let dir = tempdir().unwrap();
1989 let queue = Queue::at(dir.path().join("queue"));
1990 let questions = Questions::at(dir.path().join("questions"));
1991 let mut dep = task("prerequisite");
1992 dep.id = "20261013-193611-ca5b".into();
1993 dep.start("existing-run".into());
1994 queue.put(&mut dep).unwrap();
1995 let mut waiting = task("waiting");
1996 if held {
1997 waiting.hold_machine(Some("dependency could not be located".into()));
1998 }
1999 queue.put(&mut waiting).unwrap();
2000 let missing_reference = if reference == "ca5b" {
2001 dep.id.as_str()
2002 } else {
2003 "ca5b"
2004 };
2005 let reply = serde_json::json!({"decisions": [{
2006 "id": waiting.id, "blocked_by": [reference],
2007 "missing_dependencies": [missing_reference],
2008 "question": question.then_some("ca5b is missing; where is its replacement?"),
2009 "recovery": (!question).then_some("hold"),
2010 "reason": "ca5b is missing"
2011 }]}).to_string();
2012 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2013 let (queued, finished) = if held {
2014 (vec![], vec![waiting.clone()])
2015 } else {
2016 (vec![waiting.clone()], vec![])
2017 };
2018 Conductor::new()
2019 .run_once(
2020 &cfg,
2021 dir.path(),
2022 &queue,
2023 &questions,
2024 dir.path(),
2025 &queued,
2026 &[],
2027 &finished,
2028 2,
2029 )
2030 .await
2031 .unwrap();
2032 let after = queue.get(&waiting.id).unwrap();
2033 assert_eq!(after.status, TaskStatus::Blocked);
2034 assert_eq!(after.blocked_by, [dep.id]);
2035 assert_eq!(after.blocked_from, Some(waiting.status));
2036 assert!(!after.block_reason.unwrap().contains("missing"));
2037 assert!(questions.list().is_empty());
2038 assert_eq!(after.instruction, waiting.instruction);
2039 }
2040 }
2041 }
2042 }
2043
2044 #[tokio::test]
2045 async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2046 for held in [false, true] {
2047 for unknown in [false, true] {
2048 for question in [false, true] {
2049 let dir = tempdir().unwrap();
2050 let queue = Queue::at(dir.path().join("queue"));
2051 let questions = Questions::at(dir.path().join("questions"));
2052 let mut dep = task("running prerequisite");
2053 dep.id = "20261013-193611-ca5b".into();
2054 dep.start("existing-run".into());
2055 queue.put(&mut dep).unwrap();
2056 let reference = if unknown {
2057 std::fs::write(questions.root(), "not a directory").unwrap();
2060 dep.id.clone()
2061 } else {
2062 let mut other = task("another prerequisite");
2063 other.id = "20261014-193611-ca5b".into();
2064 queue.put(&mut other).unwrap();
2065 "ca5b".to_owned()
2066 };
2067 assert_eq!(
2068 crate::queue::dependency(&queue, &questions, &reference),
2069 if unknown {
2070 crate::queue::Dependency::Unknown
2071 } else {
2072 crate::queue::Dependency::Ambiguous
2073 }
2074 );
2075 let mut waiting = task("waiting");
2076 if held {
2077 waiting.hold_machine(Some("dependency could not be located".into()));
2078 }
2079 queue.put(&mut waiting).unwrap();
2080 let reply = serde_json::json!({"decisions": [{
2081 "id": waiting.id, "blocked_by": [reference],
2082 "missing_dependencies": [reference],
2083 "question": question.then_some("ca5b is missing"),
2084 "recovery": (!question).then_some("hold"),
2085 "reason": "ca5b is missing"
2086 }]})
2087 .to_string();
2088 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2089 let (queued, finished) = if held {
2090 (vec![], vec![waiting.clone()])
2091 } else {
2092 (vec![waiting.clone()], vec![])
2093 };
2094 Conductor::new()
2095 .run_once(
2096 &cfg,
2097 dir.path(),
2098 &queue,
2099 &questions,
2100 dir.path(),
2101 &queued,
2102 &[],
2103 &finished,
2104 2,
2105 )
2106 .await
2107 .unwrap();
2108 let after = queue.get(&waiting.id).unwrap();
2109 assert_eq!(after.status, TaskStatus::Blocked);
2110 assert_eq!(after.blocked_by, [reference]);
2111 assert_eq!(after.blocked_from, Some(waiting.status));
2112 assert!(!after.block_reason.unwrap().contains("missing"));
2113 assert_eq!(after.instruction, waiting.instruction);
2114 if unknown {
2115 assert!(questions.root().is_file());
2116 } else {
2117 assert!(questions.list().is_empty());
2118 }
2119 }
2120 }
2121 }
2122 }
2123
2124 #[tokio::test]
2125 async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2126 for held in [false, true] {
2127 for unknown in [false, true] {
2128 for question in [false, true] {
2129 for short_blocker in [false, true] {
2130 let dir = tempdir().unwrap();
2131 let queue = Queue::at(dir.path().join("queue"));
2132 let questions = Questions::at(dir.path().join("questions"));
2133 let mut dep = task("running prerequisite");
2134 dep.id = "20261013-193611-ca5b".into();
2135 dep.start("existing-run".into());
2136 queue.put(&mut dep).unwrap();
2137 if unknown {
2138 std::fs::write(questions.root(), "not a directory").unwrap();
2139 } else {
2140 let mut other = task("another prerequisite");
2141 other.id = "20261014-193611-ca5b".into();
2142 queue.put(&mut other).unwrap();
2143 }
2144 let (blocker, missing) = if short_blocker {
2145 ("ca5b", dep.id.as_str())
2146 } else {
2147 (dep.id.as_str(), "ca5b")
2148 };
2149 let mut waiting = task("waiting");
2150 if held {
2151 waiting.hold_machine(Some("dependency could not be located".into()));
2152 }
2153 queue.put(&mut waiting).unwrap();
2154 let reply = serde_json::json!({"decisions": [{
2155 "id": waiting.id, "blocked_by": [blocker],
2156 "missing_dependencies": [missing],
2157 "question": question.then_some("ca5b is missing"),
2158 "recovery": (!question).then_some("hold"),
2159 "reason": "ca5b is missing"
2160 }]})
2161 .to_string();
2162 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2163 let (queued, finished) = if held {
2164 (vec![], vec![waiting.clone()])
2165 } else {
2166 (vec![waiting.clone()], vec![])
2167 };
2168 Conductor::new()
2169 .run_once(
2170 &cfg,
2171 dir.path(),
2172 &queue,
2173 &questions,
2174 dir.path(),
2175 &queued,
2176 &[],
2177 &finished,
2178 2,
2179 )
2180 .await
2181 .unwrap();
2182 assert_eq!(
2183 serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2184 serde_json::to_value(&waiting).unwrap(),
2185 "unverifiable aliases must leave the entire task unchanged"
2186 );
2187 if unknown {
2188 assert!(questions.root().is_file());
2189 } else {
2190 assert!(questions.list().is_empty());
2191 }
2192 }
2193 }
2194 }
2195 }
2196 }
2197
2198 #[tokio::test]
2199 async fn dependency_resolution_genuine_missing_reference_still_asks() {
2200 let dir = tempdir().unwrap();
2201 let queue = Queue::at(dir.path().join("queue"));
2202 let questions = Questions::at(dir.path().join("questions"));
2203 let mut waiting = task("missing prerequisite");
2204 queue.put(&mut waiting).unwrap();
2205 let reply = serde_json::json!({"decisions": [{
2206 "id": waiting.id, "blocked_by": ["not-present"],
2207 "missing_dependencies": ["not-present"],
2208 "question": "Which prerequisite replaces the missing task?"
2209 }]})
2210 .to_string();
2211 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2212 Conductor::new()
2213 .run_once(
2214 &cfg,
2215 dir.path(),
2216 &queue,
2217 &questions,
2218 dir.path(),
2219 &[waiting.clone()],
2220 &[],
2221 &[],
2222 2,
2223 )
2224 .await
2225 .unwrap();
2226 let after = queue.get(&waiting.id).unwrap();
2227 assert_eq!(after.status, TaskStatus::Blocked);
2228 assert_eq!(
2229 questions.get(&after.blocked_by[0]).unwrap().summary,
2230 "Which prerequisite replaces the missing task?"
2231 );
2232 }
2233
2234 #[tokio::test]
2235 async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2236 let dir = tempdir().unwrap();
2237 let queue = Queue::at(dir.path().join("queue"));
2238 let questions = Questions::at(dir.path().join("questions"));
2239 let mut dep = task("prerequisite");
2240 dep.id = "20261013-193611-ca5b".into();
2241 dep.start("existing-run".into());
2242 queue.put(&mut dep).unwrap();
2243 let mut unrelated = task("unrelated work");
2244 unrelated.start("other-run".into());
2245 queue.put(&mut unrelated).unwrap();
2246 std::fs::File::options()
2249 .write(true)
2250 .open(queue.path_of(&unrelated.id))
2251 .unwrap()
2252 .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2253 .unwrap();
2254 unrelated.succeed();
2255 let mut waiting = task("waiting");
2256 let mut asking = task("needs a product decision");
2257 let mut holding = task("must stay held");
2258 queue.put(&mut waiting).unwrap();
2259 queue.put(&mut asking).unwrap();
2260 queue.put(&mut holding).unwrap();
2261 let reply = serde_json::json!({"decisions": [
2262 {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2263 {"id": asking.id, "question": "Which behavior do you want?"},
2264 {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2265 ]})
2266 .to_string();
2267 let mut vars = env(&reply);
2268 vars.insert(
2269 "UNRELATED_PATH".into(),
2270 queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2271 );
2272 vars.insert(
2273 "UNRELATED_RECORD".into(),
2274 serde_json::to_string(&unrelated).unwrap(),
2275 );
2276 let script = r#"#!/bin/sh
2277cat >/dev/null
2278printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2279printf '%s\n' "$MOCK_REPLY"
2280"#;
2281 let cfg = config(mock_agent(dir.path(), script, vars));
2282 Conductor::new()
2283 .run_once(
2284 &cfg,
2285 dir.path(),
2286 &queue,
2287 &questions,
2288 dir.path(),
2289 &[waiting.clone(), asking.clone(), holding.clone()],
2290 &[],
2291 &[],
2292 2,
2293 )
2294 .await
2295 .unwrap();
2296 assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2297 let after = queue.get(&waiting.id).unwrap();
2298 assert_eq!(after.status, TaskStatus::Blocked);
2299 assert_eq!(after.blocked_by, [dep.id]);
2300 let asked = queue.get(&asking.id).unwrap();
2301 assert_eq!(asked.status, TaskStatus::Blocked);
2302 assert_eq!(
2303 questions.get(&asked.blocked_by[0]).unwrap().summary,
2304 "Which behavior do you want?"
2305 );
2306 assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2307 }
2308
2309 #[test]
2310 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2311 let dir = tempdir().unwrap();
2312 let queue = Queue::at(dir.path().join("queue"));
2313 let questions = Questions::at(dir.path().join("questions"));
2314 let mut a = task("a");
2315 a.priority = 9;
2316 queue.put(&mut a).unwrap();
2317
2318 let verdict = Verdict {
2319 decisions: vec![Decision {
2320 id: a.id.clone(),
2321 blocked_by: vec!["20260101-000000-dead".to_owned()],
2322 reason: Some("waits on the other task".to_owned()),
2323 recovery: None,
2324 question: None,
2325 choices: Vec::new(),
2326 actions: BTreeMap::new(),
2327 question_decision: None,
2328 evidence: Vec::new(),
2329 missing_dependencies: Vec::new(),
2330 }],
2331 };
2332 apply(&queue, &questions, &verdict).unwrap();
2333
2334 let back = queue.get(&a.id).unwrap();
2335 assert_eq!(back.status, TaskStatus::Blocked);
2336 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2337 assert_eq!(
2338 back.priority, 9,
2339 "the conductor's reply cannot carry priority"
2340 );
2341 }
2342
2343 #[test]
2344 fn a_question_decision_files_one_and_blocks_on_its_id() {
2345 let dir = tempdir().unwrap();
2346 let queue = Queue::at(dir.path().join("queue"));
2347 let questions = Questions::at(dir.path().join("questions"));
2348 let mut t = task("ambiguous");
2349 queue.put(&mut t).unwrap();
2350
2351 let verdict = Verdict {
2352 decisions: vec![Decision {
2353 id: t.id.clone(),
2354 blocked_by: Vec::new(),
2355 reason: Some("which backend?".to_owned()),
2356 recovery: None,
2357 question: Some("Which storage backend?".to_owned()),
2358 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2359 actions: BTreeMap::new(),
2360 question_decision: None,
2361 evidence: Vec::new(),
2362 missing_dependencies: Vec::new(),
2363 }],
2364 };
2365 apply(&queue, &questions, &verdict).unwrap();
2366
2367 let back = queue.get(&t.id).unwrap();
2368 assert_eq!(back.status, TaskStatus::Blocked);
2369 assert_eq!(back.blocked_by.len(), 1);
2370 let q = questions.get(&back.blocked_by[0]).unwrap();
2371 assert_eq!(q.summary, "Which storage backend?");
2372 assert_eq!(q.node, NODE);
2373 assert!(q.status.open());
2374 }
2375
2376 #[test]
2377 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2378 let dir = tempdir().unwrap();
2379 let queue = Queue::at(dir.path().join("queue"));
2380 let questions = Questions::at(dir.path().join("questions"));
2381 let mut t = task("asked once");
2382 queue.put(&mut t).unwrap();
2383
2384 let decision = Decision {
2385 id: t.id.clone(),
2386 reason: Some("still deciding".to_owned()),
2387 question: Some("Which backend?".to_owned()),
2388 ..Decision::default()
2389 };
2390 apply(
2391 &queue,
2392 &questions,
2393 &Verdict {
2394 decisions: vec![decision.clone()],
2395 },
2396 )
2397 .unwrap();
2398 assert_eq!(questions.list().len(), 1);
2399 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2400
2401 let mut released = queue.get(&t.id).unwrap();
2406 released.release();
2407 queue.put(&mut released).unwrap();
2408
2409 apply(
2410 &queue,
2411 &questions,
2412 &Verdict {
2413 decisions: vec![decision],
2414 },
2415 )
2416 .unwrap();
2417 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2418 let after = queue.get(&t.id).unwrap();
2419 assert_eq!(
2420 after.blocked_by,
2421 [first_question_id],
2422 "the existing open question is reused, not replaced"
2423 );
2424 }
2425
2426 #[test]
2427 fn a_same_id_question_from_another_node_is_not_reused() {
2428 let dir = tempdir().unwrap();
2429 let queue = Queue::at(dir.path().join("queue"));
2430 let questions = Questions::at(dir.path().join("questions"));
2431 let mut t = task("must ask the conductor");
2432 queue.put(&mut t).unwrap();
2433
2434 let mut unrelated = Question::new(
2435 t.id.clone(),
2436 "review".to_owned(),
2437 "reviewer-1".to_owned(),
2438 "An unrelated review question".to_owned(),
2439 String::new(),
2440 Vec::new(),
2441 );
2442 questions.put(&mut unrelated).unwrap();
2443
2444 apply(
2445 &queue,
2446 &questions,
2447 &Verdict {
2448 decisions: vec![Decision {
2449 id: t.id.clone(),
2450 question: Some("Which backend?".to_owned()),
2451 ..Decision::default()
2452 }],
2453 },
2454 )
2455 .unwrap();
2456
2457 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2458 assert_eq!(blocked_by.len(), 1);
2459 assert_ne!(blocked_by[0], unrelated.id);
2460 assert!(questions.get(&unrelated.id).unwrap().status.open());
2461 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2462 }
2463
2464 #[test]
2465 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2466 let dir = tempdir().unwrap();
2467 let queue = Queue::at(dir.path().join("queue"));
2468 let questions = Questions::at(dir.path().join("questions"));
2469 let mut t = task("waits on an answer");
2470 queue.put(&mut t).unwrap();
2471
2472 apply(
2473 &queue,
2474 &questions,
2475 &Verdict {
2476 decisions: vec![Decision {
2477 id: t.id.clone(),
2478 blocked_by: Vec::new(),
2479 reason: None,
2480 recovery: None,
2481 question: Some("Which backend?".to_owned()),
2482 choices: Vec::new(),
2483 actions: BTreeMap::new(),
2484 question_decision: None,
2485 evidence: Vec::new(),
2486 missing_dependencies: Vec::new(),
2487 }],
2488 },
2489 )
2490 .unwrap();
2491 let blocked = queue.get(&t.id).unwrap();
2492 let question_id = blocked.blocked_by[0].clone();
2493
2494 let mut q = questions.get(&question_id).unwrap();
2495 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2496 questions.put(&mut q).unwrap();
2497 assert_eq!(q.status, QuestionStatus::Answered);
2498
2499 let mut task_after = queue.get(&t.id).unwrap();
2503 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2504 task_after.unblock(&question_id);
2505 assert_eq!(task_after.status, TaskStatus::Queued);
2506 assert_eq!(task_after.answers[0].answer, "SQLite");
2507 }
2508
2509 #[test]
2510 fn a_stalled_task_can_be_requeued_or_held() {
2511 let dir = tempdir().unwrap();
2512 let queue = Queue::at(dir.path().join("queue"));
2513 let questions = Questions::at(dir.path().join("questions"));
2514
2515 let mut requeue_me = task("stuck a");
2516 requeue_me.start("run-1".to_owned());
2517 queue.put(&mut requeue_me).unwrap();
2518
2519 let mut hold_me = task("stuck b");
2520 hold_me.start("run-2".to_owned());
2521 queue.put(&mut hold_me).unwrap();
2522
2523 apply(
2524 &queue,
2525 &questions,
2526 &Verdict {
2527 decisions: vec![
2528 Decision {
2529 id: requeue_me.id.clone(),
2530 recovery: Some(Recovery::Requeue),
2531 ..Decision::default()
2532 },
2533 Decision {
2534 id: hold_me.id.clone(),
2535 recovery: Some(Recovery::Hold),
2536 reason: Some("looks broken".to_owned()),
2537 ..Decision::default()
2538 },
2539 ],
2540 },
2541 )
2542 .unwrap();
2543
2544 let requeued = queue.get(&requeue_me.id).unwrap();
2545 assert_eq!(requeued.status, TaskStatus::Queued);
2546 assert_eq!(requeued.attempts, 0);
2547
2548 let held = queue.get(&hold_me.id).unwrap();
2549 assert_eq!(held.status, TaskStatus::Held);
2550 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2551 }
2552
2553 #[test]
2554 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2555 let dir = tempdir().unwrap();
2560 let queue = Queue::at(dir.path().join("queue"));
2561 let questions = Questions::at(dir.path().join("questions"));
2562 let mut t = task("held out of attempts");
2563 t.hold_machine(Some("out of attempts".to_owned()));
2564 queue.put(&mut t).unwrap();
2565
2566 apply(
2567 &queue,
2568 &questions,
2569 &Verdict {
2570 decisions: vec![Decision {
2571 id: t.id.clone(),
2572 reason: Some("what should happen to this one?".to_owned()),
2573 question: Some("Hold it, or try again?".to_owned()),
2574 ..Decision::default()
2575 }],
2576 },
2577 )
2578 .unwrap();
2579 let blocked = queue.get(&t.id).unwrap();
2580 assert_eq!(blocked.status, TaskStatus::Blocked);
2581 let question_id = blocked.blocked_by[0].clone();
2582
2583 let mut q = questions.get(&question_id).unwrap();
2584 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2585 questions.put(&mut q).unwrap();
2586
2587 let mut after = queue.get(&t.id).unwrap();
2589 after.record_answer(q.summary.clone(), "leave it held".to_owned());
2590 after.unblock(&question_id);
2591 assert_eq!(
2592 after.status,
2593 TaskStatus::Held,
2594 "must not fall back to queued"
2595 );
2596 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2597 }
2598
2599 #[test]
2600 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2601 let dir = tempdir().unwrap();
2611 let queue = Queue::at(dir.path().join("queue"));
2612 let questions = Questions::at(dir.path().join("questions"));
2613 let mut t = task("disk pressure, then reconsidered");
2614 t.hold_machine(Some(
2615 "not enough free space to start a run: 10 bytes free, 100 required by \
2616 `[disk] min_free_bytes`"
2617 .to_owned(),
2618 ));
2619 t.record_answer(
2620 "How should this be handled?".to_owned(),
2621 "keep it held, a human will look at it later".to_owned(),
2622 );
2623 queue.put(&mut t).unwrap();
2624
2625 apply(
2626 &queue,
2627 &questions,
2628 &Verdict {
2629 decisions: vec![Decision {
2630 id: t.id.clone(),
2631 recovery: Some(Recovery::Hold),
2632 ..Decision::default()
2633 }],
2634 },
2635 )
2636 .unwrap();
2637
2638 let after = queue.get(&t.id).unwrap();
2639 assert_eq!(after.status, TaskStatus::Held);
2640 assert!(
2641 !after
2642 .hold_reason
2643 .as_deref()
2644 .unwrap_or_default()
2645 .starts_with("not enough free space"),
2646 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2647 after.hold_reason
2648 );
2649
2650 let cfg_dir = tempdir().unwrap();
2653 let config = cfg_dir.path().join("magi.toml");
2654 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2655 let report =
2656 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2657 assert!(report.resumed.is_empty(), "must not be auto-released");
2658 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2659 }
2660
2661 #[test]
2662 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2663 crate::run::pin_test_home();
2664 let dir = tempdir().unwrap();
2665 let queue = Queue::at(dir.path().join("queue"));
2666 let questions = Questions::at(dir.path().join("questions"));
2667 let config = dir.path().join("magi.toml");
2668 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2669 let now = jiff::Timestamp::now();
2670 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2671 let open = || {
2672 questions
2673 .list()
2674 .into_iter()
2675 .filter(|q| q.node == "triage" && q.status.open())
2676 .collect::<Vec<_>>()
2677 };
2678 let hold = Decision {
2679 recovery: Some(Recovery::Hold),
2680 reason: Some("waiting on manual worktree cleanup".to_owned()),
2681 ..Decision::default()
2682 };
2683
2684 let mut t = task("looping hold");
2685 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2686 queue.put(&mut t).unwrap();
2687 let hold = Decision {
2688 id: t.id.clone(),
2689 ..hold
2690 };
2691
2692 assert_eq!(triage().asked.len(), 1);
2694 let first = open().remove(0);
2695 let mut q = questions.get(&first.id).unwrap();
2696 let resume = q.choices[0].clone();
2697 q.answer(Answer::Choice(resume)).unwrap();
2698 questions.put(&mut q).unwrap();
2699 assert_eq!(triage().answered.len(), 1);
2700 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2701
2702 apply(
2704 &queue,
2705 &questions,
2706 &Verdict {
2707 decisions: vec![hold.clone()],
2708 },
2709 )
2710 .unwrap();
2711 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2712 assert!(triage().asked.is_empty());
2713
2714 let mut evidence = Question::new(
2716 t.id.clone(),
2717 "preflight".into(),
2718 "check".into(),
2719 "New permission failure".into(),
2720 String::new(),
2721 Vec::new(),
2722 );
2723 questions.put(&mut evidence).unwrap();
2724 let hold = Decision {
2725 evidence: vec![evidence.id.clone()],
2726 reason: Some("new permission check failed".into()),
2727 ..hold
2728 };
2729 apply(
2730 &queue,
2731 &questions,
2732 &Verdict {
2733 decisions: vec![hold.clone()],
2734 },
2735 )
2736 .unwrap();
2737 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2738
2739 assert_eq!(triage().asked.len(), 1);
2741 let second = open().remove(0);
2742 assert!(second.detail.contains("new permission check failed"));
2743 assert!(triage().asked.is_empty(), "no duplicate question");
2744 assert_eq!(open().len(), 1);
2745
2746 let mut q = questions.get(&second.id).unwrap();
2748 let force = q.choices[0].clone();
2749 q.answer(Answer::Choice(force)).unwrap();
2750 questions.put(&mut q).unwrap();
2751 assert_eq!(triage().answered.len(), 1);
2752 apply(
2753 &queue,
2754 &questions,
2755 &Verdict {
2756 decisions: vec![hold],
2757 },
2758 )
2759 .unwrap();
2760 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2761 }
2762
2763 #[test]
2764 fn conductor_resumption_notice_does_not_solicit_unregistered_questions() {
2765 crate::run::pin_test_home();
2766 let dir = tempdir().unwrap();
2767 let queue = Queue::at(dir.path().join("queue"));
2768 let questions = Questions::at(dir.path().join("questions"));
2769
2770 let mut t = task("task with explicit resumption");
2771 t.status = TaskStatus::Queued;
2772 t.resumption = Some(crate::queue::Resumption {
2773 generation: 1,
2774 source: "test".into(),
2775 question_id: Some("q1".into()),
2776 at: jiff::Timestamp::now(),
2777 instruction: "resume".into(),
2778 answers_through: 0,
2779 runs_through: 0,
2780 cleared_reason: None,
2781 blocking_evidence: Vec::new(),
2782 });
2783 queue.put(&mut t).unwrap();
2784
2785 let decision = Decision {
2787 id: t.id.clone(),
2788 question: Some("How should we proceed with this task?".into()),
2789 blocked_by: vec!["dep-x".into()],
2790 reason: Some("needs clarification".into()),
2791 recovery: Some(Recovery::Hold),
2792 ..Decision::default()
2793 };
2794
2795 apply(
2796 &queue,
2797 &questions,
2798 &Verdict {
2799 decisions: vec![decision],
2800 },
2801 )
2802 .unwrap();
2803
2804 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2806 assert!(questions.list().is_empty());
2808
2809 let notices = crate::notices::Notices::open();
2811 let notice = notices
2812 .get(&crate::notices::id_of(&format!(
2813 "conductor-resumption:{}",
2814 t.id
2815 )))
2816 .expect("conductor-resumption notice filed");
2817
2818 assert_eq!(
2819 notice.link,
2820 Some(crate::notices::Link::Task { id: t.id.clone() })
2821 );
2822 assert!(!notice.message.contains("How should we proceed"));
2824 assert!(
2826 notice
2827 .message
2828 .contains("No reply to this notice is needed.")
2829 );
2830 assert!(!notice.message.contains("Some("));
2832 assert!(!notice.message.contains("None"));
2833 assert!(!notice.message.contains("[]"));
2834 assert!(!notice.message.contains("{:?}"));
2835 assert!(notice.message.contains("Proposed dependencies: dep-x."));
2837 }
2838
2839 #[test]
2840 fn a_runnable_task_can_be_held_directly_without_a_question() {
2841 let dir = tempdir().unwrap();
2842 let queue = Queue::at(dir.path().join("queue"));
2843 let questions = Questions::at(dir.path().join("questions"));
2844 let mut t = task("already answered, should stay put");
2845 queue.put(&mut t).unwrap();
2846
2847 apply(
2848 &queue,
2849 &questions,
2850 &Verdict {
2851 decisions: vec![Decision {
2852 id: t.id.clone(),
2853 recovery: Some(Recovery::Hold),
2854 reason: Some("operator already said keep this held".to_owned()),
2855 ..Decision::default()
2856 }],
2857 },
2858 )
2859 .unwrap();
2860
2861 let after = queue.get(&t.id).unwrap();
2862 assert_eq!(after.status, TaskStatus::Held);
2863 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2864 }
2865
2866 #[test]
2867 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2868 crate::run::pin_test_home();
2869 let dir = tempdir().unwrap();
2875 let queue = Queue::at(dir.path().join("queue"));
2876 let questions = Questions::at(dir.path().join("questions"));
2877 let mut t = task("already merged by hand");
2878 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2879 t.record_answer(
2880 "Handle this one?".to_owned(),
2881 "already merged and cleaned up, close it".to_owned(),
2882 );
2883 queue.put(&mut t).unwrap();
2884
2885 apply(
2886 &queue,
2887 &questions,
2888 &Verdict {
2889 decisions: vec![Decision {
2890 id: t.id.clone(),
2891 recovery: Some(Recovery::Done),
2892 reason: Some("operator confirmed this already landed".to_owned()),
2893 ..Decision::default()
2894 }],
2895 },
2896 )
2897 .unwrap();
2898
2899 let after = queue.get(&t.id).unwrap();
2900 assert_eq!(after.status, TaskStatus::Done);
2901 assert!(after.hold_reason.is_none());
2902 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2903 }
2904
2905 #[test]
2906 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2907 let dir = tempdir().unwrap();
2908 let queue = Queue::at(dir.path().join("queue"));
2909 let questions = Questions::at(dir.path().join("questions"));
2910
2911 let mut queued = task("never ran yet");
2912 queue.put(&mut queued).unwrap();
2913
2914 let mut running = task("mid-run");
2915 running.start("run-1".to_owned());
2916 queue.put(&mut running).unwrap();
2917
2918 for id in [queued.id.clone(), running.id.clone()] {
2919 apply(
2920 &queue,
2921 &questions,
2922 &Verdict {
2923 decisions: vec![Decision {
2924 id,
2925 recovery: Some(Recovery::Done),
2926 ..Decision::default()
2927 }],
2928 },
2929 )
2930 .unwrap();
2931 }
2932
2933 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2934 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2935 }
2936
2937 #[test]
2938 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2939 let dir = tempdir().unwrap();
2942 let queue = Queue::at(dir.path().join("queue"));
2943 let questions = Questions::at(dir.path().join("questions"));
2944 let mut t = task("asked about repeatedly");
2945 t.hold_machine(Some("out of attempts".to_owned()));
2946 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2947 t.record_answer(
2948 "Handle this one? (2)".to_owned(),
2949 "still not yet".to_owned(),
2950 );
2951 queue.put(&mut t).unwrap();
2952 assert_eq!(questions.list().len(), 0);
2953
2954 let verdict = Verdict {
2955 decisions: vec![Decision {
2956 id: t.id.clone(),
2957 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2958 ..Decision::default()
2959 }],
2960 };
2961 apply(&queue, &questions, &verdict).unwrap();
2962
2963 let filed = questions.list();
2964 assert_eq!(filed.len(), 1, "the question was filed");
2965 assert_eq!(
2966 filed[0].summary,
2967 "Branch conflicts with origin/main, how do we proceed?"
2968 );
2969 let deputy = filed[0]
2970 .deputy
2971 .as_ref()
2972 .expect("the wait is handed to a deputy");
2973 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2974 assert_eq!(
2975 deputy.starts, 0,
2976 "filing never starts anything: the loop does not wait"
2977 );
2978 assert!(filed[0].status.open());
2979 assert_eq!(filed[0].node, NODE);
2980 let after = queue.get(&t.id).unwrap();
2981 assert_eq!(after.status, TaskStatus::Blocked);
2982 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2983 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2984 assert!(
2985 !after
2986 .hold_reason
2987 .clone()
2988 .unwrap_or_default()
2989 .contains("conduct tried to ask"),
2990 "no hold was applied"
2991 );
2992
2993 apply(&queue, &questions, &verdict).unwrap();
2994 let again = questions.list();
2995 assert_eq!(again.len(), 1, "the open question is reused");
2996 assert_eq!(again[0].id, filed[0].id);
2997 }
2998
2999 #[test]
3000 fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
3001 let dir = tempdir().unwrap();
3002 init_repo_with_branch(dir.path(), "source-a");
3003 let mut t = task("source decision");
3004 t.repo = dir.path().into();
3005 let decision = |reference: &str| QuestionDecision {
3006 operation: QuestionOperation::Recover,
3007 target: QuestionTarget::Source {
3008 reference: reference.into(),
3009 },
3010 };
3011 assert_eq!(
3012 decision("source-a").identity(&t).unwrap(),
3013 decision("refs/heads/source-a").identity(&t).unwrap()
3014 );
3015 assert_ne!(
3016 decision("source-a").identity(&t).unwrap(),
3017 decision("main").identity(&t).unwrap()
3018 );
3019 assert!(decision("invented-source-label").identity(&t).is_err());
3020 let unrelated = QuestionDecision {
3021 operation: QuestionOperation::Resume,
3022 target: QuestionTarget::Run {
3023 id: "unrelated-run".into(),
3024 },
3025 };
3026 assert!(unrelated.identity(&t).is_err());
3027 }
3028
3029 #[test]
3030 fn invalid_question_targets_still_file_and_block_queued_tasks() {
3031 let dir = tempdir().unwrap();
3032 init_repo_with_branch(dir.path(), "source-a");
3033 let queue = Queue::at(dir.path().join("queue"));
3034 let questions = Questions::at(dir.path().join("questions"));
3035 for target in [
3036 QuestionTarget::Source {
3037 reference: "missing-source".into(),
3038 },
3039 QuestionTarget::Run {
3040 id: "unrelated-run".into(),
3041 },
3042 ] {
3043 let mut t = task("decision with invalid metadata");
3044 t.repo = dir.path().into();
3045 queue.put(&mut t).unwrap();
3046 let d = Decision {
3047 id: t.id.clone(),
3048 question: Some("Which recovery should proceed?".into()),
3049 choices: vec!["inspect first".into(), "keep waiting".into()],
3050 question_decision: Some(QuestionDecision {
3051 operation: QuestionOperation::Recover,
3052 target,
3053 }),
3054 ..Decision::default()
3055 };
3056 apply_one(&queue, &questions, &d, None).unwrap();
3057 let q = questions
3058 .list()
3059 .into_iter()
3060 .find(|q| q.run == t.id)
3061 .unwrap();
3062 assert_eq!(q.summary, d.question.as_deref().unwrap());
3063 assert_eq!(q.choices, d.choices);
3064 assert!(q.status.open());
3065 let context: serde_json::Value =
3066 serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
3067 assert!(context[3].is_null(), "invalid metadata was discarded");
3068 let after = queue.get(&t.id).unwrap();
3069 assert_eq!(after.status, TaskStatus::Blocked);
3070 assert_eq!(after.blocked_by, vec![q.id.clone()]);
3071 apply_one(&queue, &questions, &d, None).unwrap();
3072 assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
3073 }
3074 }
3075
3076 #[test]
3077 fn answered_conductor_question_needs_changed_facts_before_reasking() {
3078 let dir = tempdir().unwrap();
3079 let queue = Queue::at(dir.path().join("queue"));
3080 let questions = Questions::at(dir.path().join("questions"));
3081 let mut t = task("recovery decision");
3082 t.hold_machine(Some("out of attempts".into()));
3083 t.runs = vec!["source-run-a".into(), "source-run-b".into()];
3084 queue.put(&mut t).unwrap();
3085 let mut d = Decision {
3086 id: t.id.clone(),
3087 question: Some("Recover from source A?".into()),
3088 choices: vec!["approve source A".into(), "keep held".into()],
3089 question_decision: Some(QuestionDecision {
3090 operation: QuestionOperation::Recover,
3091 target: QuestionTarget::Run {
3092 id: "source-run-a".into(),
3093 },
3094 }),
3095 ..Decision::default()
3096 };
3097 apply_one(&queue, &questions, &d, None).unwrap();
3098 let mut q = questions.list().pop().unwrap();
3099 q.answer(crate::ask::Answer::Choice("keep held".into()))
3100 .unwrap();
3101 questions.put(&mut q).unwrap();
3102 queue
3103 .modify(&t.id, |task| {
3104 task.unblock(&q.id);
3105 task.hold_machine(Some("recovery remains under consideration".into()));
3106 true
3107 })
3108 .unwrap();
3109 d.question = Some("Should we keep waiting on recovery?".into());
3110 d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3111 apply_one(&queue, &questions, &d, None).unwrap();
3112 assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3113 d.question = Some("Recover from source B?".into());
3114 d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3116 id: "source-run-b".into(),
3117 };
3118 apply_one(&queue, &questions, &d, None).unwrap();
3119 assert_eq!(
3120 questions.list().len(),
3121 2,
3122 "new decision with unchanged task facts"
3123 );
3124 let after = queue.get(&t.id).unwrap();
3125 assert_eq!(after.instruction, t.instruction);
3126 assert_eq!(after.runs, t.runs);
3127 assert_eq!(after.status, TaskStatus::Blocked);
3128 }
3129
3130 #[test]
3131 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3132 let dir = tempdir().unwrap();
3133 let queue = Queue::at(dir.path().join("queue"));
3134 let questions = Questions::at(dir.path().join("questions"));
3135 let mut t = task("asked about once already");
3136 t.hold_machine(Some("out of attempts".to_owned()));
3137 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3138 queue.put(&mut t).unwrap();
3139
3140 apply(
3141 &queue,
3142 &questions,
3143 &Verdict {
3144 decisions: vec![Decision {
3145 id: t.id.clone(),
3146 question: Some("Still not sure - now what?".to_owned()),
3147 ..Decision::default()
3148 }],
3149 },
3150 )
3151 .unwrap();
3152
3153 assert_eq!(questions.list().len(), 1, "the second question was filed");
3154 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3155 }
3156
3157 #[test]
3158 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3159 let dir = tempdir().unwrap();
3165 let queue = Queue::at(dir.path().join("queue"));
3166 let questions = Questions::at(dir.path().join("questions"));
3167 let mut t = task("held, triage already asking about it");
3168 t.hold_machine(Some("cause unclear".to_owned()));
3169 queue.put(&mut t).unwrap();
3170
3171 let mut triage_q = Question::new(
3172 t.id.clone(),
3173 crate::triage::NODE.to_owned(),
3174 "triage".to_owned(),
3175 "Still needed?".to_owned(),
3176 String::new(),
3177 vec![
3178 "resume".to_owned(),
3179 "not yet".to_owned(),
3180 "discard".to_owned(),
3181 ],
3182 );
3183 questions.put(&mut triage_q).unwrap();
3184
3185 for decision in [
3186 Decision {
3187 id: t.id.clone(),
3188 question: Some("what now?".to_owned()),
3189 ..Decision::default()
3190 },
3191 Decision {
3192 id: t.id.clone(),
3193 recovery: Some(Recovery::Requeue),
3194 ..Decision::default()
3195 },
3196 ] {
3197 apply(
3198 &queue,
3199 &questions,
3200 &Verdict {
3201 decisions: vec![decision],
3202 },
3203 )
3204 .unwrap();
3205 }
3206
3207 let after = queue.get(&t.id).unwrap();
3208 assert_eq!(
3209 after.status,
3210 TaskStatus::Held,
3211 "triage still owns this hold"
3212 );
3213 assert!(after.blocked_by.is_empty());
3214 assert_eq!(
3215 questions.list().len(),
3216 1,
3217 "no second, conductor-owned question was filed"
3218 );
3219 }
3220
3221 #[test]
3222 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3223 let dir = tempdir().unwrap();
3231 let queue = Queue::at(dir.path().join("queue"));
3232 let questions = Questions::at(dir.path().join("questions"));
3233 let mut t = task("held, triage question answered but not yet applied");
3234 t.hold_machine(Some("cause unclear".to_owned()));
3235 queue.put(&mut t).unwrap();
3236
3237 let mut triage_q = Question::new(
3238 t.id.clone(),
3239 crate::triage::NODE.to_owned(),
3240 "triage".to_owned(),
3241 "Still needed?".to_owned(),
3242 String::new(),
3243 vec![
3244 "resume".to_owned(),
3245 "not yet".to_owned(),
3246 "discard".to_owned(),
3247 ],
3248 );
3249 questions.put(&mut triage_q).unwrap();
3250 triage_q
3251 .answer(Answer::Choice("not yet".to_owned()))
3252 .unwrap();
3253 questions.put(&mut triage_q).unwrap();
3254 assert!(!triage_q.status.open());
3255
3256 apply(
3257 &queue,
3258 &questions,
3259 &Verdict {
3260 decisions: vec![Decision {
3261 id: t.id.clone(),
3262 question: Some("what now?".to_owned()),
3263 ..Decision::default()
3264 }],
3265 },
3266 )
3267 .unwrap();
3268
3269 let after = queue.get(&t.id).unwrap();
3270 assert_eq!(
3271 after.status,
3272 TaskStatus::Held,
3273 "triage's own answer is not yet applied - conduct must wait"
3274 );
3275 assert_eq!(
3276 questions.list().len(),
3277 1,
3278 "no conductor question was filed over the pending triage answer"
3279 );
3280 }
3281
3282 #[test]
3283 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3284 let dir = tempdir().unwrap();
3285 let queue = Queue::at(dir.path().join("queue"));
3286 let questions = Questions::at(dir.path().join("questions"));
3287 let mut held = task("manual recovery");
3288 held.priority = 300;
3289 held.runs.push("run20260912-224242-daf5".to_owned());
3290 held.hold_manual(Some(
3291 "active manual recovery run20260912-224242-daf5".to_owned(),
3292 ));
3293 queue.put(&mut held).unwrap();
3294
3295 for decision in [
3299 Decision {
3300 id: held.id.clone(),
3301 recovery: Some(Recovery::Requeue),
3302 ..Decision::default()
3303 },
3304 Decision {
3305 id: held.id.clone(),
3306 recovery: Some(Recovery::Hold),
3307 reason: Some("stale replacement reason".to_owned()),
3308 ..Decision::default()
3309 },
3310 Decision {
3311 id: held.id.clone(),
3312 recovery: Some(Recovery::Review),
3313 ..Decision::default()
3314 },
3315 Decision {
3316 id: held.id.clone(),
3317 blocked_by: vec!["other-task".to_owned()],
3318 question: Some("retry now?".to_owned()),
3319 ..Decision::default()
3320 },
3321 ] {
3322 apply(
3323 &queue,
3324 &questions,
3325 &Verdict {
3326 decisions: vec![decision],
3327 },
3328 )
3329 .unwrap();
3330 }
3331
3332 let after = queue.get(&held.id).unwrap();
3333 assert_eq!(after.status, TaskStatus::Held);
3334 assert!(after.operator_held());
3335 assert_eq!(after.priority, 300);
3336 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3337 assert_eq!(
3338 after.hold_reason.as_deref(),
3339 Some("active manual recovery run20260912-224242-daf5")
3340 );
3341 assert!(after.blocked_by.is_empty());
3342 assert!(questions.list().is_empty());
3343 assert!(
3344 queue.next_runnable().is_none(),
3345 "must not dispatch a duplicate"
3346 );
3347 }
3348
3349 #[test]
3350 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3351 let dir = tempdir().unwrap();
3352 let queue = Queue::at(dir.path().join("queue"));
3353 let questions = Questions::at(dir.path().join("questions"));
3354
3355 let mut automatic = task("disk gate");
3356 automatic.hold_machine(Some("disk full".to_owned()));
3357 queue.put(&mut automatic).unwrap();
3358 let requeue = || Verdict {
3359 decisions: vec![Decision {
3360 id: automatic.id.clone(),
3361 recovery: Some(Recovery::Requeue),
3362 ..Decision::default()
3363 }],
3364 };
3365 apply(&queue, &questions, &requeue()).unwrap();
3366 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3367
3368 let mut manual = task("operator gate");
3369 manual.hold_manual(Some("wait for operator".to_owned()));
3370 queue.put(&mut manual).unwrap();
3371 apply(
3372 &queue,
3373 &questions,
3374 &Verdict {
3375 decisions: vec![Decision {
3376 id: manual.id.clone(),
3377 recovery: Some(Recovery::Requeue),
3378 ..Decision::default()
3379 }],
3380 },
3381 )
3382 .unwrap();
3383 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3384
3385 let mut released = queue.get(&manual.id).unwrap();
3388 released.release();
3389 queue.put(&mut released).unwrap();
3390 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3391 }
3392
3393 #[test]
3394 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3395 let dir = tempdir().unwrap();
3396 let queue = Queue::at(dir.path().join("queue"));
3397 let questions = Questions::at(dir.path().join("questions"));
3398 let mut legacy = task("old explicit hold");
3399 legacy.status = TaskStatus::Held;
3400 legacy.hold_reason = Some("manual recovery already active".to_owned());
3401 legacy.hold_source = None;
3402 legacy.blocked_by = vec!["dependency".to_owned()];
3403 queue.put(&mut legacy).unwrap();
3404
3405 apply(
3406 &queue,
3407 &questions,
3408 &Verdict {
3409 decisions: vec![Decision {
3410 id: legacy.id.clone(),
3411 recovery: Some(Recovery::Requeue),
3412 ..Decision::default()
3413 }],
3414 },
3415 )
3416 .unwrap();
3417
3418 let after = queue.get(&legacy.id).unwrap();
3419 assert_eq!(after.status, TaskStatus::Held);
3420 assert_eq!(after.hold_source, None);
3421 assert_eq!(after.hold_reason, legacy.hold_reason);
3422 assert_eq!(after.blocked_by, legacy.blocked_by);
3423 }
3424
3425 #[test]
3426 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3427 crate::run::pin_test_home();
3433 let dir = tempdir().unwrap();
3434 let queue = Queue::at(dir.path().join("queue"));
3435 let questions = Questions::at(dir.path().join("questions"));
3436 let mut t = task("blocked with no readable run");
3437 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
3439 queue.put(&mut t).unwrap();
3440
3441 apply(
3442 &queue,
3443 &questions,
3444 &Verdict {
3445 decisions: vec![Decision {
3446 id: t.id.clone(),
3447 recovery: Some(Recovery::Review),
3448 ..Decision::default()
3449 }],
3450 },
3451 )
3452 .unwrap();
3453
3454 let after = queue.get(&t.id).unwrap();
3455 assert_eq!(
3456 after.status,
3457 TaskStatus::Failed,
3458 "with nothing to reopen, the decision is dropped rather than guessed at"
3459 );
3460 assert!(after.review_branch.is_none());
3461 }
3462
3463 #[test]
3464 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3465 let dir = tempdir().unwrap();
3469 let queue = Queue::at(dir.path().join("queue"));
3470 let questions = Questions::at(dir.path().join("questions"));
3471
3472 for recovery in [Recovery::Requeue, Recovery::Review] {
3473 let mut t = task("ordinary");
3474 queue.put(&mut t).unwrap();
3475
3476 apply(
3477 &queue,
3478 &questions,
3479 &Verdict {
3480 decisions: vec![Decision {
3481 id: t.id.clone(),
3482 recovery: Some(recovery),
3483 ..Decision::default()
3484 }],
3485 },
3486 )
3487 .unwrap();
3488
3489 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3490 }
3491 }
3492
3493 #[tokio::test]
3494 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3495 let dir = tempdir().unwrap();
3496 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3497 let queue = Queue::at(dir.path().join("queue"));
3498 let questions = Questions::at(dir.path().join("questions"));
3499 let mut t = task("normal");
3500 queue.put(&mut t).unwrap();
3501
3502 let mut conductor = Conductor::new();
3503 conductor
3504 .maybe_run(
3505 &cfg,
3506 dir.path(),
3507 &queue,
3508 &questions,
3509 dir.path(),
3510 &[t.clone()],
3511 &[],
3512 &[],
3513 2,
3514 )
3515 .await;
3516
3517 assert_eq!(
3518 queue.get(&t.id).unwrap().status,
3519 TaskStatus::Queued,
3520 "a failed invocation must change nothing"
3521 );
3522 assert!(
3523 queue.next_runnable().is_some(),
3524 "the loop must still be able to take the next task"
3525 );
3526 }
3527
3528 #[tokio::test]
3529 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3530 let dir = tempdir().unwrap();
3531 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3532 let queue = Queue::at(dir.path().join("queue"));
3533 let questions = Questions::at(dir.path().join("questions"));
3534 let mut t = task("normal");
3535 queue.put(&mut t).unwrap();
3536
3537 let mut conductor = Conductor::new();
3538 conductor
3539 .maybe_run(
3540 &cfg,
3541 dir.path(),
3542 &queue,
3543 &questions,
3544 dir.path(),
3545 &[t.clone()],
3546 &[],
3547 &[],
3548 2,
3549 )
3550 .await;
3551
3552 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3553 }
3554
3555 #[tokio::test]
3556 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3557 let dir = tempdir().unwrap();
3558 let mut t = task("chained");
3559 let reply = format!(
3560 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3561 t.id
3562 );
3563 let counted = |id: &str, tail: &str| {
3564 let calls = dir.path().join(format!("{id}.calls"));
3565 let path = dir.path().join(format!("{id}.sh"));
3566 std::fs::write(
3567 &path,
3568 format!(
3569 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3570 calls.display()
3571 ),
3572 )
3573 .unwrap();
3574 AgentSpec {
3575 id: id.to_owned(),
3576 kind: AgentKind::Command,
3577 model: None,
3578 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3579 extra_args: Vec::new(),
3580 env: BTreeMap::new(),
3581 prompt_delivery: None,
3582 }
3583 };
3584 let n_calls = |id: &str| {
3585 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3586 .map_or(0, |s| s.lines().count())
3587 };
3588 let mut cfg = config(counted("a", "exit 3"));
3589 cfg.agents = vec![
3590 counted("a", "exit 3"),
3591 counted("b", &format!("printf '%s' '{reply}'")),
3592 ];
3593 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3594 "a".into(),
3595 "b".into(),
3596 "a".into(),
3597 ]));
3598 let queue = Queue::at(dir.path().join("queue"));
3599 let questions = Questions::at(dir.path().join("questions"));
3600 queue.put(&mut t).unwrap();
3601
3602 let mut conductor = Conductor::new();
3603 conductor
3604 .maybe_run(
3605 &cfg,
3606 dir.path(),
3607 &queue,
3608 &questions,
3609 dir.path(),
3610 &[t.clone()],
3611 &[],
3612 &[],
3613 2,
3614 )
3615 .await;
3616
3617 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3618 assert_eq!(n_calls("a"), 1, "each id is tried once");
3619 assert_eq!(n_calls("b"), 1);
3620 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3621 assert!(
3622 saved.contains("\"b\""),
3623 "the seat on disk is the one that ran: {saved}"
3624 );
3625 }
3626
3627 #[tokio::test]
3628 async fn json_survives_code_fences_and_a_preamble() {
3629 let dir = tempdir().unwrap();
3630 let mut t = task("fenced");
3631 let reply = format!(
3632 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3633 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3634 t.id
3635 );
3636 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3637 let queue = Queue::at(dir.path().join("queue"));
3638 let questions = Questions::at(dir.path().join("questions"));
3639 queue.put(&mut t).unwrap();
3640
3641 let mut conductor = Conductor::new();
3642 conductor
3643 .maybe_run(
3644 &cfg,
3645 dir.path(),
3646 &queue,
3647 &questions,
3648 dir.path(),
3649 &[t.clone()],
3650 &[],
3651 &[],
3652 2,
3653 )
3654 .await;
3655
3656 let back = queue.get(&t.id).unwrap();
3657 assert_eq!(back.status, TaskStatus::Blocked);
3658 assert_eq!(back.blocked_by, ["x"]);
3659 }
3660
3661 #[tokio::test]
3662 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3663 let dir = tempdir().unwrap();
3666 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3667 let queue = Queue::at(dir.path().join("queue"));
3668 let questions = Questions::at(dir.path().join("questions"));
3669 let mut t = task("stable");
3670 queue.put(&mut t).unwrap();
3671 let artifacts = dir.path().join("conduct").join("artifacts");
3672 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3673
3674 let mut conductor = Conductor::new();
3675 conductor
3676 .maybe_run(
3677 &cfg,
3678 dir.path(),
3679 &queue,
3680 &questions,
3681 dir.path(),
3682 &[t.clone()],
3683 &[],
3684 &[],
3685 2,
3686 )
3687 .await;
3688 assert!(turn(1).is_file(), "the first cycle must call the conductor");
3689
3690 conductor
3691 .maybe_run(
3692 &cfg,
3693 dir.path(),
3694 &queue,
3695 &questions,
3696 dir.path(),
3697 &[t.clone()],
3698 &[],
3699 &[],
3700 2,
3701 )
3702 .await;
3703 assert!(
3704 !turn(2).is_file(),
3705 "an unchanged revision and an unchanged stalled/finished set must not call the \
3706 conductor twice"
3707 );
3708
3709 t.priority = 1;
3711 queue.put(&mut t).unwrap();
3712 conductor
3713 .maybe_run(
3714 &cfg,
3715 dir.path(),
3716 &queue,
3717 &questions,
3718 dir.path(),
3719 &[t.clone()],
3720 &[],
3721 &[],
3722 2,
3723 )
3724 .await;
3725 assert!(turn(2).is_file(), "a moved revision calls it again");
3726 }
3727
3728 #[tokio::test]
3729 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3730 let dir = tempdir().unwrap();
3736 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3737 let queue = Queue::at(dir.path().join("queue"));
3738 let questions = Questions::at(dir.path().join("questions"));
3739 let mut t = task("quiet");
3740 queue.put(&mut t).unwrap();
3741 let artifacts = dir.path().join("conduct").join("artifacts");
3742 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3743
3744 let mut conductor = Conductor::new();
3745 conductor
3746 .maybe_run(
3747 &cfg,
3748 dir.path(),
3749 &queue,
3750 &questions,
3751 dir.path(),
3752 &[t.clone()],
3753 &[],
3754 &[],
3755 2,
3756 )
3757 .await;
3758 assert!(turn(1).is_file());
3759
3760 conductor
3761 .maybe_run(
3762 &cfg,
3763 dir.path(),
3764 &queue,
3765 &questions,
3766 dir.path(),
3767 &[],
3768 &[t.clone()],
3769 &[],
3770 2,
3771 )
3772 .await;
3773 assert!(
3774 turn(2).is_file(),
3775 "a task turning stalled must call the conductor again"
3776 );
3777
3778 conductor
3781 .maybe_run(
3782 &cfg,
3783 dir.path(),
3784 &queue,
3785 &questions,
3786 dir.path(),
3787 &[],
3788 &[t.clone()],
3789 &[],
3790 2,
3791 )
3792 .await;
3793 assert!(
3794 !turn(3).is_file(),
3795 "the same stalled task lingering must not call the conductor every cycle"
3796 );
3797 }
3798
3799 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3800 apply(
3801 queue,
3802 questions,
3803 &Verdict {
3804 decisions: vec![Decision {
3805 id: id.to_owned(),
3806 reason: Some(note.to_owned()),
3807 recovery: Some(Recovery::Hold),
3808 ..Decision::default()
3809 }],
3810 },
3811 )
3812 .unwrap();
3813 }
3814
3815 #[test]
3816 fn reholding_with_an_identical_note_changes_nothing() {
3817 let dir = tempdir().unwrap();
3818 let queue = Queue::at(dir.path().join("queue"));
3819 let questions = Questions::at(dir.path().join("questions"));
3820 let mut t = task("held");
3821 t.hold_machine(Some("waiting on a human".to_owned()));
3822 queue.put(&mut t).unwrap();
3823
3824 rehold(&queue, &questions, &t.id, "owner said keep it held");
3825 let first = queue.get(&t.id).unwrap();
3826 let rev = queue.revision();
3827 for _ in 0..5 {
3828 rehold(&queue, &questions, &t.id, "owner said keep it held");
3829 }
3830 let after = queue.get(&t.id).unwrap();
3831 assert_eq!(after.hold_reason, first.hold_reason);
3832 assert_eq!(after.updated_at, first.updated_at);
3833 assert_eq!(queue.revision(), rev, "nothing was written");
3834 }
3835
3836 #[test]
3837 fn reholding_with_different_notes_keeps_one_previously_level() {
3838 let dir = tempdir().unwrap();
3839 let queue = Queue::at(dir.path().join("queue"));
3840 let questions = Questions::at(dir.path().join("questions"));
3841 let mut t = task("held");
3842 t.hold_machine(Some("original".to_owned()));
3843 queue.put(&mut t).unwrap();
3844
3845 for i in 0..20 {
3846 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3847 }
3848 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3849 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3850 assert!(reason.starts_with("note "), "the new note leads: {reason}");
3851 }
3852
3853 #[test]
3854 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3855 let mut t = task("deep");
3856 let nested = "same\n\n(previously: same\n\n(previously: same))";
3857 t.hold_machine(Some(nested.to_owned()));
3858 let d = Decision {
3859 id: t.id.clone(),
3860 reason: Some("same".to_owned()),
3861 ..Decision::default()
3862 };
3863 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3864 let d2 = Decision {
3865 reason: Some("other".to_owned()),
3866 ..d
3867 };
3868 assert_eq!(
3869 reaffirmed_hold_reason(&t, &d2).as_deref(),
3870 Some("other\n\n(previously: same)")
3871 );
3872 }
3873
3874 #[tokio::test]
3875 async fn a_repeated_hold_decision_settles_instead_of_looping() {
3876 let dir = tempdir().unwrap();
3877 let queue = Queue::at(dir.path().join("queue"));
3878 let questions = Questions::at(dir.path().join("questions"));
3879 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3880 let mut t = task("held");
3881 t.hold_machine(Some("first".to_owned()));
3882 queue.put(&mut t).unwrap();
3883 let cfg = config(mock_agent(
3884 dir.path(),
3885 REPLY,
3886 env(&reply.replace("ID", &t.id)),
3887 ));
3888 let mut conductor = Conductor::new();
3889 for _ in 0..2 {
3891 let held = queue.get(&t.id).unwrap();
3892 conductor
3893 .maybe_run(
3894 &cfg,
3895 dir.path(),
3896 &queue,
3897 &questions,
3898 dir.path(),
3899 &[],
3900 &[],
3901 &[held],
3902 2,
3903 )
3904 .await;
3905 }
3906 let held = queue.get(&t.id).unwrap();
3907 assert!(
3908 held.hold_reason
3909 .as_deref()
3910 .unwrap()
3911 .starts_with("keep held")
3912 );
3913 assert!(
3914 !conductor.worth_a_look(&queue, &[], &[held]),
3915 "an unchanged held task must stop being reconsidered"
3916 );
3917 }
3918
3919 #[tokio::test]
3920 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3921 let dir = tempdir().unwrap();
3922 let queue = Queue::at(dir.path().join("queue"));
3923 let questions = Questions::at(dir.path().join("questions"));
3924 let mut t = task("held");
3925 t.hold_machine(Some("first".to_owned()));
3926 queue.put(&mut t).unwrap();
3927 let reply = |note: &str| {
3928 format!(
3929 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3930 t.id
3931 )
3932 };
3933 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3934 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3935 let mut conductor = Conductor::new();
3936 let mut writes = 0;
3937 for i in 0..6 {
3938 let held = queue.get(&t.id).unwrap();
3939 let before = queue.revision();
3940 conductor
3941 .maybe_run(
3942 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3943 dir.path(),
3944 &queue,
3945 &questions,
3946 dir.path(),
3947 &[],
3948 &[],
3949 &[held],
3950 2,
3951 )
3952 .await;
3953 if queue.revision() != before {
3954 writes += 1;
3955 }
3956 }
3957 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3958 let mut held = queue.get(&t.id).unwrap();
3959 held.instruction.push_str(" (edited)");
3960 queue.put(&mut held).unwrap();
3961 let held = queue.get(&t.id).unwrap();
3962 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3963 }
3964
3965 #[test]
3966 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3967 let dir = tempdir().unwrap();
3968 let queue = Queue::at(dir.path().join("queue"));
3969 let mut t = task("t");
3970 queue.put(&mut t).unwrap();
3971
3972 let mut conductor = Conductor::new();
3973 assert!(
3974 conductor.worth_a_look(&queue, &[], &[]),
3975 "a conductor that has never run has something to look at"
3976 );
3977
3978 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3979 assert!(
3980 !conductor.worth_a_look(&queue, &[], &[]),
3981 "nothing changed and nothing is stalled or finished"
3982 );
3983 assert!(
3984 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3985 "a stalled task is worth a look even at the same revision"
3986 );
3987 assert!(
3988 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3989 "a finished task is worth a look even at the same revision"
3990 );
3991 }
3992
3993 #[tokio::test]
3994 async fn the_conduct_path_never_calls_ask_and_wait() {
3995 let dir = tempdir().unwrap();
4001 let queue = Queue::at(dir.path().join("queue"));
4002 let questions = Questions::at(dir.path().join("questions"));
4003 let mut t = task("asks without blocking");
4004 queue.put(&mut t).unwrap();
4005
4006 apply(
4007 &queue,
4008 &questions,
4009 &Verdict {
4010 decisions: vec![Decision {
4011 id: t.id.clone(),
4012 question: Some("ok?".to_owned()),
4013 ..Decision::default()
4014 }],
4015 },
4016 )
4017 .unwrap();
4018 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
4020 }
4021
4022 #[test]
4023 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
4024 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
4025 t.resume_override = Some(crate::queue::OperatorResume {
4026 question_id: "q".into(),
4027 at: jiff::Timestamp::now(),
4028 conductor_rehold: None,
4029 forced: false,
4030 pinned_run: Some("run-1".into()),
4031 });
4032 assert!(pinned_resume(&t));
4033 assert!(!may_hold(&mut t, "waiting for magi resume"));
4034 assert!(
4035 t.resume_override
4036 .as_ref()
4037 .unwrap()
4038 .conductor_rehold
4039 .is_none(),
4040 "a refused hold is not recorded as an override"
4041 );
4042 t.resume_override = None;
4043 assert!(!pinned_resume(&t));
4044 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
4045 }
4046
4047 #[test]
4048 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
4049 let dir = tempdir().unwrap();
4050 let queue = Queue::at(dir.path().join("queue"));
4051 let questions = Questions::at(dir.path().join("questions"));
4052 let mut t = task("resume me");
4053 t.resume_override = Some(crate::queue::OperatorResume {
4054 question_id: "q".into(),
4055 at: jiff::Timestamp::now(),
4056 conductor_rehold: None,
4057 forced: true,
4058 pinned_run: Some("run-1".into()),
4059 });
4060 queue.put(&mut t).unwrap();
4061
4062 apply(
4063 &queue,
4064 &questions,
4065 &Verdict {
4066 decisions: vec![
4067 Decision {
4068 id: t.id.clone(),
4069 question: Some("really?".to_owned()),
4070 ..Decision::default()
4071 },
4072 Decision {
4073 id: t.id.clone(),
4074 blocked_by: vec!["other".to_owned()],
4075 ..Decision::default()
4076 },
4077 ],
4078 },
4079 )
4080 .unwrap();
4081
4082 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
4083 assert!(questions.list().is_empty());
4084 }
4085}