1use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::Deserialize;
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58const SEAT: &str = "conduct";
61
62pub const NODE: &str = "conduct";
66
67const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78 Requeue,
81 Hold,
84 Review,
89 Done,
97}
98
99#[derive(Debug, Clone, Default, Deserialize)]
102pub struct Decision {
103 pub id: String,
105 #[serde(default)]
109 pub blocked_by: Vec<String>,
110 #[serde(default)]
115 pub missing_dependencies: Vec<String>,
116 #[serde(default)]
118 pub reason: Option<String>,
119 #[serde(default)]
126 pub recovery: Option<Recovery>,
127 #[serde(default)]
133 pub question: Option<String>,
134 #[serde(default)]
136 pub choices: Vec<String>,
137 #[serde(default)]
139 pub evidence: Vec<String>,
140}
141
142#[derive(Debug, Clone, Default, Deserialize)]
155pub struct Verdict {
156 pub decisions: Vec<Decision>,
159}
160
161fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
164 prompt::ConductTask {
165 id: t.id.clone(),
166 title: t.title.clone(),
167 instruction: t.instruction.clone(),
168 repo: t.repo.display().to_string(),
169 priority: t.priority,
170 status: t.status.as_str().to_owned(),
171 attempts: t.attempts,
172 max_attempts: t.effective_attempt_limit(max_attempts),
173 last_error: t.last_error.clone(),
174 hold_reason: t.hold_reason.clone(),
175 hold_source: t.hold_source.map(|source| source.label().to_owned()),
176 blocked_by: t.blocked_by.clone(),
177 answers: t
178 .answers
179 .iter()
180 .map(|a| prompt::ConductAnswer {
181 question: a.question.clone(),
182 answer: a.answer.clone(),
183 })
184 .collect(),
185 operator_resume: t.resumption.as_ref().map(|r| format!(
186 "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
187 The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
188 To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
189 wording alone is insufficient.",
190 r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
191 )).or_else(|| t.resume_override.as_ref().map(|o| {
192 format!(
193 "the operator explicitly answered \"resume\" at {}; do not hold this \
194 task again for the same reason unless there is new information",
195 o.at
196 )
197 })),
198 }
199}
200
201fn pinned_resume(task: &Task) -> bool {
205 task.resume_override
206 .as_ref()
207 .is_some_and(|o| o.pinned_run.is_some())
208}
209
210fn new_blocking_evidence(
213 queue: &Queue,
214 questions: &Questions,
215 task: &Task,
216 d: &Decision,
217) -> Vec<String> {
218 let Some(r) = &task.resumption else {
219 return Vec::new();
220 };
221 d.evidence
222 .iter()
223 .filter_map(|id| {
224 let version = if let Ok(q) = questions.get(id) {
225 (q.asked_at > r.at
226 && (q.run == task.id || task.runs.contains(&q.run))
227 && q.node != NODE
228 && q.status != crate::ask::QuestionStatus::Abandoned)
229 .then(|| {
230 format!(
231 "question:{id}:{}:{:?}:{}",
232 q.asked_at,
233 q.status,
234 q.resolution().unwrap_or_default()
235 )
236 })
237 } else if d.blocked_by.contains(id) {
238 queue
239 .get(id)
240 .ok()
241 .filter(|dep| {
242 dep.id != task.id && dep.updated_at > r.at && dep.status != TaskStatus::Done
243 })
244 .map(|dep| {
245 format!(
246 "dependency:{id}:{:?}:{:?}:{:?}",
247 dep.status, dep.hold_reason, dep.last_error
248 )
249 })
250 } else if task.runs.iter().any(|run| run == id) {
251 RunState::load(id)
252 .ok()
253 .filter(|run| {
254 run.updated_at > r.at
257 && run
258 .passes
259 .last()
260 .map_or(run.created_at > r.at, |p| p.started_at > r.at)
261 && matches!(
262 run.status,
263 crate::run::RunStatus::Failed
264 | crate::run::RunStatus::Blocked
265 | crate::run::RunStatus::Stalled
266 )
267 })
268 .map(|run| {
269 format!(
270 "run:{id}:{:?}:{}:{}:{:?}",
271 run.status,
272 run.reviews.len(),
273 task.attempts,
274 run.passes.last().map(|p| (p.index, p.started_at))
275 )
276 })
277 } else {
278 None
279 };
280 version.filter(|v| !r.blocking_evidence.contains(v))
281 })
282 .collect()
283}
284
285fn may_hold(task: &mut Task, reason: &str) -> bool {
289 if pinned_resume(task) {
290 return false;
291 }
292 if task.resumption.is_some() {
293 return true; }
295 let Some(o) = task.resume_override.as_mut() else {
296 return true;
297 };
298 if o.forced || o.pinned_run.is_some() {
299 tracing::warn!(
300 "conductor tried to hold task {} after the operator forced a resume: {reason}",
301 task.id
302 );
303 return false;
304 }
305 if o.conductor_rehold.is_some() {
306 return false;
307 }
308 o.conductor_rehold = Some(reason.to_owned());
309 true
310}
311
312fn severity_str(s: crate::verdict::Severity) -> &'static str {
315 match s {
316 crate::verdict::Severity::Nit => "nit",
317 crate::verdict::Severity::Minor => "minor",
318 crate::verdict::Severity::Major => "major",
319 crate::verdict::Severity::Blocker => "blocker",
320 }
321}
322
323fn surviving_branch(task: &Task) -> Option<String> {
328 let last = task.runs.last()?;
329 let state = RunState::load(last).ok()?;
330 if !crate::daemon::compatible_run_instruction(&state, task) {
331 return None;
332 }
333 state.winner().map(|c| c.branch.clone())
334}
335
336fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
363 let note = match &d.reason {
364 Some(reason) => reason.clone(),
365 None => match task.answers.last() {
366 Some(a) => format!(
367 "conduct held this again with no new reason given; last operator \
368 answer on record: {}",
369 a.answer
370 ),
371 None => "conduct held this again with no reason given".to_owned(),
372 },
373 };
374 match task.hold_reason.as_deref() {
375 Some(prior) if !prior.is_empty() => {
376 let outer = outermost_hold_note(prior);
377 if note.trim() == prior.trim() || note.trim() == outer {
378 None
379 } else {
380 Some(format!("{note}\n\n(previously: {outer})"))
381 }
382 }
383 _ => Some(note),
384 }
385}
386
387const PREVIOUSLY: &str = "\n\n(previously: ";
390
391fn outermost_hold_note(reason: &str) -> &str {
394 reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
395}
396
397fn hold_note(d: &Decision) -> String {
399 d.reason
400 .clone()
401 .unwrap_or_else(|| "(no reason given)".to_owned())
402}
403
404async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
406 let Some(run_id) = task.runs.last().cloned() else {
407 return prompt::ConductOutcome {
408 run_id: "(none)".to_owned(),
409 unreadable: Some("this task has not produced a run yet".to_owned()),
410 run_status: None,
411 open_findings: Vec::new(),
412 rounds_used: 0,
413 rounds_max: 0,
414 rounds: Vec::new(),
415 branch: None,
416 branch_head: None,
417 references: None,
418 empty_candidate: false,
419 };
420 };
421 let state = match RunState::load(&run_id) {
422 Ok(s) => s,
423 Err(e) => {
424 tracing::warn!(
429 "conductor: could not read run {run_id} for task {}: {e:#}",
430 task.short()
431 );
432 return prompt::ConductOutcome {
433 run_id,
434 unreadable: Some(format!("{e:#}")),
435 run_status: None,
436 open_findings: Vec::new(),
437 rounds_used: 0,
438 rounds_max: 0,
439 rounds: Vec::new(),
440 branch: None,
441 branch_head: None,
442 references: None,
443 empty_candidate: false,
444 };
445 }
446 };
447
448 let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
449 id: f.id.clone(),
450 title: f.title.clone(),
451 severity: severity_str(f.severity).to_owned(),
452 };
453 let open_findings = state
454 .open_findings()
455 .into_iter()
456 .map(finding_view)
457 .collect();
458 let rounds = state
459 .reviews
460 .iter()
461 .map(|r| prompt::ConductRound {
462 round: r.round,
463 findings: r
464 .reviews
465 .iter()
466 .flat_map(|rec| rec.findings.iter())
467 .map(finding_view)
468 .collect(),
469 addressed: r
470 .fix
471 .as_ref()
472 .map(|fx| fx.addressed.clone())
473 .unwrap_or_default(),
474 rejected: r
475 .fix
476 .as_ref()
477 .map(|fx| {
478 fx.rejected
479 .iter()
480 .map(|rej| prompt::ConductRejection {
481 id: rej.id.clone(),
482 why: rej.why.clone(),
483 })
484 .collect()
485 })
486 .unwrap_or_default(),
487 })
488 .collect();
489 let branch = state.winner().map(|c| c.branch.clone());
490 let branch_head = match &branch {
491 Some(b) => crate::git::rev_parse(repo, b)
492 .await
493 .ok()
494 .map(|h| h.chars().take(8).collect()),
495 None => None,
496 };
497
498 prompt::ConductOutcome {
499 run_id,
500 unreadable: None,
501 run_status: Some(state.status.as_str().to_owned()),
502 open_findings,
503 rounds_used: state.reviews.len(),
504 rounds_max: state.config.graph.review_rounds,
505 rounds,
506 branch,
507 branch_head,
508 references: crate::refs::describe(&state.seeds),
509 empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
510 }
511}
512
513async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
517 let id = task.runs.last()?;
518 let state = match RunState::load(id) {
519 Ok(s) => s,
520 Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
521 };
522 let branch = state.winner()?.branch.clone();
523 let base = state.base_branch.clone();
524 match crate::land::find_open_pr(repo, &branch, &base).await {
525 Ok(crate::land::OpenPr::None) => None,
526 Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
527 "pull request {url} is already open for {branch} into {base}"
528 )),
529 Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
530 "several pull requests are already open for {branch} into {base}: {}",
531 urls.join(" ")
532 )),
533 Err(e) => Some(format!("could not check open pull requests: {e:#}")),
534 }
535}
536
537async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
544 for d in verdict
545 .decisions
546 .iter_mut()
547 .filter(|d| d.question.is_some())
548 {
549 let Ok(task) = queue.get(&d.id) else {
550 continue;
551 };
552 let text = format!("{}\n{}", task.title, task.instruction);
553 let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
554 if crate::refs::scan(&text).is_empty() {
555 if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
556 q.push_str(&format!(
557 "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
558 ));
559 }
560 continue;
561 }
562 let repo = repo_for(&task, repo);
563 let remote = &cfg.merge.remote;
564 let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
565 .await
566 .ok();
567 let facts = match base {
568 Some(base) => {
569 let base_name = base.clone();
570 let tracking = format!("{remote}/{base}");
571 let refreshed = crate::git::fetch(&repo, remote, &base)
574 .await
575 .is_ok_and(|o| o.ok());
576 let against = tracking;
579 match crate::git::rev_parse(&repo, &against).await {
580 Ok(tip) => crate::refs::describe(
581 &crate::refs::resolve(&repo, &tip, remote, &text).await,
582 )
583 .map(|facts| {
584 if refreshed {
585 facts
586 } else {
587 format!(
588 "{facts}\n(could not fetch {remote}/{base_name}: this is \
589 against the last fetched `{against}`, which may be behind \
590 the remote)"
591 )
592 }
593 }),
594 Err(_) => Some(format!(
595 "could NOT be checked against the repository: {against} is not \
596 available here, so whether the work is in the base is unknown"
597 )),
598 }
599 }
600 None => Some("could not check the repository: no base branch known".to_owned()),
601 };
602 let facts = match (facts, pr_fact) {
603 (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
604 (f, p) => f.or(p),
605 };
606 if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
607 q.push_str(&format!(
608 "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
609 ));
610 }
611 }
612}
613
614async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
616 prompt::ConductFinished {
617 task: view(t, max_attempts),
618 outcome: outcome_for(t, &repo_for(t, repo)).await,
619 }
620}
621
622fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
625 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
626 fallback.to_path_buf()
627 } else {
628 task.repo.clone()
629 }
630}
631
632fn apply_one(
649 queue: &Queue,
650 questions: &Questions,
651 d: &Decision,
652 snapshot: Option<&Task>,
653) -> Result<()> {
654 let _claim = queue
655 .claim(&d.id)
656 .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
657 let mut task = queue.get(&d.id).context("no such task")?;
658
659 if snapshot
662 .is_some_and(|old| serde_json::to_value(old).ok() != serde_json::to_value(&task).ok())
663 {
664 return Ok(());
665 }
666 let blocking =
667 d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
668 let evidence = new_blocking_evidence(queue, questions, &task, d);
669 if blocking && task.resumption.is_some() && evidence.is_empty() {
670 crate::notices::raise(
671 crate::notices::Notice::warn(
672 &format!("conductor-resumption:{}", task.id),
673 format!(
674 "Conductor concern after explicit resumption (no verified new evidence): {}; question: {:?}; dependencies: {:?}; evidence: {:?}",
675 hold_note(d), d.question, d.blocked_by, d.evidence
676 ),
677 )
678 .about([task.id.clone()]),
679 );
680 return Ok(());
681 }
682
683 if blocking {
684 if let Some(r) = task.resumption.as_mut() {
685 r.blocking_evidence.extend(evidence);
686 }
687 }
688
689 if task.operator_held() {
693 return Ok(());
694 }
695
696 if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
700 return Ok(());
701 }
702
703 let pinned = pinned_resume(&task);
706 if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
707 return Ok(());
708 }
709
710 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
714 && !d.missing_dependencies.is_empty()
715 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
716 && d.missing_dependencies
717 .iter()
718 .any(|id| !d.blocked_by.contains(id))
719 && d.missing_dependencies
720 .iter()
721 .chain(&d.blocked_by)
722 .any(|id| {
723 matches!(
724 crate::queue::dependency(queue, questions, id),
725 crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
726 )
727 })
728 {
729 return Ok(());
730 }
731
732 if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
738 && !d.missing_dependencies.is_empty()
739 && d.missing_dependencies.iter().all(|id| {
740 let claim = crate::queue::dependency(queue, questions, id);
741 claim != crate::queue::Dependency::Missing
742 && d.blocked_by.iter().any(|blocker| {
743 blocker == id
744 || (matches!(
745 &claim,
746 crate::queue::Dependency::Task(_)
747 | crate::queue::Dependency::Question(_)
748 | crate::queue::Dependency::Deleted(_)
749 ) && crate::queue::dependency(queue, questions, blocker) == claim)
750 })
751 })
752 && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
753 && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
754 {
755 let mut ids = d.blocked_by.clone();
756 crate::queue::normalize_blockers(queue, questions, &mut ids);
757 let reason = format!("Waiting on dependencies: {}", ids.join(", "));
758 task.block(ids, Some(reason));
759 return queue.put(&mut task);
760 }
761
762 if let Some(text) = &d.question {
763 if task.status == TaskStatus::Done {
764 return Ok(());
765 }
766 let question_id = match questions
770 .list()
771 .into_iter()
772 .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
773 {
774 Some(existing) => existing.id,
775 None => {
783 let mut q = Question::new(
784 task.id.clone(),
785 NODE.to_owned(),
786 SEAT.to_owned(),
787 text.clone(),
788 d.reason.clone().unwrap_or_default(),
789 d.choices.clone(),
790 );
791 q.deputy = Some(Deputy::new(crate::deputy::brief(
796 &task.id,
797 d.reason.as_deref().unwrap_or_default(),
798 &q.choices,
799 &q.actions,
800 )));
801 if task.repo.is_dir() && task.repo != Path::new(".") {
802 q.cwd = Some(task.repo.to_string_lossy().into_owned());
803 }
804 questions.put(&mut q)?;
805 q.id
806 }
807 };
808 task.block(vec![question_id], d.reason.clone());
809 return queue.put(&mut task);
810 }
811
812 match task.status {
813 TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
817 if may_hold(&mut task, &hold_note(d)) {
818 task.hold_machine(d.reason.clone());
819 queue.put(&mut task)?;
820 }
821 }
822 TaskStatus::Queued if !d.blocked_by.is_empty() => {
823 let mut ids = d.blocked_by.clone();
824 crate::queue::normalize_blockers(queue, questions, &mut ids);
825 task.block(ids, d.reason.clone());
826 queue.put(&mut task)?;
827 }
828 TaskStatus::Running => match d.recovery {
829 Some(Recovery::Requeue) if pinned_resume(&task) => {}
830 Some(Recovery::Requeue) => {
831 task.requeue();
832 queue.put(&mut task)?;
833 }
834 Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
835 task.hold_machine(d.reason.clone());
836 queue.put(&mut task)?;
837 }
838 _ => {}
842 },
843 TaskStatus::Failed | TaskStatus::Held => match d.recovery {
844 Some(Recovery::Requeue) if pinned_resume(&task) => {}
847 Some(Recovery::Requeue) => {
848 task.requeue();
849 queue.put(&mut task)?;
850 }
851 Some(Recovery::Hold) => {
852 if let Some(reason) = reaffirmed_hold_reason(&task, d) {
856 if may_hold(&mut task, &hold_note(d)) {
857 task.hold_machine(Some(reason));
858 queue.put(&mut task)?;
859 }
860 }
861 }
862 Some(Recovery::Review) => {
863 if let Some(branch) = surviving_branch(&task) {
864 task.request_review(branch);
865 queue.put(&mut task)?;
866 }
867 }
873 Some(Recovery::Done) => {
874 task.succeed();
875 crate::daemon::supersede_prior_runs(&task, &crate::run::home());
879 queue.put(&mut task)?;
880 }
881 None => {}
882 },
883 _ => {}
886 }
887 Ok(())
888}
889
890pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
894 for d in &verdict.decisions {
895 if let Err(e) = apply_one(queue, questions, d, None) {
896 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
897 }
898 }
899 Ok(())
900}
901
902pub fn seat_path(home: &Path) -> PathBuf {
908 home.join("conduct").join("seat.json")
909}
910
911pub fn load_seat(home: &Path) -> Option<SeatState> {
913 serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
914}
915
916fn busy_path(home: &Path) -> PathBuf {
917 home.join("conduct").join("busy")
918}
919
920pub fn busy(home: &Path) -> bool {
926 std::fs::metadata(busy_path(home))
927 .and_then(|m| m.modified())
928 .ok()
929 .and_then(|t| t.elapsed().ok())
930 .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
931}
932
933struct Busy(PathBuf);
935
936impl Busy {
937 fn mark(home: &Path) -> Self {
938 let path = busy_path(home);
939 if let Some(dir) = path.parent() {
940 let _ = std::fs::create_dir_all(dir);
941 }
942 let _ = std::fs::write(&path, std::process::id().to_string());
943 Self(path)
944 }
945}
946
947impl Drop for Busy {
948 fn drop(&mut self) {
949 let _ = std::fs::remove_file(&self.0);
950 }
951}
952
953#[derive(Debug, Default)]
956pub struct Conductor {
957 seat: Option<SeatState>,
958 last_seen: Option<(u64, BTreeSet<String>)>,
959 considered: BTreeMap<String, String>,
963}
964
965fn input_fingerprint(task: &Task) -> String {
970 let mut v = serde_json::to_value(task).unwrap_or_default();
971 if let Some(o) = v.as_object_mut() {
972 o.remove("updated_at");
973 o.remove("hold_reason");
974 if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
975 ro.remove("conductor_rehold");
976 }
977 }
978 v.to_string()
979}
980
981impl Conductor {
982 #[must_use]
984 pub fn new() -> Self {
985 Self::default()
986 }
987
988 fn settled(
993 considered: &BTreeMap<String, String>,
994 stalled: &[Task],
995 finished: &[Task],
996 ) -> BTreeSet<String> {
997 stalled
998 .iter()
999 .chain(finished)
1000 .filter(|t| t.status == TaskStatus::Held)
1001 .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1002 .map(|t| t.id.clone())
1003 .collect()
1004 }
1005
1006 fn snapshot_with(
1007 considered: &BTreeMap<String, String>,
1008 queue: &Queue,
1009 stalled: &[Task],
1010 finished: &[Task],
1011 ) -> (u64, BTreeSet<String>) {
1012 let ids = stalled
1013 .iter()
1014 .chain(finished)
1015 .map(|t| t.id.clone())
1016 .collect();
1017 let skip = Self::settled(considered, stalled, finished);
1018 (queue.revision_excluding(&skip), ids)
1019 }
1020
1021 fn snapshot(
1022 &self,
1023 queue: &Queue,
1024 stalled: &[Task],
1025 finished: &[Task],
1026 ) -> (u64, BTreeSet<String>) {
1027 Self::snapshot_with(&self.considered, queue, stalled, finished)
1028 }
1029
1030 #[must_use]
1047 pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1048 self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1049 }
1050
1051 #[allow(clippy::too_many_arguments)]
1055 pub async fn maybe_run(
1056 &mut self,
1057 cfg: &Config,
1058 repo: &Path,
1059 queue: &Queue,
1060 questions: &Questions,
1061 home: &Path,
1062 queued: &[Task],
1063 stalled: &[Task],
1064 finished: &[Task],
1065 max_attempts: usize,
1066 ) {
1067 if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1068 return;
1069 }
1070 self.considered = stalled
1073 .iter()
1074 .chain(finished)
1075 .filter(|t| t.status == TaskStatus::Held)
1076 .map(|t| (t.id.clone(), input_fingerprint(t)))
1077 .collect();
1078 self.last_seen = Some(self.snapshot(queue, stalled, finished));
1079 if let Err(e) = self
1080 .run_once(
1081 cfg,
1082 repo,
1083 queue,
1084 questions,
1085 home,
1086 queued,
1087 stalled,
1088 finished,
1089 max_attempts,
1090 )
1091 .await
1092 {
1093 tracing::warn!("conductor: {e:#}");
1099 }
1100 }
1101
1102 #[allow(clippy::too_many_arguments)]
1103 async fn run_once(
1104 &mut self,
1105 cfg: &Config,
1106 repo: &Path,
1107 queue: &Queue,
1108 questions: &Questions,
1109 home: &Path,
1110 queued: &[Task],
1111 stalled: &[Task],
1112 finished: &[Task],
1113 max_attempts: usize,
1114 ) -> Result<()> {
1115 if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1116 return Ok(());
1117 }
1118
1119 let primary = cfg
1120 .resolve_roles()
1121 .context("resolving the conductor seat")?
1122 .conductor;
1123 let chain = match cfg.roles.conductor.as_ref() {
1127 Some(c) if c.ids().len() > 1 => {
1128 agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1129 }
1130 _ => vec![primary],
1131 };
1132
1133 let runnable_views: Vec<prompt::ConductTask> =
1134 queued.iter().map(|t| view(t, max_attempts)).collect();
1135 let stalled_views: Vec<prompt::ConductTask> =
1136 stalled.iter().map(|t| view(t, max_attempts)).collect();
1137 let mut finished_views = Vec::with_capacity(finished.len());
1138 for t in finished {
1139 finished_views.push(finished_view(t, repo, max_attempts).await);
1140 }
1141
1142 let body = prompt::with_overlay(
1143 prompt::conduct(
1144 &runnable_views,
1145 &stalled_views,
1146 &finished_views,
1147 &cfg.graph.language,
1148 ),
1149 cfg.prompts.overlay(NODE),
1150 );
1151
1152 let body = format!(
1153 "{body}\n\n{}",
1154 crate::queue::dependency_facts(queue, questions)
1155 );
1156 let artifacts = home.join("conduct").join("artifacts");
1157 let cache_dir = cfg.cache_dir();
1160
1161 let busy = Busy::mark(home);
1163 let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1164 for spec in &chain {
1165 let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1168 if needs_new_seat {
1169 self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1170 }
1171 let seat = self.seat.as_mut().expect("just ensured a seat exists");
1172 let stem = if std::ptr::eq(spec, &chain[0]) {
1175 format!("turn-{}", seat.turns + 1)
1176 } else {
1177 format!("turn-{}-{}", seat.turns + 1, spec.id)
1178 };
1179 let inv = Invocation {
1180 cwd: repo,
1181 prompt: &body,
1182 timeout: TURN_TIMEOUT,
1183 allow_write: false,
1186 unsandboxed: false,
1187 sessions: cfg.graph.sessions,
1188 artifacts: &artifacts,
1189 stem: &stem,
1190 run: NODE,
1191 node: NODE,
1192 cache_dir: cache_dir.as_deref(),
1193 attachments: &[],
1194 writable: &[],
1195 };
1196 let out = agent::invoke(spec, seat, &inv).await;
1197 if let Ok(body) = serde_json::to_string(&*seat) {
1200 let path = seat_path(home);
1201 if let Some(dir) = path.parent() {
1202 let _ = std::fs::create_dir_all(dir);
1203 }
1204 let _ = std::fs::write(path, body);
1205 }
1206 let advance = agent::chain_advances(&out);
1207 result = match out {
1208 Err(e) => Err(e.context("invoking the conductor")),
1209 Ok(out) if advance => Err(anyhow::anyhow!(
1210 "no usable reply (exit {:?}, timed out {})",
1211 out.exit_code,
1212 out.timed_out
1213 )),
1214 Ok(out) => verdict::extract_json(&out.text)
1215 .context("the conductor's reply could not be parsed"),
1216 };
1217 if result.is_ok() {
1218 break;
1219 }
1220 if chain.len() > 1 {
1221 tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1222 }
1223 }
1224 drop(busy);
1225 let mut verdict = result?;
1226 attach_facts(cfg, repo, queue, &mut verdict).await;
1227 for d in &verdict.decisions {
1228 let snapshot = queued
1229 .iter()
1230 .chain(stalled)
1231 .chain(finished)
1232 .find(|t| t.id == d.id);
1233 if let Some(snapshot) = snapshot {
1234 if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1235 tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1236 }
1237 }
1238 }
1239 Ok(())
1240 }
1241}
1242
1243#[cfg(test)]
1244mod tests {
1245 use std::collections::BTreeMap;
1246
1247 use tempfile::tempdir;
1248
1249 use super::*;
1250 use crate::ask::{Answer, QuestionStatus};
1251 use crate::config::{AgentKind, AgentSpec, Graph};
1252 use crate::queue::Source;
1253
1254 fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1255 let path = dir.join("mock-conduct-agent.sh");
1256 std::fs::write(&path, script).expect("write mock");
1257 AgentSpec {
1258 id: "mock".to_owned(),
1259 kind: AgentKind::Command,
1260 model: None,
1261 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1262 extra_args: Vec::new(),
1263 env,
1264 prompt_delivery: None,
1265 }
1266 }
1267
1268 fn config(spec: AgentSpec) -> Config {
1269 Config {
1270 agents: vec![spec],
1271 graph: Graph {
1272 language: "en".to_owned(),
1273 ..Graph::default()
1274 },
1275 ..Config::default()
1276 }
1277 }
1278
1279 fn task(title: &str) -> Task {
1280 Task::new(
1281 title.to_owned(),
1282 format!("do {title}"),
1283 std::path::PathBuf::from("."),
1284 Source::Human,
1285 )
1286 }
1287
1288 const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1289 const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1290
1291 fn env(reply: &str) -> BTreeMap<String, String> {
1292 BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1293 }
1294
1295 const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1296
1297 fn init_repo_with_branch(dir: &Path, branch: &str) {
1301 use crate::proc::Quiet as _;
1302 let run = |args: &[&str]| {
1303 let out = std::process::Command::new("git")
1304 .args(args)
1305 .current_dir(dir)
1306 .quiet()
1307 .output()
1308 .expect("spawn git");
1309 assert!(
1310 out.status.success(),
1311 "git {args:?} failed: {}",
1312 String::from_utf8_lossy(&out.stderr)
1313 );
1314 };
1315 run(&["init", "-b", "main"]);
1316 run(&["config", "user.name", "magi test"]);
1317 run(&["config", "user.email", "magi@example.com"]);
1318 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1319 run(&["add", "-A"]);
1320 run(&["commit", "-m", "init"]);
1321 run(&["checkout", "-b", branch]);
1322 std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1323 run(&["add", "-A"]);
1324 run(&["commit", "-m", "candidate work"]);
1325 }
1326
1327 fn review_round_with_finding(
1328 round: usize,
1329 finding_id: &str,
1330 title: &str,
1331 addressed: &[&str],
1332 rejected: &[(&str, &str)],
1333 ) -> crate::run::ReviewRound {
1334 crate::run::ReviewRound {
1335 round,
1336 head: "deadbeef".to_owned(),
1337 verified_head: None,
1338 verified_at: None,
1339 reviews: vec![crate::run::ReviewRecord {
1340 attempts: 0,
1341 reviewer: 1,
1342 agent: "mock".to_owned(),
1343 summary: String::new(),
1344 findings: vec![crate::verdict::Finding {
1345 id: finding_id.to_owned(),
1346 severity: crate::verdict::Severity::Major,
1347 file: None,
1348 line: None,
1349 title: title.to_owned(),
1350 detail: String::new(),
1351 }],
1352 vote: None,
1353 failed: None,
1354 duration_ms: 0,
1355 }],
1356 e2e: Vec::new(),
1357 verify_retried: false,
1358 e2e_deferred: false,
1359 e2e_defer_reason: None,
1360 fix: Some(crate::run::FixRecord {
1361 agent: "mock".to_owned(),
1362 addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1363 rejected: rejected
1364 .iter()
1365 .map(|(id, why)| crate::verdict::Rejection {
1366 id: (*id).to_owned(),
1367 why: (*why).to_owned(),
1368 })
1369 .collect(),
1370 notes: String::new(),
1371 committed: false,
1372 failed: None,
1373 duration_ms: 0,
1374 continuation: None,
1375 }),
1376 blocking: 1,
1377 answered: 1,
1378 expected: 1,
1379 clean: false,
1380 progressed: true,
1381 vote_split: false,
1382 reconsideration: Vec::new(),
1383 verdict: None,
1384 }
1385 }
1386
1387 #[test]
1388 fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1389 crate::run::pin_test_home();
1390 let dir = tempdir().unwrap();
1391 let queue = Queue::at(dir.path().join("queue"));
1392 let questions = Questions::at(dir.path().join("questions"));
1393 let mut t = task("recover");
1394 t.answers.push(crate::queue::AnsweredQuestion {
1395 question: "Inspect remote first?".into(),
1396 answer: "Inspect, then await resumption instructions".into(),
1397 });
1398 t.hold_machine(Some("waiting for remote inspection".into()));
1399 queue.put(&mut t).unwrap();
1400 let snapshot = queue.get(&t.id).unwrap();
1401 t.edit(
1402 "recover".into(),
1403 "Reuse the measured PR head; fix Clippy; review".into(),
1404 )
1405 .unwrap();
1406 t.release_operator("cli", None);
1407 queue.put(&mut t).unwrap();
1408 let approval = t.resumption.clone();
1409 let d = Decision {
1410 id: t.id.clone(),
1411 recovery: Some(Recovery::Hold),
1412 reason: Some("remote inspection still required".into()),
1413 ..Decision::default()
1414 };
1415 apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1416 let stale_question = Decision {
1417 question: Some("Inspect remote?".into()),
1418 ..d.clone()
1419 };
1420 apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1421 assert!(questions.list().is_empty());
1422 for reason in [
1423 "waiting for remote inspection",
1424 "inspection has not been confirmed",
1425 ] {
1426 let repeated = Decision {
1427 reason: Some(reason.into()),
1428 ..d.clone()
1429 };
1430 apply(
1431 &queue,
1432 &questions,
1433 &Verdict {
1434 decisions: vec![repeated],
1435 },
1436 )
1437 .unwrap();
1438 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1439 }
1440 let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1441 assert_eq!(reloaded.resumption, approval);
1442 assert_eq!(reloaded.answers, t.answers);
1443 let v = view(&reloaded, 3);
1444 assert!(v.operator_resume.unwrap().contains("superseded"));
1445 let mut dep = task("new prerequisite");
1446 dep.hold_machine(Some("new authentication failure".into()));
1447 queue.put(&mut dep).unwrap();
1448 let new_blocker = Decision {
1449 blocked_by: vec![dep.id.clone()],
1450 evidence: vec![dep.id.clone()],
1451 recovery: None,
1453 reason: Some("new prerequisite failed".into()),
1454 ..d.clone()
1455 };
1456 apply(
1457 &queue,
1458 &questions,
1459 &Verdict {
1460 decisions: vec![new_blocker.clone()],
1461 },
1462 )
1463 .unwrap();
1464 let mut after = queue.get(&t.id).unwrap();
1465 assert_eq!(after.status, TaskStatus::Blocked);
1466 after.release();
1469 queue.put(&mut after).unwrap();
1470 apply(
1471 &queue,
1472 &questions,
1473 &Verdict {
1474 decisions: vec![new_blocker],
1475 },
1476 )
1477 .unwrap();
1478 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1479 after.start("new-run".into());
1480 assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1481 after.hold_manual(Some("operator says wait".into()));
1482 queue.put(&mut after).unwrap();
1483 apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1484 assert!(queue.get(&t.id).unwrap().operator_held());
1485 }
1486
1487 #[test]
1488 fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1489 crate::run::pin_test_home();
1490 let dir = tempdir().unwrap();
1491 let queue = Queue::at(dir.path().join("queue"));
1492 let questions = Questions::at(dir.path().join("questions"));
1493 let mut t = task("resume evidence");
1494 let mut run = RunState::new(
1495 t.repo.clone(),
1496 "main".into(),
1497 "base".into(),
1498 "work".into(),
1499 crate::config::Config::default(),
1500 );
1501 run.begin_pass();
1502 run.status = crate::run::RunStatus::Failed;
1503 run.save().unwrap();
1504 t.start(run.id.clone());
1505 t.runs.push(run.id.clone());
1507 t.release_operator("cli", None);
1508 let approval = t.resumption.as_ref().unwrap().at;
1509 let decision = Decision {
1510 evidence: vec![run.id.clone()],
1511 ..Decision::default()
1512 };
1513 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1514 t.start(run.id.clone());
1515 assert_eq!(t.runs.len(), 1);
1516 run.begin_pass();
1517 run.save().unwrap();
1518 assert!(run.passes.last().unwrap().started_at > approval);
1519 let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1520 assert_eq!(evidence.len(), 1);
1521 t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1522 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1523 run.begin_pass();
1524 run.save().unwrap();
1525 assert_eq!(
1526 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1527 1
1528 );
1529
1530 let mut q = Question::new(
1531 run.id.clone(),
1532 "review".into(),
1533 "check".into(),
1534 "New failure".into(),
1535 String::new(),
1536 Vec::new(),
1537 );
1538 questions.put(&mut q).unwrap();
1539 let decision = Decision {
1540 evidence: vec![q.id.clone()],
1541 ..Decision::default()
1542 };
1543 assert_eq!(
1544 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1545 1
1546 );
1547 q.asked_at = approval;
1548 questions.put(&mut q).unwrap();
1549 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1550 q.asked_at = jiff::Timestamp::now();
1551 q.run = "unrelated".into();
1552 questions.put(&mut q).unwrap();
1553 assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1554
1555 let mut fresh = RunState::new(
1556 t.repo.clone(),
1557 "main".into(),
1558 "base".into(),
1559 "fresh work".into(),
1560 crate::config::Config::default(),
1561 );
1562 fresh.status = crate::run::RunStatus::Failed;
1563 fresh.save().unwrap();
1564 t.start(fresh.id.clone());
1565 let decision = Decision {
1566 evidence: vec![fresh.id.clone()],
1567 ..Decision::default()
1568 };
1569 assert_eq!(
1570 new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1571 1
1572 );
1573 }
1574
1575 #[test]
1576 fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1577 crate::run::pin_test_home();
1578 let dir = tempdir().unwrap();
1579 let default_repo = dir.path().join("default");
1580 let task_repo = dir.path().join("task");
1581 std::fs::create_dir_all(&default_repo).unwrap();
1582 std::fs::create_dir_all(&task_repo).unwrap();
1583 init_repo_with_branch(&default_repo, "other-branch");
1584 init_repo_with_branch(&task_repo, "magi/f00d/A");
1585
1586 let mut config = Config::default();
1587 config.graph.review_rounds = 6;
1588 let mut state = crate::run::RunState::new(
1589 task_repo.clone(),
1590 "main".to_owned(),
1591 "deadbeef".to_owned(),
1592 "task".to_owned(),
1593 config,
1594 );
1595 state.status = crate::run::RunStatus::Blocked;
1596 state.candidates.push(crate::run::Candidate {
1597 index: 0,
1598 label: 'A',
1599 agent: "mock".to_owned(),
1600 branch: "magi/f00d/A".to_owned(),
1601 worktree: task_repo.clone(),
1602 summary: String::new(),
1603 stat: String::new(),
1604 files: 1,
1605 commits: 1,
1606 empty: false,
1607 failed: None,
1608 verified_noop: None,
1609 duration_ms: 0,
1610 folded: false,
1611 });
1612 state.tally = Some(crate::run::Tally {
1613 first_choice: std::collections::BTreeMap::new(),
1614 borda: std::collections::BTreeMap::new(),
1615 winner: 'A',
1616 rankings: 0,
1617 unanimous_initial: false,
1618 deliberated: false,
1619 changed_votes: 0,
1620 unanimous_final: false,
1621 tie_break: None,
1622 judges: 0,
1623 present: 0,
1624 quorum: 0,
1625 met_quorum: true,
1626 uncontested: Some("solo".to_owned()),
1627 });
1628 state.reviews = vec![
1629 review_round_with_finding(
1630 1,
1631 "R1-1-2",
1632 "answer content is dropped",
1633 &[],
1634 &[("R1-1-2", "the id leaving blocked_by is enough")],
1635 ),
1636 review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1637 ];
1638 state.save().unwrap();
1639
1640 let mut t = task("outcome test");
1641 t.repo = task_repo;
1642 t.runs.push(state.id.clone());
1643
1644 let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1645 let outcome = finished.outcome;
1646
1647 assert!(outcome.unreadable.is_none());
1648 assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1649 assert_eq!(outcome.rounds_used, 2);
1650 assert_eq!(outcome.rounds_max, 6);
1651 assert_eq!(outcome.rounds.len(), 2);
1652 assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1653 assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1654 assert!(outcome.rounds[1].addressed.is_empty());
1655 assert!(outcome.rounds[1].rejected.is_empty());
1656 assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1657 assert!(
1658 outcome.branch_head.is_some(),
1659 "a real branch must resolve a head commit: {outcome:?}"
1660 );
1661 }
1662
1663 fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1667 tokio::runtime::Builder::new_current_thread()
1668 .enable_all()
1669 .build()
1670 .unwrap()
1671 .block_on(f)
1672 }
1673
1674 #[test]
1675 fn view_resolves_task_attempt_limit_before_daemon_default() {
1676 let mut t = task("budget");
1677 assert_eq!(view(&t, 7).max_attempts, 7);
1678 t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1679 .unwrap();
1680 assert_eq!(view(&t, 7).max_attempts, 3);
1681 t.change_attempt_limit(None).unwrap();
1682 assert_eq!(view(&t, 7).max_attempts, 7);
1683 }
1684
1685 #[test]
1686 fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1687 let mut t = task("answered");
1688 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1689 let v = view(&t, 2);
1690 assert_eq!(v.answers.len(), 1);
1691 assert_eq!(v.answers[0].question, "Which backend?");
1692 assert_eq!(v.answers[0].answer, "SQLite");
1693 }
1694
1695 #[tokio::test]
1696 async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1697 for held in [false, true] {
1698 for reference in ["ca5b", "20261013-193611-ca5b"] {
1699 for question in [false, true] {
1700 let dir = tempdir().unwrap();
1701 let queue = Queue::at(dir.path().join("queue"));
1702 let questions = Questions::at(dir.path().join("questions"));
1703 let mut dep = task("prerequisite");
1704 dep.id = "20261013-193611-ca5b".into();
1705 dep.start("existing-run".into());
1706 queue.put(&mut dep).unwrap();
1707 let mut waiting = task("waiting");
1708 if held {
1709 waiting.hold_machine(Some("dependency could not be located".into()));
1710 }
1711 queue.put(&mut waiting).unwrap();
1712 let missing_reference = if reference == "ca5b" {
1713 dep.id.as_str()
1714 } else {
1715 "ca5b"
1716 };
1717 let reply = serde_json::json!({"decisions": [{
1718 "id": waiting.id, "blocked_by": [reference],
1719 "missing_dependencies": [missing_reference],
1720 "question": question.then_some("ca5b is missing; where is its replacement?"),
1721 "recovery": (!question).then_some("hold"),
1722 "reason": "ca5b is missing"
1723 }]}).to_string();
1724 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1725 let (queued, finished) = if held {
1726 (vec![], vec![waiting.clone()])
1727 } else {
1728 (vec![waiting.clone()], vec![])
1729 };
1730 Conductor::new()
1731 .run_once(
1732 &cfg,
1733 dir.path(),
1734 &queue,
1735 &questions,
1736 dir.path(),
1737 &queued,
1738 &[],
1739 &finished,
1740 2,
1741 )
1742 .await
1743 .unwrap();
1744 let after = queue.get(&waiting.id).unwrap();
1745 assert_eq!(after.status, TaskStatus::Blocked);
1746 assert_eq!(after.blocked_by, [dep.id]);
1747 assert_eq!(after.blocked_from, Some(waiting.status));
1748 assert!(!after.block_reason.unwrap().contains("missing"));
1749 assert!(questions.list().is_empty());
1750 assert_eq!(after.instruction, waiting.instruction);
1751 }
1752 }
1753 }
1754 }
1755
1756 #[tokio::test]
1757 async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
1758 for held in [false, true] {
1759 for unknown in [false, true] {
1760 for question in [false, true] {
1761 let dir = tempdir().unwrap();
1762 let queue = Queue::at(dir.path().join("queue"));
1763 let questions = Questions::at(dir.path().join("questions"));
1764 let mut dep = task("running prerequisite");
1765 dep.id = "20261013-193611-ca5b".into();
1766 dep.start("existing-run".into());
1767 queue.put(&mut dep).unwrap();
1768 let reference = if unknown {
1769 std::fs::write(questions.root(), "not a directory").unwrap();
1772 dep.id.clone()
1773 } else {
1774 let mut other = task("another prerequisite");
1775 other.id = "20261014-193611-ca5b".into();
1776 queue.put(&mut other).unwrap();
1777 "ca5b".to_owned()
1778 };
1779 assert_eq!(
1780 crate::queue::dependency(&queue, &questions, &reference),
1781 if unknown {
1782 crate::queue::Dependency::Unknown
1783 } else {
1784 crate::queue::Dependency::Ambiguous
1785 }
1786 );
1787 let mut waiting = task("waiting");
1788 if held {
1789 waiting.hold_machine(Some("dependency could not be located".into()));
1790 }
1791 queue.put(&mut waiting).unwrap();
1792 let reply = serde_json::json!({"decisions": [{
1793 "id": waiting.id, "blocked_by": [reference],
1794 "missing_dependencies": [reference],
1795 "question": question.then_some("ca5b is missing"),
1796 "recovery": (!question).then_some("hold"),
1797 "reason": "ca5b is missing"
1798 }]})
1799 .to_string();
1800 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1801 let (queued, finished) = if held {
1802 (vec![], vec![waiting.clone()])
1803 } else {
1804 (vec![waiting.clone()], vec![])
1805 };
1806 Conductor::new()
1807 .run_once(
1808 &cfg,
1809 dir.path(),
1810 &queue,
1811 &questions,
1812 dir.path(),
1813 &queued,
1814 &[],
1815 &finished,
1816 2,
1817 )
1818 .await
1819 .unwrap();
1820 let after = queue.get(&waiting.id).unwrap();
1821 assert_eq!(after.status, TaskStatus::Blocked);
1822 assert_eq!(after.blocked_by, [reference]);
1823 assert_eq!(after.blocked_from, Some(waiting.status));
1824 assert!(!after.block_reason.unwrap().contains("missing"));
1825 assert_eq!(after.instruction, waiting.instruction);
1826 if unknown {
1827 assert!(questions.root().is_file());
1828 } else {
1829 assert!(questions.list().is_empty());
1830 }
1831 }
1832 }
1833 }
1834 }
1835
1836 #[tokio::test]
1837 async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
1838 for held in [false, true] {
1839 for unknown in [false, true] {
1840 for question in [false, true] {
1841 for short_blocker in [false, true] {
1842 let dir = tempdir().unwrap();
1843 let queue = Queue::at(dir.path().join("queue"));
1844 let questions = Questions::at(dir.path().join("questions"));
1845 let mut dep = task("running prerequisite");
1846 dep.id = "20261013-193611-ca5b".into();
1847 dep.start("existing-run".into());
1848 queue.put(&mut dep).unwrap();
1849 if unknown {
1850 std::fs::write(questions.root(), "not a directory").unwrap();
1851 } else {
1852 let mut other = task("another prerequisite");
1853 other.id = "20261014-193611-ca5b".into();
1854 queue.put(&mut other).unwrap();
1855 }
1856 let (blocker, missing) = if short_blocker {
1857 ("ca5b", dep.id.as_str())
1858 } else {
1859 (dep.id.as_str(), "ca5b")
1860 };
1861 let mut waiting = task("waiting");
1862 if held {
1863 waiting.hold_machine(Some("dependency could not be located".into()));
1864 }
1865 queue.put(&mut waiting).unwrap();
1866 let reply = serde_json::json!({"decisions": [{
1867 "id": waiting.id, "blocked_by": [blocker],
1868 "missing_dependencies": [missing],
1869 "question": question.then_some("ca5b is missing"),
1870 "recovery": (!question).then_some("hold"),
1871 "reason": "ca5b is missing"
1872 }]})
1873 .to_string();
1874 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1875 let (queued, finished) = if held {
1876 (vec![], vec![waiting.clone()])
1877 } else {
1878 (vec![waiting.clone()], vec![])
1879 };
1880 Conductor::new()
1881 .run_once(
1882 &cfg,
1883 dir.path(),
1884 &queue,
1885 &questions,
1886 dir.path(),
1887 &queued,
1888 &[],
1889 &finished,
1890 2,
1891 )
1892 .await
1893 .unwrap();
1894 assert_eq!(
1895 serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
1896 serde_json::to_value(&waiting).unwrap(),
1897 "unverifiable aliases must leave the entire task unchanged"
1898 );
1899 if unknown {
1900 assert!(questions.root().is_file());
1901 } else {
1902 assert!(questions.list().is_empty());
1903 }
1904 }
1905 }
1906 }
1907 }
1908 }
1909
1910 #[tokio::test]
1911 async fn dependency_resolution_genuine_missing_reference_still_asks() {
1912 let dir = tempdir().unwrap();
1913 let queue = Queue::at(dir.path().join("queue"));
1914 let questions = Questions::at(dir.path().join("questions"));
1915 let mut waiting = task("missing prerequisite");
1916 queue.put(&mut waiting).unwrap();
1917 let reply = serde_json::json!({"decisions": [{
1918 "id": waiting.id, "blocked_by": ["not-present"],
1919 "missing_dependencies": ["not-present"],
1920 "question": "Which prerequisite replaces the missing task?"
1921 }]})
1922 .to_string();
1923 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1924 Conductor::new()
1925 .run_once(
1926 &cfg,
1927 dir.path(),
1928 &queue,
1929 &questions,
1930 dir.path(),
1931 &[waiting.clone()],
1932 &[],
1933 &[],
1934 2,
1935 )
1936 .await
1937 .unwrap();
1938 let after = queue.get(&waiting.id).unwrap();
1939 assert_eq!(after.status, TaskStatus::Blocked);
1940 assert_eq!(
1941 questions.get(&after.blocked_by[0]).unwrap().summary,
1942 "Which prerequisite replaces the missing task?"
1943 );
1944 }
1945
1946 #[tokio::test]
1947 async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
1948 let dir = tempdir().unwrap();
1949 let queue = Queue::at(dir.path().join("queue"));
1950 let questions = Questions::at(dir.path().join("questions"));
1951 let mut dep = task("prerequisite");
1952 dep.id = "20261013-193611-ca5b".into();
1953 dep.start("existing-run".into());
1954 queue.put(&mut dep).unwrap();
1955 let mut unrelated = task("unrelated work");
1956 unrelated.start("other-run".into());
1957 queue.put(&mut unrelated).unwrap();
1958 std::fs::File::options()
1961 .write(true)
1962 .open(queue.path_of(&unrelated.id))
1963 .unwrap()
1964 .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
1965 .unwrap();
1966 unrelated.succeed();
1967 let mut waiting = task("waiting");
1968 let mut asking = task("needs a product decision");
1969 let mut holding = task("must stay held");
1970 queue.put(&mut waiting).unwrap();
1971 queue.put(&mut asking).unwrap();
1972 queue.put(&mut holding).unwrap();
1973 let reply = serde_json::json!({"decisions": [
1974 {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
1975 {"id": asking.id, "question": "Which behavior do you want?"},
1976 {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
1977 ]})
1978 .to_string();
1979 let mut vars = env(&reply);
1980 vars.insert(
1981 "UNRELATED_PATH".into(),
1982 queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
1983 );
1984 vars.insert(
1985 "UNRELATED_RECORD".into(),
1986 serde_json::to_string(&unrelated).unwrap(),
1987 );
1988 let script = r#"#!/bin/sh
1989cat >/dev/null
1990printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
1991printf '%s\n' "$MOCK_REPLY"
1992"#;
1993 let cfg = config(mock_agent(dir.path(), script, vars));
1994 Conductor::new()
1995 .run_once(
1996 &cfg,
1997 dir.path(),
1998 &queue,
1999 &questions,
2000 dir.path(),
2001 &[waiting.clone(), asking.clone(), holding.clone()],
2002 &[],
2003 &[],
2004 2,
2005 )
2006 .await
2007 .unwrap();
2008 assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2009 let after = queue.get(&waiting.id).unwrap();
2010 assert_eq!(after.status, TaskStatus::Blocked);
2011 assert_eq!(after.blocked_by, [dep.id]);
2012 let asked = queue.get(&asking.id).unwrap();
2013 assert_eq!(asked.status, TaskStatus::Blocked);
2014 assert_eq!(
2015 questions.get(&asked.blocked_by[0]).unwrap().summary,
2016 "Which behavior do you want?"
2017 );
2018 assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2019 }
2020
2021 #[test]
2022 fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2023 let dir = tempdir().unwrap();
2024 let queue = Queue::at(dir.path().join("queue"));
2025 let questions = Questions::at(dir.path().join("questions"));
2026 let mut a = task("a");
2027 a.priority = 9;
2028 queue.put(&mut a).unwrap();
2029
2030 let verdict = Verdict {
2031 decisions: vec![Decision {
2032 id: a.id.clone(),
2033 blocked_by: vec!["20260101-000000-dead".to_owned()],
2034 reason: Some("waits on the other task".to_owned()),
2035 recovery: None,
2036 question: None,
2037 choices: Vec::new(),
2038 evidence: Vec::new(),
2039 missing_dependencies: Vec::new(),
2040 }],
2041 };
2042 apply(&queue, &questions, &verdict).unwrap();
2043
2044 let back = queue.get(&a.id).unwrap();
2045 assert_eq!(back.status, TaskStatus::Blocked);
2046 assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2047 assert_eq!(
2048 back.priority, 9,
2049 "the conductor's reply cannot carry priority"
2050 );
2051 }
2052
2053 #[test]
2054 fn a_question_decision_files_one_and_blocks_on_its_id() {
2055 let dir = tempdir().unwrap();
2056 let queue = Queue::at(dir.path().join("queue"));
2057 let questions = Questions::at(dir.path().join("questions"));
2058 let mut t = task("ambiguous");
2059 queue.put(&mut t).unwrap();
2060
2061 let verdict = Verdict {
2062 decisions: vec![Decision {
2063 id: t.id.clone(),
2064 blocked_by: Vec::new(),
2065 reason: Some("which backend?".to_owned()),
2066 recovery: None,
2067 question: Some("Which storage backend?".to_owned()),
2068 choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2069 evidence: Vec::new(),
2070 missing_dependencies: Vec::new(),
2071 }],
2072 };
2073 apply(&queue, &questions, &verdict).unwrap();
2074
2075 let back = queue.get(&t.id).unwrap();
2076 assert_eq!(back.status, TaskStatus::Blocked);
2077 assert_eq!(back.blocked_by.len(), 1);
2078 let q = questions.get(&back.blocked_by[0]).unwrap();
2079 assert_eq!(q.summary, "Which storage backend?");
2080 assert_eq!(q.node, NODE);
2081 assert!(q.status.open());
2082 }
2083
2084 #[test]
2085 fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2086 let dir = tempdir().unwrap();
2087 let queue = Queue::at(dir.path().join("queue"));
2088 let questions = Questions::at(dir.path().join("questions"));
2089 let mut t = task("asked once");
2090 queue.put(&mut t).unwrap();
2091
2092 let decision = Decision {
2093 id: t.id.clone(),
2094 reason: Some("still deciding".to_owned()),
2095 question: Some("Which backend?".to_owned()),
2096 ..Decision::default()
2097 };
2098 apply(
2099 &queue,
2100 &questions,
2101 &Verdict {
2102 decisions: vec![decision.clone()],
2103 },
2104 )
2105 .unwrap();
2106 assert_eq!(questions.list().len(), 1);
2107 let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2108
2109 let mut released = queue.get(&t.id).unwrap();
2114 released.release();
2115 queue.put(&mut released).unwrap();
2116
2117 apply(
2118 &queue,
2119 &questions,
2120 &Verdict {
2121 decisions: vec![decision],
2122 },
2123 )
2124 .unwrap();
2125 assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2126 let after = queue.get(&t.id).unwrap();
2127 assert_eq!(
2128 after.blocked_by,
2129 [first_question_id],
2130 "the existing open question is reused, not replaced"
2131 );
2132 }
2133
2134 #[test]
2135 fn a_same_id_question_from_another_node_is_not_reused() {
2136 let dir = tempdir().unwrap();
2137 let queue = Queue::at(dir.path().join("queue"));
2138 let questions = Questions::at(dir.path().join("questions"));
2139 let mut t = task("must ask the conductor");
2140 queue.put(&mut t).unwrap();
2141
2142 let mut unrelated = Question::new(
2143 t.id.clone(),
2144 "review".to_owned(),
2145 "reviewer-1".to_owned(),
2146 "An unrelated review question".to_owned(),
2147 String::new(),
2148 Vec::new(),
2149 );
2150 questions.put(&mut unrelated).unwrap();
2151
2152 apply(
2153 &queue,
2154 &questions,
2155 &Verdict {
2156 decisions: vec![Decision {
2157 id: t.id.clone(),
2158 question: Some("Which backend?".to_owned()),
2159 ..Decision::default()
2160 }],
2161 },
2162 )
2163 .unwrap();
2164
2165 let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2166 assert_eq!(blocked_by.len(), 1);
2167 assert_ne!(blocked_by[0], unrelated.id);
2168 assert!(questions.get(&unrelated.id).unwrap().status.open());
2169 assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2170 }
2171
2172 #[test]
2173 fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2174 let dir = tempdir().unwrap();
2175 let queue = Queue::at(dir.path().join("queue"));
2176 let questions = Questions::at(dir.path().join("questions"));
2177 let mut t = task("waits on an answer");
2178 queue.put(&mut t).unwrap();
2179
2180 apply(
2181 &queue,
2182 &questions,
2183 &Verdict {
2184 decisions: vec![Decision {
2185 id: t.id.clone(),
2186 blocked_by: Vec::new(),
2187 reason: None,
2188 recovery: None,
2189 question: Some("Which backend?".to_owned()),
2190 choices: Vec::new(),
2191 evidence: Vec::new(),
2192 missing_dependencies: Vec::new(),
2193 }],
2194 },
2195 )
2196 .unwrap();
2197 let blocked = queue.get(&t.id).unwrap();
2198 let question_id = blocked.blocked_by[0].clone();
2199
2200 let mut q = questions.get(&question_id).unwrap();
2201 q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2202 questions.put(&mut q).unwrap();
2203 assert_eq!(q.status, QuestionStatus::Answered);
2204
2205 let mut task_after = queue.get(&t.id).unwrap();
2209 task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2210 task_after.unblock(&question_id);
2211 assert_eq!(task_after.status, TaskStatus::Queued);
2212 assert_eq!(task_after.answers[0].answer, "SQLite");
2213 }
2214
2215 #[test]
2216 fn a_stalled_task_can_be_requeued_or_held() {
2217 let dir = tempdir().unwrap();
2218 let queue = Queue::at(dir.path().join("queue"));
2219 let questions = Questions::at(dir.path().join("questions"));
2220
2221 let mut requeue_me = task("stuck a");
2222 requeue_me.start("run-1".to_owned());
2223 queue.put(&mut requeue_me).unwrap();
2224
2225 let mut hold_me = task("stuck b");
2226 hold_me.start("run-2".to_owned());
2227 queue.put(&mut hold_me).unwrap();
2228
2229 apply(
2230 &queue,
2231 &questions,
2232 &Verdict {
2233 decisions: vec![
2234 Decision {
2235 id: requeue_me.id.clone(),
2236 recovery: Some(Recovery::Requeue),
2237 ..Decision::default()
2238 },
2239 Decision {
2240 id: hold_me.id.clone(),
2241 recovery: Some(Recovery::Hold),
2242 reason: Some("looks broken".to_owned()),
2243 ..Decision::default()
2244 },
2245 ],
2246 },
2247 )
2248 .unwrap();
2249
2250 let requeued = queue.get(&requeue_me.id).unwrap();
2251 assert_eq!(requeued.status, TaskStatus::Queued);
2252 assert_eq!(requeued.attempts, 0);
2253
2254 let held = queue.get(&hold_me.id).unwrap();
2255 assert_eq!(held.status, TaskStatus::Held);
2256 assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2257 }
2258
2259 #[test]
2260 fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2261 let dir = tempdir().unwrap();
2266 let queue = Queue::at(dir.path().join("queue"));
2267 let questions = Questions::at(dir.path().join("questions"));
2268 let mut t = task("held out of attempts");
2269 t.hold_machine(Some("out of attempts".to_owned()));
2270 queue.put(&mut t).unwrap();
2271
2272 apply(
2273 &queue,
2274 &questions,
2275 &Verdict {
2276 decisions: vec![Decision {
2277 id: t.id.clone(),
2278 reason: Some("what should happen to this one?".to_owned()),
2279 question: Some("Hold it, or try again?".to_owned()),
2280 ..Decision::default()
2281 }],
2282 },
2283 )
2284 .unwrap();
2285 let blocked = queue.get(&t.id).unwrap();
2286 assert_eq!(blocked.status, TaskStatus::Blocked);
2287 let question_id = blocked.blocked_by[0].clone();
2288
2289 let mut q = questions.get(&question_id).unwrap();
2290 q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2291 questions.put(&mut q).unwrap();
2292
2293 let mut after = queue.get(&t.id).unwrap();
2295 after.record_answer(q.summary.clone(), "leave it held".to_owned());
2296 after.unblock(&question_id);
2297 assert_eq!(
2298 after.status,
2299 TaskStatus::Held,
2300 "must not fall back to queued"
2301 );
2302 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2303 }
2304
2305 #[test]
2306 fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2307 let dir = tempdir().unwrap();
2317 let queue = Queue::at(dir.path().join("queue"));
2318 let questions = Questions::at(dir.path().join("questions"));
2319 let mut t = task("disk pressure, then reconsidered");
2320 t.hold_machine(Some(
2321 "not enough free space to start a run: 10 bytes free, 100 required by \
2322 `[disk] min_free_bytes`"
2323 .to_owned(),
2324 ));
2325 t.record_answer(
2326 "How should this be handled?".to_owned(),
2327 "keep it held, a human will look at it later".to_owned(),
2328 );
2329 queue.put(&mut t).unwrap();
2330
2331 apply(
2332 &queue,
2333 &questions,
2334 &Verdict {
2335 decisions: vec![Decision {
2336 id: t.id.clone(),
2337 recovery: Some(Recovery::Hold),
2338 ..Decision::default()
2339 }],
2340 },
2341 )
2342 .unwrap();
2343
2344 let after = queue.get(&t.id).unwrap();
2345 assert_eq!(after.status, TaskStatus::Held);
2346 assert!(
2347 !after
2348 .hold_reason
2349 .as_deref()
2350 .unwrap_or_default()
2351 .starts_with("not enough free space"),
2352 "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2353 after.hold_reason
2354 );
2355
2356 let cfg_dir = tempdir().unwrap();
2359 let config = cfg_dir.path().join("magi.toml");
2360 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2361 let report =
2362 crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2363 assert!(report.resumed.is_empty(), "must not be auto-released");
2364 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2365 }
2366
2367 #[test]
2368 fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2369 crate::run::pin_test_home();
2370 let dir = tempdir().unwrap();
2371 let queue = Queue::at(dir.path().join("queue"));
2372 let questions = Questions::at(dir.path().join("questions"));
2373 let config = dir.path().join("magi.toml");
2374 std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2375 let now = jiff::Timestamp::now();
2376 let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2377 let open = || {
2378 questions
2379 .list()
2380 .into_iter()
2381 .filter(|q| q.node == "triage" && q.status.open())
2382 .collect::<Vec<_>>()
2383 };
2384 let hold = Decision {
2385 recovery: Some(Recovery::Hold),
2386 reason: Some("waiting on manual worktree cleanup".to_owned()),
2387 ..Decision::default()
2388 };
2389
2390 let mut t = task("looping hold");
2391 t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2392 queue.put(&mut t).unwrap();
2393 let hold = Decision {
2394 id: t.id.clone(),
2395 ..hold
2396 };
2397
2398 assert_eq!(triage().asked.len(), 1);
2400 let first = open().remove(0);
2401 let mut q = questions.get(&first.id).unwrap();
2402 let resume = q.choices[0].clone();
2403 q.answer(Answer::Choice(resume)).unwrap();
2404 questions.put(&mut q).unwrap();
2405 assert_eq!(triage().answered.len(), 1);
2406 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2407
2408 apply(
2410 &queue,
2411 &questions,
2412 &Verdict {
2413 decisions: vec![hold.clone()],
2414 },
2415 )
2416 .unwrap();
2417 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2418 assert!(triage().asked.is_empty());
2419
2420 let mut evidence = Question::new(
2422 t.id.clone(),
2423 "preflight".into(),
2424 "check".into(),
2425 "New permission failure".into(),
2426 String::new(),
2427 Vec::new(),
2428 );
2429 questions.put(&mut evidence).unwrap();
2430 let hold = Decision {
2431 evidence: vec![evidence.id.clone()],
2432 reason: Some("new permission check failed".into()),
2433 ..hold
2434 };
2435 apply(
2436 &queue,
2437 &questions,
2438 &Verdict {
2439 decisions: vec![hold.clone()],
2440 },
2441 )
2442 .unwrap();
2443 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2444
2445 assert_eq!(triage().asked.len(), 1);
2447 let second = open().remove(0);
2448 assert!(second.detail.contains("new permission check failed"));
2449 assert!(triage().asked.is_empty(), "no duplicate question");
2450 assert_eq!(open().len(), 1);
2451
2452 let mut q = questions.get(&second.id).unwrap();
2454 let force = q.choices[0].clone();
2455 q.answer(Answer::Choice(force)).unwrap();
2456 questions.put(&mut q).unwrap();
2457 assert_eq!(triage().answered.len(), 1);
2458 apply(
2459 &queue,
2460 &questions,
2461 &Verdict {
2462 decisions: vec![hold],
2463 },
2464 )
2465 .unwrap();
2466 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2467 }
2468
2469 #[test]
2470 fn a_runnable_task_can_be_held_directly_without_a_question() {
2471 let dir = tempdir().unwrap();
2472 let queue = Queue::at(dir.path().join("queue"));
2473 let questions = Questions::at(dir.path().join("questions"));
2474 let mut t = task("already answered, should stay put");
2475 queue.put(&mut t).unwrap();
2476
2477 apply(
2478 &queue,
2479 &questions,
2480 &Verdict {
2481 decisions: vec![Decision {
2482 id: t.id.clone(),
2483 recovery: Some(Recovery::Hold),
2484 reason: Some("operator already said keep this held".to_owned()),
2485 ..Decision::default()
2486 }],
2487 },
2488 )
2489 .unwrap();
2490
2491 let after = queue.get(&t.id).unwrap();
2492 assert_eq!(after.status, TaskStatus::Held);
2493 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2494 }
2495
2496 #[test]
2497 fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2498 crate::run::pin_test_home();
2499 let dir = tempdir().unwrap();
2505 let queue = Queue::at(dir.path().join("queue"));
2506 let questions = Questions::at(dir.path().join("questions"));
2507 let mut t = task("already merged by hand");
2508 t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2509 t.record_answer(
2510 "Handle this one?".to_owned(),
2511 "already merged and cleaned up, close it".to_owned(),
2512 );
2513 queue.put(&mut t).unwrap();
2514
2515 apply(
2516 &queue,
2517 &questions,
2518 &Verdict {
2519 decisions: vec![Decision {
2520 id: t.id.clone(),
2521 recovery: Some(Recovery::Done),
2522 reason: Some("operator confirmed this already landed".to_owned()),
2523 ..Decision::default()
2524 }],
2525 },
2526 )
2527 .unwrap();
2528
2529 let after = queue.get(&t.id).unwrap();
2530 assert_eq!(after.status, TaskStatus::Done);
2531 assert!(after.hold_reason.is_none());
2532 assert_eq!(after.answers.len(), 1, "the record of why is kept");
2533 }
2534
2535 #[test]
2536 fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2537 let dir = tempdir().unwrap();
2538 let queue = Queue::at(dir.path().join("queue"));
2539 let questions = Questions::at(dir.path().join("questions"));
2540
2541 let mut queued = task("never ran yet");
2542 queue.put(&mut queued).unwrap();
2543
2544 let mut running = task("mid-run");
2545 running.start("run-1".to_owned());
2546 queue.put(&mut running).unwrap();
2547
2548 for id in [queued.id.clone(), running.id.clone()] {
2549 apply(
2550 &queue,
2551 &questions,
2552 &Verdict {
2553 decisions: vec![Decision {
2554 id,
2555 recovery: Some(Recovery::Done),
2556 ..Decision::default()
2557 }],
2558 },
2559 )
2560 .unwrap();
2561 }
2562
2563 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2564 assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2565 }
2566
2567 #[test]
2568 fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2569 let dir = tempdir().unwrap();
2572 let queue = Queue::at(dir.path().join("queue"));
2573 let questions = Questions::at(dir.path().join("questions"));
2574 let mut t = task("asked about repeatedly");
2575 t.hold_machine(Some("out of attempts".to_owned()));
2576 t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2577 t.record_answer(
2578 "Handle this one? (2)".to_owned(),
2579 "still not yet".to_owned(),
2580 );
2581 queue.put(&mut t).unwrap();
2582 assert_eq!(questions.list().len(), 0);
2583
2584 let verdict = Verdict {
2585 decisions: vec![Decision {
2586 id: t.id.clone(),
2587 question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2588 ..Decision::default()
2589 }],
2590 };
2591 apply(&queue, &questions, &verdict).unwrap();
2592
2593 let filed = questions.list();
2594 assert_eq!(filed.len(), 1, "the question was filed");
2595 assert_eq!(
2596 filed[0].summary,
2597 "Branch conflicts with origin/main, how do we proceed?"
2598 );
2599 let deputy = filed[0]
2600 .deputy
2601 .as_ref()
2602 .expect("the wait is handed to a deputy");
2603 assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2604 assert_eq!(
2605 deputy.starts, 0,
2606 "filing never starts anything: the loop does not wait"
2607 );
2608 assert!(filed[0].status.open());
2609 assert_eq!(filed[0].node, NODE);
2610 let after = queue.get(&t.id).unwrap();
2611 assert_eq!(after.status, TaskStatus::Blocked);
2612 assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2613 assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2614 assert!(
2615 !after
2616 .hold_reason
2617 .clone()
2618 .unwrap_or_default()
2619 .contains("conduct tried to ask"),
2620 "no hold was applied"
2621 );
2622
2623 apply(&queue, &questions, &verdict).unwrap();
2624 let again = questions.list();
2625 assert_eq!(again.len(), 1, "the open question is reused");
2626 assert_eq!(again[0].id, filed[0].id);
2627 }
2628
2629 #[test]
2630 fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
2631 let dir = tempdir().unwrap();
2632 let queue = Queue::at(dir.path().join("queue"));
2633 let questions = Questions::at(dir.path().join("questions"));
2634 let mut t = task("asked about once already");
2635 t.hold_machine(Some("out of attempts".to_owned()));
2636 t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
2637 queue.put(&mut t).unwrap();
2638
2639 apply(
2640 &queue,
2641 &questions,
2642 &Verdict {
2643 decisions: vec![Decision {
2644 id: t.id.clone(),
2645 question: Some("Still not sure - now what?".to_owned()),
2646 ..Decision::default()
2647 }],
2648 },
2649 )
2650 .unwrap();
2651
2652 assert_eq!(questions.list().len(), 1, "the second question was filed");
2653 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2654 }
2655
2656 #[test]
2657 fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
2658 let dir = tempdir().unwrap();
2664 let queue = Queue::at(dir.path().join("queue"));
2665 let questions = Questions::at(dir.path().join("questions"));
2666 let mut t = task("held, triage already asking about it");
2667 t.hold_machine(Some("cause unclear".to_owned()));
2668 queue.put(&mut t).unwrap();
2669
2670 let mut triage_q = Question::new(
2671 t.id.clone(),
2672 crate::triage::NODE.to_owned(),
2673 "triage".to_owned(),
2674 "Still needed?".to_owned(),
2675 String::new(),
2676 vec![
2677 "resume".to_owned(),
2678 "not yet".to_owned(),
2679 "discard".to_owned(),
2680 ],
2681 );
2682 questions.put(&mut triage_q).unwrap();
2683
2684 for decision in [
2685 Decision {
2686 id: t.id.clone(),
2687 question: Some("what now?".to_owned()),
2688 ..Decision::default()
2689 },
2690 Decision {
2691 id: t.id.clone(),
2692 recovery: Some(Recovery::Requeue),
2693 ..Decision::default()
2694 },
2695 ] {
2696 apply(
2697 &queue,
2698 &questions,
2699 &Verdict {
2700 decisions: vec![decision],
2701 },
2702 )
2703 .unwrap();
2704 }
2705
2706 let after = queue.get(&t.id).unwrap();
2707 assert_eq!(
2708 after.status,
2709 TaskStatus::Held,
2710 "triage still owns this hold"
2711 );
2712 assert!(after.blocked_by.is_empty());
2713 assert_eq!(
2714 questions.list().len(),
2715 1,
2716 "no second, conductor-owned question was filed"
2717 );
2718 }
2719
2720 #[test]
2721 fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
2722 let dir = tempdir().unwrap();
2730 let queue = Queue::at(dir.path().join("queue"));
2731 let questions = Questions::at(dir.path().join("questions"));
2732 let mut t = task("held, triage question answered but not yet applied");
2733 t.hold_machine(Some("cause unclear".to_owned()));
2734 queue.put(&mut t).unwrap();
2735
2736 let mut triage_q = Question::new(
2737 t.id.clone(),
2738 crate::triage::NODE.to_owned(),
2739 "triage".to_owned(),
2740 "Still needed?".to_owned(),
2741 String::new(),
2742 vec![
2743 "resume".to_owned(),
2744 "not yet".to_owned(),
2745 "discard".to_owned(),
2746 ],
2747 );
2748 questions.put(&mut triage_q).unwrap();
2749 triage_q
2750 .answer(Answer::Choice("not yet".to_owned()))
2751 .unwrap();
2752 questions.put(&mut triage_q).unwrap();
2753 assert!(!triage_q.status.open());
2754
2755 apply(
2756 &queue,
2757 &questions,
2758 &Verdict {
2759 decisions: vec![Decision {
2760 id: t.id.clone(),
2761 question: Some("what now?".to_owned()),
2762 ..Decision::default()
2763 }],
2764 },
2765 )
2766 .unwrap();
2767
2768 let after = queue.get(&t.id).unwrap();
2769 assert_eq!(
2770 after.status,
2771 TaskStatus::Held,
2772 "triage's own answer is not yet applied - conduct must wait"
2773 );
2774 assert_eq!(
2775 questions.list().len(),
2776 1,
2777 "no conductor question was filed over the pending triage answer"
2778 );
2779 }
2780
2781 #[test]
2782 fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
2783 let dir = tempdir().unwrap();
2784 let queue = Queue::at(dir.path().join("queue"));
2785 let questions = Questions::at(dir.path().join("questions"));
2786 let mut held = task("manual recovery");
2787 held.priority = 300;
2788 held.runs.push("run20260912-224242-daf5".to_owned());
2789 held.hold_manual(Some(
2790 "active manual recovery run20260912-224242-daf5".to_owned(),
2791 ));
2792 queue.put(&mut held).unwrap();
2793
2794 for decision in [
2798 Decision {
2799 id: held.id.clone(),
2800 recovery: Some(Recovery::Requeue),
2801 ..Decision::default()
2802 },
2803 Decision {
2804 id: held.id.clone(),
2805 recovery: Some(Recovery::Hold),
2806 reason: Some("stale replacement reason".to_owned()),
2807 ..Decision::default()
2808 },
2809 Decision {
2810 id: held.id.clone(),
2811 recovery: Some(Recovery::Review),
2812 ..Decision::default()
2813 },
2814 Decision {
2815 id: held.id.clone(),
2816 blocked_by: vec!["other-task".to_owned()],
2817 question: Some("retry now?".to_owned()),
2818 ..Decision::default()
2819 },
2820 ] {
2821 apply(
2822 &queue,
2823 &questions,
2824 &Verdict {
2825 decisions: vec![decision],
2826 },
2827 )
2828 .unwrap();
2829 }
2830
2831 let after = queue.get(&held.id).unwrap();
2832 assert_eq!(after.status, TaskStatus::Held);
2833 assert!(after.operator_held());
2834 assert_eq!(after.priority, 300);
2835 assert_eq!(after.runs, ["run20260912-224242-daf5"]);
2836 assert_eq!(
2837 after.hold_reason.as_deref(),
2838 Some("active manual recovery run20260912-224242-daf5")
2839 );
2840 assert!(after.blocked_by.is_empty());
2841 assert!(questions.list().is_empty());
2842 assert!(
2843 queue.next_runnable().is_none(),
2844 "must not dispatch a duplicate"
2845 );
2846 }
2847
2848 #[test]
2849 fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
2850 let dir = tempdir().unwrap();
2851 let queue = Queue::at(dir.path().join("queue"));
2852 let questions = Questions::at(dir.path().join("questions"));
2853
2854 let mut automatic = task("disk gate");
2855 automatic.hold_machine(Some("disk full".to_owned()));
2856 queue.put(&mut automatic).unwrap();
2857 let requeue = || Verdict {
2858 decisions: vec![Decision {
2859 id: automatic.id.clone(),
2860 recovery: Some(Recovery::Requeue),
2861 ..Decision::default()
2862 }],
2863 };
2864 apply(&queue, &questions, &requeue()).unwrap();
2865 assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
2866
2867 let mut manual = task("operator gate");
2868 manual.hold_manual(Some("wait for operator".to_owned()));
2869 queue.put(&mut manual).unwrap();
2870 apply(
2871 &queue,
2872 &questions,
2873 &Verdict {
2874 decisions: vec![Decision {
2875 id: manual.id.clone(),
2876 recovery: Some(Recovery::Requeue),
2877 ..Decision::default()
2878 }],
2879 },
2880 )
2881 .unwrap();
2882 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
2883
2884 let mut released = queue.get(&manual.id).unwrap();
2887 released.release();
2888 queue.put(&mut released).unwrap();
2889 assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
2890 }
2891
2892 #[test]
2893 fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
2894 let dir = tempdir().unwrap();
2895 let queue = Queue::at(dir.path().join("queue"));
2896 let questions = Questions::at(dir.path().join("questions"));
2897 let mut legacy = task("old explicit hold");
2898 legacy.status = TaskStatus::Held;
2899 legacy.hold_reason = Some("manual recovery already active".to_owned());
2900 legacy.hold_source = None;
2901 legacy.blocked_by = vec!["dependency".to_owned()];
2902 queue.put(&mut legacy).unwrap();
2903
2904 apply(
2905 &queue,
2906 &questions,
2907 &Verdict {
2908 decisions: vec![Decision {
2909 id: legacy.id.clone(),
2910 recovery: Some(Recovery::Requeue),
2911 ..Decision::default()
2912 }],
2913 },
2914 )
2915 .unwrap();
2916
2917 let after = queue.get(&legacy.id).unwrap();
2918 assert_eq!(after.status, TaskStatus::Held);
2919 assert_eq!(after.hold_source, None);
2920 assert_eq!(after.hold_reason, legacy.hold_reason);
2921 assert_eq!(after.blocked_by, legacy.blocked_by);
2922 }
2923
2924 #[test]
2925 fn review_recovery_is_a_no_op_without_a_survivable_branch() {
2926 crate::run::pin_test_home();
2932 let dir = tempdir().unwrap();
2933 let queue = Queue::at(dir.path().join("queue"));
2934 let questions = Questions::at(dir.path().join("questions"));
2935 let mut t = task("blocked with no readable run");
2936 t.start("20260101-000000-dead".to_owned()); t.fail("blocked", 5);
2938 queue.put(&mut t).unwrap();
2939
2940 apply(
2941 &queue,
2942 &questions,
2943 &Verdict {
2944 decisions: vec![Decision {
2945 id: t.id.clone(),
2946 recovery: Some(Recovery::Review),
2947 ..Decision::default()
2948 }],
2949 },
2950 )
2951 .unwrap();
2952
2953 let after = queue.get(&t.id).unwrap();
2954 assert_eq!(
2955 after.status,
2956 TaskStatus::Failed,
2957 "with nothing to reopen, the decision is dropped rather than guessed at"
2958 );
2959 assert!(after.review_branch.is_none());
2960 }
2961
2962 #[test]
2963 fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
2964 let dir = tempdir().unwrap();
2968 let queue = Queue::at(dir.path().join("queue"));
2969 let questions = Questions::at(dir.path().join("questions"));
2970
2971 for recovery in [Recovery::Requeue, Recovery::Review] {
2972 let mut t = task("ordinary");
2973 queue.put(&mut t).unwrap();
2974
2975 apply(
2976 &queue,
2977 &questions,
2978 &Verdict {
2979 decisions: vec![Decision {
2980 id: t.id.clone(),
2981 recovery: Some(recovery),
2982 ..Decision::default()
2983 }],
2984 },
2985 )
2986 .unwrap();
2987
2988 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2989 }
2990 }
2991
2992 #[tokio::test]
2993 async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
2994 let dir = tempdir().unwrap();
2995 let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
2996 let queue = Queue::at(dir.path().join("queue"));
2997 let questions = Questions::at(dir.path().join("questions"));
2998 let mut t = task("normal");
2999 queue.put(&mut t).unwrap();
3000
3001 let mut conductor = Conductor::new();
3002 conductor
3003 .maybe_run(
3004 &cfg,
3005 dir.path(),
3006 &queue,
3007 &questions,
3008 dir.path(),
3009 &[t.clone()],
3010 &[],
3011 &[],
3012 2,
3013 )
3014 .await;
3015
3016 assert_eq!(
3017 queue.get(&t.id).unwrap().status,
3018 TaskStatus::Queued,
3019 "a failed invocation must change nothing"
3020 );
3021 assert!(
3022 queue.next_runnable().is_some(),
3023 "the loop must still be able to take the next task"
3024 );
3025 }
3026
3027 #[tokio::test]
3028 async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3029 let dir = tempdir().unwrap();
3030 let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3031 let queue = Queue::at(dir.path().join("queue"));
3032 let questions = Questions::at(dir.path().join("questions"));
3033 let mut t = task("normal");
3034 queue.put(&mut t).unwrap();
3035
3036 let mut conductor = Conductor::new();
3037 conductor
3038 .maybe_run(
3039 &cfg,
3040 dir.path(),
3041 &queue,
3042 &questions,
3043 dir.path(),
3044 &[t.clone()],
3045 &[],
3046 &[],
3047 2,
3048 )
3049 .await;
3050
3051 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3052 }
3053
3054 #[tokio::test]
3055 async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3056 let dir = tempdir().unwrap();
3057 let mut t = task("chained");
3058 let reply = format!(
3059 "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3060 t.id
3061 );
3062 let counted = |id: &str, tail: &str| {
3063 let calls = dir.path().join(format!("{id}.calls"));
3064 let path = dir.path().join(format!("{id}.sh"));
3065 std::fs::write(
3066 &path,
3067 format!(
3068 "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3069 calls.display()
3070 ),
3071 )
3072 .unwrap();
3073 AgentSpec {
3074 id: id.to_owned(),
3075 kind: AgentKind::Command,
3076 model: None,
3077 command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3078 extra_args: Vec::new(),
3079 env: BTreeMap::new(),
3080 prompt_delivery: None,
3081 }
3082 };
3083 let n_calls = |id: &str| {
3084 std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3085 .map_or(0, |s| s.lines().count())
3086 };
3087 let mut cfg = config(counted("a", "exit 3"));
3088 cfg.agents = vec![
3089 counted("a", "exit 3"),
3090 counted("b", &format!("printf '%s' '{reply}'")),
3091 ];
3092 cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3093 "a".into(),
3094 "b".into(),
3095 "a".into(),
3096 ]));
3097 let queue = Queue::at(dir.path().join("queue"));
3098 let questions = Questions::at(dir.path().join("questions"));
3099 queue.put(&mut t).unwrap();
3100
3101 let mut conductor = Conductor::new();
3102 conductor
3103 .maybe_run(
3104 &cfg,
3105 dir.path(),
3106 &queue,
3107 &questions,
3108 dir.path(),
3109 &[t.clone()],
3110 &[],
3111 &[],
3112 2,
3113 )
3114 .await;
3115
3116 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3117 assert_eq!(n_calls("a"), 1, "each id is tried once");
3118 assert_eq!(n_calls("b"), 1);
3119 let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3120 assert!(
3121 saved.contains("\"b\""),
3122 "the seat on disk is the one that ran: {saved}"
3123 );
3124 }
3125
3126 #[tokio::test]
3127 async fn json_survives_code_fences_and_a_preamble() {
3128 let dir = tempdir().unwrap();
3129 let mut t = task("fenced");
3130 let reply = format!(
3131 "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3132 \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3133 t.id
3134 );
3135 let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3136 let queue = Queue::at(dir.path().join("queue"));
3137 let questions = Questions::at(dir.path().join("questions"));
3138 queue.put(&mut t).unwrap();
3139
3140 let mut conductor = Conductor::new();
3141 conductor
3142 .maybe_run(
3143 &cfg,
3144 dir.path(),
3145 &queue,
3146 &questions,
3147 dir.path(),
3148 &[t.clone()],
3149 &[],
3150 &[],
3151 2,
3152 )
3153 .await;
3154
3155 let back = queue.get(&t.id).unwrap();
3156 assert_eq!(back.status, TaskStatus::Blocked);
3157 assert_eq!(back.blocked_by, ["x"]);
3158 }
3159
3160 #[tokio::test]
3161 async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3162 let dir = tempdir().unwrap();
3165 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3166 let queue = Queue::at(dir.path().join("queue"));
3167 let questions = Questions::at(dir.path().join("questions"));
3168 let mut t = task("stable");
3169 queue.put(&mut t).unwrap();
3170 let artifacts = dir.path().join("conduct").join("artifacts");
3171 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3172
3173 let mut conductor = Conductor::new();
3174 conductor
3175 .maybe_run(
3176 &cfg,
3177 dir.path(),
3178 &queue,
3179 &questions,
3180 dir.path(),
3181 &[t.clone()],
3182 &[],
3183 &[],
3184 2,
3185 )
3186 .await;
3187 assert!(turn(1).is_file(), "the first cycle must call the conductor");
3188
3189 conductor
3190 .maybe_run(
3191 &cfg,
3192 dir.path(),
3193 &queue,
3194 &questions,
3195 dir.path(),
3196 &[t.clone()],
3197 &[],
3198 &[],
3199 2,
3200 )
3201 .await;
3202 assert!(
3203 !turn(2).is_file(),
3204 "an unchanged revision and an unchanged stalled/finished set must not call the \
3205 conductor twice"
3206 );
3207
3208 t.priority = 1;
3210 queue.put(&mut t).unwrap();
3211 conductor
3212 .maybe_run(
3213 &cfg,
3214 dir.path(),
3215 &queue,
3216 &questions,
3217 dir.path(),
3218 &[t.clone()],
3219 &[],
3220 &[],
3221 2,
3222 )
3223 .await;
3224 assert!(turn(2).is_file(), "a moved revision calls it again");
3225 }
3226
3227 #[tokio::test]
3228 async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3229 let dir = tempdir().unwrap();
3235 let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3236 let queue = Queue::at(dir.path().join("queue"));
3237 let questions = Questions::at(dir.path().join("questions"));
3238 let mut t = task("quiet");
3239 queue.put(&mut t).unwrap();
3240 let artifacts = dir.path().join("conduct").join("artifacts");
3241 let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3242
3243 let mut conductor = Conductor::new();
3244 conductor
3245 .maybe_run(
3246 &cfg,
3247 dir.path(),
3248 &queue,
3249 &questions,
3250 dir.path(),
3251 &[t.clone()],
3252 &[],
3253 &[],
3254 2,
3255 )
3256 .await;
3257 assert!(turn(1).is_file());
3258
3259 conductor
3260 .maybe_run(
3261 &cfg,
3262 dir.path(),
3263 &queue,
3264 &questions,
3265 dir.path(),
3266 &[],
3267 &[t.clone()],
3268 &[],
3269 2,
3270 )
3271 .await;
3272 assert!(
3273 turn(2).is_file(),
3274 "a task turning stalled must call the conductor again"
3275 );
3276
3277 conductor
3280 .maybe_run(
3281 &cfg,
3282 dir.path(),
3283 &queue,
3284 &questions,
3285 dir.path(),
3286 &[],
3287 &[t.clone()],
3288 &[],
3289 2,
3290 )
3291 .await;
3292 assert!(
3293 !turn(3).is_file(),
3294 "the same stalled task lingering must not call the conductor every cycle"
3295 );
3296 }
3297
3298 fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3299 apply(
3300 queue,
3301 questions,
3302 &Verdict {
3303 decisions: vec![Decision {
3304 id: id.to_owned(),
3305 reason: Some(note.to_owned()),
3306 recovery: Some(Recovery::Hold),
3307 ..Decision::default()
3308 }],
3309 },
3310 )
3311 .unwrap();
3312 }
3313
3314 #[test]
3315 fn reholding_with_an_identical_note_changes_nothing() {
3316 let dir = tempdir().unwrap();
3317 let queue = Queue::at(dir.path().join("queue"));
3318 let questions = Questions::at(dir.path().join("questions"));
3319 let mut t = task("held");
3320 t.hold_machine(Some("waiting on a human".to_owned()));
3321 queue.put(&mut t).unwrap();
3322
3323 rehold(&queue, &questions, &t.id, "owner said keep it held");
3324 let first = queue.get(&t.id).unwrap();
3325 let rev = queue.revision();
3326 for _ in 0..5 {
3327 rehold(&queue, &questions, &t.id, "owner said keep it held");
3328 }
3329 let after = queue.get(&t.id).unwrap();
3330 assert_eq!(after.hold_reason, first.hold_reason);
3331 assert_eq!(after.updated_at, first.updated_at);
3332 assert_eq!(queue.revision(), rev, "nothing was written");
3333 }
3334
3335 #[test]
3336 fn reholding_with_different_notes_keeps_one_previously_level() {
3337 let dir = tempdir().unwrap();
3338 let queue = Queue::at(dir.path().join("queue"));
3339 let questions = Questions::at(dir.path().join("questions"));
3340 let mut t = task("held");
3341 t.hold_machine(Some("original".to_owned()));
3342 queue.put(&mut t).unwrap();
3343
3344 for i in 0..20 {
3345 rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3346 }
3347 let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3348 assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3349 assert!(reason.starts_with("note "), "the new note leads: {reason}");
3350 }
3351
3352 #[test]
3353 fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3354 let mut t = task("deep");
3355 let nested = "same\n\n(previously: same\n\n(previously: same))";
3356 t.hold_machine(Some(nested.to_owned()));
3357 let d = Decision {
3358 id: t.id.clone(),
3359 reason: Some("same".to_owned()),
3360 ..Decision::default()
3361 };
3362 assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3363 let d2 = Decision {
3364 reason: Some("other".to_owned()),
3365 ..d
3366 };
3367 assert_eq!(
3368 reaffirmed_hold_reason(&t, &d2).as_deref(),
3369 Some("other\n\n(previously: same)")
3370 );
3371 }
3372
3373 #[tokio::test]
3374 async fn a_repeated_hold_decision_settles_instead_of_looping() {
3375 let dir = tempdir().unwrap();
3376 let queue = Queue::at(dir.path().join("queue"));
3377 let questions = Questions::at(dir.path().join("questions"));
3378 let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3379 let mut t = task("held");
3380 t.hold_machine(Some("first".to_owned()));
3381 queue.put(&mut t).unwrap();
3382 let cfg = config(mock_agent(
3383 dir.path(),
3384 REPLY,
3385 env(&reply.replace("ID", &t.id)),
3386 ));
3387 let mut conductor = Conductor::new();
3388 for _ in 0..2 {
3390 let held = queue.get(&t.id).unwrap();
3391 conductor
3392 .maybe_run(
3393 &cfg,
3394 dir.path(),
3395 &queue,
3396 &questions,
3397 dir.path(),
3398 &[],
3399 &[],
3400 &[held],
3401 2,
3402 )
3403 .await;
3404 }
3405 let held = queue.get(&t.id).unwrap();
3406 assert!(
3407 held.hold_reason
3408 .as_deref()
3409 .unwrap()
3410 .starts_with("keep held")
3411 );
3412 assert!(
3413 !conductor.worth_a_look(&queue, &[], &[held]),
3414 "an unchanged held task must stop being reconsidered"
3415 );
3416 }
3417
3418 #[tokio::test]
3419 async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3420 let dir = tempdir().unwrap();
3421 let queue = Queue::at(dir.path().join("queue"));
3422 let questions = Questions::at(dir.path().join("questions"));
3423 let mut t = task("held");
3424 t.hold_machine(Some("first".to_owned()));
3425 queue.put(&mut t).unwrap();
3426 let reply = |note: &str| {
3427 format!(
3428 r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3429 t.id
3430 )
3431 };
3432 let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3433 let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3434 let mut conductor = Conductor::new();
3435 let mut writes = 0;
3436 for i in 0..6 {
3437 let held = queue.get(&t.id).unwrap();
3438 let before = queue.revision();
3439 conductor
3440 .maybe_run(
3441 if i % 2 == 0 { &cfg_a } else { &cfg_b },
3442 dir.path(),
3443 &queue,
3444 &questions,
3445 dir.path(),
3446 &[],
3447 &[],
3448 &[held],
3449 2,
3450 )
3451 .await;
3452 if queue.revision() != before {
3453 writes += 1;
3454 }
3455 }
3456 assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3457 let mut held = queue.get(&t.id).unwrap();
3458 held.instruction.push_str(" (edited)");
3459 queue.put(&mut held).unwrap();
3460 let held = queue.get(&t.id).unwrap();
3461 assert!(conductor.worth_a_look(&queue, &[], &[held]));
3462 }
3463
3464 #[test]
3465 fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3466 let dir = tempdir().unwrap();
3467 let queue = Queue::at(dir.path().join("queue"));
3468 let mut t = task("t");
3469 queue.put(&mut t).unwrap();
3470
3471 let mut conductor = Conductor::new();
3472 assert!(
3473 conductor.worth_a_look(&queue, &[], &[]),
3474 "a conductor that has never run has something to look at"
3475 );
3476
3477 conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3478 assert!(
3479 !conductor.worth_a_look(&queue, &[], &[]),
3480 "nothing changed and nothing is stalled or finished"
3481 );
3482 assert!(
3483 conductor.worth_a_look(&queue, &[t.clone()], &[]),
3484 "a stalled task is worth a look even at the same revision"
3485 );
3486 assert!(
3487 conductor.worth_a_look(&queue, &[], &[t.clone()]),
3488 "a finished task is worth a look even at the same revision"
3489 );
3490 }
3491
3492 #[tokio::test]
3493 async fn the_conduct_path_never_calls_ask_and_wait() {
3494 let dir = tempdir().unwrap();
3500 let queue = Queue::at(dir.path().join("queue"));
3501 let questions = Questions::at(dir.path().join("questions"));
3502 let mut t = task("asks without blocking");
3503 queue.put(&mut t).unwrap();
3504
3505 apply(
3506 &queue,
3507 &questions,
3508 &Verdict {
3509 decisions: vec![Decision {
3510 id: t.id.clone(),
3511 question: Some("ok?".to_owned()),
3512 ..Decision::default()
3513 }],
3514 },
3515 )
3516 .unwrap();
3517 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3519 }
3520
3521 #[test]
3522 fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3523 let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3524 t.resume_override = Some(crate::queue::OperatorResume {
3525 question_id: "q".into(),
3526 at: jiff::Timestamp::now(),
3527 conductor_rehold: None,
3528 forced: false,
3529 pinned_run: Some("run-1".into()),
3530 });
3531 assert!(pinned_resume(&t));
3532 assert!(!may_hold(&mut t, "waiting for magi resume"));
3533 assert!(
3534 t.resume_override
3535 .as_ref()
3536 .unwrap()
3537 .conductor_rehold
3538 .is_none(),
3539 "a refused hold is not recorded as an override"
3540 );
3541 t.resume_override = None;
3542 assert!(!pinned_resume(&t));
3543 assert!(may_hold(&mut t, "no override, so a hold is allowed"));
3544 }
3545
3546 #[test]
3547 fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
3548 let dir = tempdir().unwrap();
3549 let queue = Queue::at(dir.path().join("queue"));
3550 let questions = Questions::at(dir.path().join("questions"));
3551 let mut t = task("resume me");
3552 t.resume_override = Some(crate::queue::OperatorResume {
3553 question_id: "q".into(),
3554 at: jiff::Timestamp::now(),
3555 conductor_rehold: None,
3556 forced: true,
3557 pinned_run: Some("run-1".into()),
3558 });
3559 queue.put(&mut t).unwrap();
3560
3561 apply(
3562 &queue,
3563 &questions,
3564 &Verdict {
3565 decisions: vec![
3566 Decision {
3567 id: t.id.clone(),
3568 question: Some("really?".to_owned()),
3569 ..Decision::default()
3570 },
3571 Decision {
3572 id: t.id.clone(),
3573 blocked_by: vec!["other".to_owned()],
3574 ..Decision::default()
3575 },
3576 ],
3577 },
3578 )
3579 .unwrap();
3580
3581 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3582 assert!(questions.list().is_empty());
3583 }
3584}