1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 pub task: String,
101 pub run: String,
103}
104
105#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct Status {
113 pub schema: u32,
115 pub pid: u32,
117 pub started_at: Timestamp,
119 pub updated_at: Timestamp,
121 pub idle: bool,
123 pub current: Vec<Current>,
129 pub completed: usize,
131 pub polls: u64,
133 #[serde(default)]
135 pub question_recovery: Option<crate::question_delivery::Recovery>,
136 #[serde(default)]
138 pub worker_protocol: Option<u32>,
139}
140
141impl Status {
142 #[must_use]
144 pub fn new() -> Self {
145 let now = Timestamp::now();
146 Self {
147 schema: SCHEMA,
148 pid: std::process::id(),
149 started_at: now,
150 updated_at: now,
151 idle: true,
152 current: Vec::new(),
153 completed: 0,
154 polls: 0,
155 question_recovery: None,
156 worker_protocol: Some(1),
157 }
158 }
159}
160
161impl Default for Status {
162 fn default() -> Self {
163 Self::new()
164 }
165}
166
167#[derive(Debug, Clone, Serialize, Deserialize)]
169#[serde(default)]
170pub struct Opts {
171 pub repo: PathBuf,
173 pub config: Option<PathBuf>,
175 pub poll: Duration,
177 pub max_attempts: usize,
179 pub once: bool,
181 pub merge: Option<String>,
183 pub worktrees_root: Option<PathBuf>,
192}
193
194impl Default for Opts {
195 fn default() -> Self {
196 Self {
197 repo: PathBuf::from("."),
198 config: None,
199 poll: POLL,
200 max_attempts: 2,
201 once: false,
202 merge: None,
203 worktrees_root: None,
204 }
205 }
206}
207
208impl Opts {
209 fn absolute(&self) -> Result<Self> {
210 let mut opts = self.clone();
211 opts.repo = std::path::absolute(&opts.repo)?;
212 opts.config = opts
213 .config
214 .as_deref()
215 .map(std::path::absolute)
216 .transpose()?;
217 opts.worktrees_root = opts
218 .worktrees_root
219 .as_deref()
220 .map(std::path::absolute)
221 .transpose()?;
222 Ok(opts)
223 }
224}
225
226fn max_concurrent(n: usize) -> usize {
231 n.max(1)
232}
233
234#[must_use]
236pub fn status_path() -> PathBuf {
237 crate::run::home().join("daemon.json")
238}
239
240pub fn write_status(status: &Status) -> Result<()> {
242 write_status_to(&status_path(), status)
243}
244
245pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
250 if let Some(parent) = path.parent() {
251 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
252 }
253 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
254 let tmp = path.with_extension("json.tmp");
255 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
256 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
257 Ok(())
258}
259
260pub fn clear_status() {
263 clear_status_at(&status_path());
264}
265
266fn clear_status_at(path: &Path) {
270 let _ = std::fs::remove_file(path);
271}
272
273#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
292 wake: Arc<Notify>,
296 pause: crate::graph::Pause,
299}
300
301impl Stop {
302 #[must_use]
304 pub fn new() -> Self {
305 Self::default()
306 }
307
308 pub fn stop(&self) {
311 self.stopped.store(true, Ordering::SeqCst);
312 self.wake.notify_one();
316 }
317
318 #[must_use]
320 pub fn stopped(&self) -> bool {
321 self.stopped.load(Ordering::SeqCst)
322 }
323
324 #[must_use]
327 pub fn finishing(&self) -> bool {
328 self.stopped() && self.busy_now()
329 }
330
331 pub fn park(&self, cause: crate::run::ParkCause) {
337 self.pause.park_for(cause);
338 self.stop();
339 }
340
341 #[must_use]
343 pub fn parking(&self) -> bool {
344 self.pause.parked()
345 }
346
347 #[must_use]
349 pub fn pause(&self) -> crate::graph::Pause {
350 self.pause.clone()
351 }
352
353 #[must_use]
355 pub fn busy_now(&self) -> bool {
356 self.busy.load(Ordering::SeqCst) > 0
357 }
358
359 async fn idle(&self, poll: Duration) {
361 tokio::select! {
362 () = tokio::time::sleep(poll) => {}
363 () = self.wake.notified() => {}
364 }
365 }
366}
367
368#[derive(Debug, Clone, Default, Deserialize)]
375#[serde(default)]
376pub struct Reading {
377 pub schema: u32,
379 pub pid: Option<u32>,
381 pub started_at: Option<Timestamp>,
383 pub updated_at: Option<Timestamp>,
385 pub idle: bool,
387 #[serde(deserialize_with = "de_current")]
399 pub current: Vec<Current>,
400 pub completed: u64,
402 pub polls: u64,
404 #[serde(default)]
406 pub question_recovery: Option<crate::question_delivery::Recovery>,
407 pub worker_protocol: Option<u32>,
409}
410
411fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
414where
415 D: serde::Deserializer<'de>,
416{
417 #[derive(Deserialize)]
418 #[serde(untagged)]
419 enum Shape {
420 Many(Vec<Current>),
421 One(Current),
422 }
423 Ok(
424 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
425 Shape::Many(v) => v,
426 Shape::One(c) => vec![c],
427 }),
428 )
429}
430
431impl Reading {
432 #[must_use]
435 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
436 self.updated_at
437 .map(|at| (now.as_second() - at.as_second()).max(0))
438 }
439
440 #[must_use]
444 pub fn running(&self, now: Timestamp) -> bool {
445 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
446 }
447}
448
449#[must_use]
456pub fn read_status(home: &Path) -> Option<Reading> {
457 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
458 serde_json::from_str(&body).ok()
459}
460
461#[must_use]
470pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
471 let mut current = read_status(home)
472 .filter(|reading| reading.running(now))
473 .map(|reading| reading.current)
474 .unwrap_or_default();
475 let workers = crate::worker::observe_active(home).and_then(|jobs| {
476 jobs.into_iter()
477 .filter(|job| job.kind == "task")
478 .map(|job| {
479 let request: TaskWorker = serde_json::from_value(job.payload)?;
480 let run = Queue::at(request.queue)
481 .get(&request.task)
482 .ok()
483 .and_then(|task| task.runs.last().cloned())
484 .unwrap_or_default();
485 Ok(Current {
486 task: request.task,
487 run,
488 })
489 })
490 .collect::<Result<Vec<_>>>()
491 });
492 match workers {
493 Ok(workers) => {
494 for worker in workers {
495 if let Some(existing) = current.iter_mut().find(|c| c.task == worker.task) {
496 *existing = worker;
497 } else {
498 current.push(worker);
499 }
500 }
501 }
502 Err(error) => {
503 tracing::error!("cannot determine active workers: {error:#}");
506 current.extend(
507 Queue::at(home.join("queue"))
508 .list()
509 .into_iter()
510 .map(|task| Current {
511 task: task.id,
512 run: task.runs.last().cloned().unwrap_or_default(),
513 }),
514 );
515 current.extend(
516 crate::run::list_ids_in(&home.join("runs"))
517 .into_iter()
518 .map(|run| Current {
519 task: String::new(),
520 run,
521 }),
522 );
523 }
524 }
525 current
526}
527
528#[must_use]
530pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
531 current_work(home, now).iter().any(|c| c.run == run)
532}
533
534#[must_use]
544pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
545 current_work(home, now)
546 .iter()
547 .any(|c| crate::run::short_of(&c.run) == short)
548}
549
550#[must_use]
552pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
553 current_work(home, now).iter().any(|c| c.task == task)
554}
555
556pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
596 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
597}
598
599fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
603where
604 F: Fn(u32) -> bool,
605{
606 let this_process = std::process::id();
607 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
608 .into_iter()
609 .flatten()
610 .flatten()
611 .map(|e| e.path())
612 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
613 .filter(|p| {
614 match std::fs::read_to_string(p)
615 .ok()
616 .and_then(|body| body.trim().parse::<u32>().ok())
617 {
618 Some(pid) if pid == this_process => false,
622 Some(pid) => !pid_alive(pid),
623 None => p
624 .metadata()
625 .and_then(|m| m.modified())
626 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
627 .is_ok_and(|age| age >= older_than),
628 }
629 })
630 .filter(|p| std::fs::remove_file(p).is_ok())
631 .filter_map(|p| {
632 p.file_stem()
633 .and_then(|s| s.to_str())
634 .map(std::borrow::ToOwned::to_owned)
635 })
636 .collect();
637 swept.sort_unstable();
638 swept
639}
640
641fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
646 task.status == TaskStatus::Running
647 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
648 && !is_working_on_task(home, &task.id, now)
649}
650
651fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
654 queue
655 .list()
656 .into_iter()
657 .filter(|t| is_stalled(t, home, now))
658 .collect()
659}
660
661fn queued_tasks(queue: &Queue) -> Vec<Task> {
667 queue
668 .list()
669 .into_iter()
670 .filter(|t| t.status == TaskStatus::Queued)
671 .collect()
672}
673
674fn finished_tasks(queue: &Queue) -> Vec<Task> {
677 queue
678 .list()
679 .into_iter()
680 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
681 .collect()
682}
683
684fn resolve_blockers(queue: &Queue, questions: &Questions) {
701 for listed in queue.list() {
702 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
703 continue;
704 }
705 let Ok(_claim) = queue.claim(&listed.id) else {
706 continue;
707 };
708 let Ok(mut task) = queue.get(&listed.id) else {
709 continue;
710 };
711 if task.status != TaskStatus::Blocked {
712 continue;
713 }
714 let deleted = queue.apply_deleted_blockers(&mut task, questions);
718 if !deleted.is_empty() {
719 record(queue, &mut task);
720 for id in &deleted {
721 queue.note_dependency_deleted(&task, id);
722 }
723 if task.status != TaskStatus::Blocked {
724 continue;
725 }
726 }
727 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
728 if !missing.is_empty() {
729 let language = language_of(&task, Path::new("."));
730 let mut cause = missing.clone();
731 cause.sort();
732 cause.dedup();
733 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
734 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
735 &task.blocked_by,
736 &missing,
737 &language,
738 )));
739 record(queue, &mut task);
740 continue;
741 }
742 if let Some(q) = task.blocked_by.iter().find_map(|id| {
745 let crate::queue::Dependency::Question(full) =
746 crate::queue::dependency(queue, questions, id)
747 else {
748 return None;
749 };
750 questions.get(&full).ok().filter(|q| {
751 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
752 })
753 }) {
754 let language = language_of(&task, Path::new("."));
755 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
756 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
757 record(queue, &mut task);
758 continue;
759 }
760 let mut changed = false;
761 for id in task.blocked_by.clone() {
762 let resolved = crate::queue::dependency(queue, questions, &id);
763 if let crate::queue::Dependency::Task(full) = &resolved {
764 if queue
765 .get(full)
766 .is_ok_and(|dep| dep.status == TaskStatus::Done)
767 {
768 task.unblock(&id);
769 changed = true;
770 }
771 continue;
772 }
773 if let crate::queue::Dependency::Question(full) = resolved
774 && let Ok(q) = questions.get(&full)
775 && q.status == ask::QuestionStatus::Answered
776 {
777 if q.node == crate::conduct::NODE
781 && q.chosen_action().is_some()
782 && !task.action_applied(&q.id)
783 {
784 continue;
785 }
786 let answer = match &q.answer {
787 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
788 None => String::new(),
789 };
790 task.record_answer(q.summary.clone(), answer);
791 task.unblock(&id);
792 changed = true;
793 }
794 }
795 if changed {
796 record(queue, &mut task);
797 }
798 }
799}
800
801fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
803 if crate::lang::is_japanese(language) {
804 format!(
805 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
806 q.short(),
807 q.summary
808 )
809 } else {
810 format!(
811 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
812 q.short(),
813 q.summary
814 )
815 }
816}
817
818#[derive(Debug, Clone, Copy, PartialEq, Eq)]
821pub(crate) enum ActionStanding {
822 NoAction,
824 Pending,
826 Applied,
829 Stale,
831 Busy,
833}
834
835impl ActionStanding {
836 pub(crate) fn daemon_owns(self) -> bool {
838 matches!(self, Self::Pending | Self::Applied | Self::Stale)
839 }
840}
841
842pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
846 if q.chosen_action().is_none() {
847 ActionStanding::NoAction
848 } else if task.action_applied(&q.id) {
849 ActionStanding::Applied
850 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
851 ActionStanding::Stale
852 } else if matches!(
853 task.status,
854 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
855 ) {
856 ActionStanding::Busy
857 } else {
858 ActionStanding::Pending
859 }
860}
861
862#[derive(Debug, Clone, PartialEq, Eq)]
864enum ActionDecision {
865 Skip,
867 Resume(String),
869 Requeue,
871 Hold,
873 Done,
875 Stale,
878 Refuse(String),
881}
882
883fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
891where
892 F: FnOnce(&str) -> Result<RunState>,
893{
894 let Some(action) = q.chosen_action() else {
895 return ActionDecision::Skip;
896 };
897 if task.action_applied(&q.id)
898 || matches!(
899 task.status,
900 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
901 )
902 {
903 return ActionDecision::Skip;
904 }
905 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
908 return ActionDecision::Stale;
909 }
910 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
911 let stopping = q.node == land::APPROVAL_NODE
912 && q.resolution()
913 .as_deref()
914 .is_some_and(crate::merge_decision::stops);
915 let decision_at = if repair || stopping {
919 q.answered_at.unwrap_or(q.asked_at)
920 } else {
921 q.asked_at
922 };
923 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
924 return if repair || stopping {
925 ActionDecision::Refuse(format!(
926 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
927 if repair { "repair" } else { "stop" },
928 q.id,
929 q.run,
930 task.id
931 ))
932 } else {
933 ActionDecision::Stale
934 };
935 }
936 match action {
937 a if a.maintenance() => ActionDecision::Skip,
938 ask::ChoiceAction::ResumeRun { .. }
939 | ask::ChoiceAction::ReleaseTask { .. }
940 | ask::ChoiceAction::RerunChecks { .. }
941 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
942 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
943 ask::ChoiceAction::Hold => ActionDecision::Hold,
944 ask::ChoiceAction::Done => ActionDecision::Done,
945 ask::ChoiceAction::Resume { run } => {
946 if task.runs.last() != Some(run) {
947 return ActionDecision::Refuse((p.resume_not_latest)(
948 q.short(),
949 ask::short_id(run),
950 ));
951 }
952 match load(run) {
953 Ok(s) if stopping => {
954 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
955 || !s
956 .land_approval
957 .as_ref()
958 .is_some_and(|binding| binding.question == q.id)
959 {
960 ActionDecision::Stale
961 } else if s.liveness(false) == Liveness::Live {
962 ActionDecision::Skip
963 } else if s.status == RunStatus::Landing
964 && !s.released()
965 && s.retired_to.is_none()
966 {
967 ActionDecision::Resume(run.clone())
968 } else {
969 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
970 }
971 }
972 Ok(s)
973 if q.node == land::APPROVAL_NODE
974 && q.resolution().as_deref() == Some(land::REPAIR) =>
975 {
976 if s.merge_repairs.iter().any(|r| r.question == q.id)
977 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
978 {
979 ActionDecision::Stale
982 } else if s.liveness(false) == Liveness::Live {
983 ActionDecision::Skip
984 } else if s.status != RunStatus::Landing
985 || s.released()
986 || s.reviews.is_empty()
987 || !implementers_match(task, &s)
988 || !s
989 .land_approval
990 .as_ref()
991 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
992 {
993 ActionDecision::Refuse(format!(
994 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
995 q.id, run, run
996 ))
997 } else {
998 ActionDecision::Resume(run.clone())
999 }
1000 }
1001 Ok(s) if s.liveness(false) == Liveness::Live => {
1002 ActionDecision::Refuse("run already has a live driver".into())
1003 }
1004 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
1005 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
1006 q.short(),
1007 ask::short_id(run),
1008 )),
1009 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1010 q.short(),
1011 ask::short_id(run),
1012 &format!("{e:#}"),
1013 )),
1014 }
1015 }
1016 }
1017}
1018
1019struct Phrases {
1033 graph_stopped: fn(&str, &str) -> String,
1035 quorum_lost: &'static str,
1036 quota_took_out: &'static str,
1038 run_ended: &'static str,
1040 waiting_for_answer: &'static str,
1042 recovered_running: &'static str,
1043 no_run_to_recover: &'static str,
1044 could_not_start: &'static str,
1045 resume_not_latest: fn(&str, &str) -> String,
1046 resume_cannot_progress: fn(&str, &str) -> String,
1047 resume_unreadable: fn(&str, &str, &str) -> String,
1048 handover_refused: fn(&crate::handover::Refused) -> String,
1051 handover_hint: &'static str,
1053 reviewers_never_answered: fn(usize, usize) -> String,
1056}
1057
1058const PHRASES_EN: Phrases = Phrases {
1059 graph_stopped: |status, detail| {
1060 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1061 },
1062 quorum_lost: "the judging panel lost its quorum",
1063 quota_took_out: "; quota took out ",
1064 run_ended: "run ended ",
1065 waiting_for_answer: " - waiting for operator answer to question ",
1066 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1067 no_run_to_recover: "task was `running` with no live daemon and no readable \
1068 run to recover; held for a human to check what happened",
1069 could_not_start: "could not start the run: ",
1070 resume_not_latest: |q, run| {
1071 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1072 },
1073 resume_cannot_progress: |q, run| {
1074 format!("question {q} asked to resume run {run}, which cannot make progress")
1075 },
1076 resume_unreadable: |q, run, e| {
1077 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1078 },
1079 handover_refused: |r| r.to_string(),
1080 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1081 reviewers_never_answered: |missing, rounds| {
1082 format!(
1083 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1084 refusing to call it clean"
1085 )
1086 },
1087};
1088
1089const PHRASES_JA: Phrases = Phrases {
1090 graph_stopped: |status, detail| {
1091 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1092 },
1093 quorum_lost: "審査パネルが定足数を失いました",
1094 quota_took_out: "。クォータで脱落: ",
1095 run_ended: "run 終了: ",
1096 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1097 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1098 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1099 何が起きたか人が確認するため保留にしました",
1100 could_not_start: "run を開始できませんでした: ",
1101 resume_not_latest: |q, run| {
1102 format!(
1103 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1104 )
1105 },
1106 resume_cannot_progress: |q, run| {
1107 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1108 },
1109 resume_unreadable: |q, run, e| {
1110 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1111 },
1112 handover_refused: |r| {
1113 use crate::handover::Refused;
1114 match r {
1115 Refused::Foreign { branch, path, why } => format!(
1116 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1117 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1118 から、やり直してください(詳細: {why})"
1119 ),
1120 Refused::Unsafe { branch, path, why } => format!(
1121 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1122 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1123 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1124 ),
1125 Refused::ReleaseFailed { branch, path, run } => format!(
1126 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1127 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1128 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1129 ),
1130 }
1131 },
1132 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1133 reviewers_never_answered: |missing, rounds| {
1134 format!(
1135 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1136 クリーンとは認めません"
1137 )
1138 },
1139};
1140
1141fn phrases(language: &str) -> &'static Phrases {
1142 if crate::lang::is_japanese(language) {
1143 &PHRASES_JA
1144 } else {
1145 &PHRASES_EN
1146 }
1147}
1148
1149fn language_of(task: &Task, fallback: &Path) -> String {
1153 crate::lang::of_repo(&repo_for(task, fallback))
1154}
1155
1156pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1160 if q.node == crate::conduct::NODE {
1161 return tasks.iter().find(|t| t.id == q.run);
1162 }
1163 tasks.iter().find(|t| t.runs.contains(&q.run))
1164}
1165
1166fn repair_choice_offers(questions: &Questions) {
1169 for q in questions.list().into_iter().filter(|q| {
1170 q.status.open()
1171 && q.node == crate::conduct::NODE
1172 && q.choices
1173 .iter()
1174 .any(|c| crate::choice_effect::describe(q, c).is_none())
1175 }) {
1176 let _ = questions.update(&q.id, |current| {
1177 if current.status.open() {
1178 crate::choice_effect::executable_conductor(current);
1179 if let Some(deputy) = &mut current.deputy {
1180 deputy.brief = crate::deputy::brief(
1181 ¤t.run,
1182 ¤t.detail,
1183 ¤t.choices,
1184 ¤t.actions,
1185 );
1186 }
1187 }
1188 Ok(())
1189 });
1190 }
1191}
1192
1193pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1196 apply_maintenance_choices(queue, questions, home).await;
1197 apply_task_choice_actions(queue, questions, home);
1198}
1199
1200async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1203 for q in questions.list() {
1204 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1205 continue;
1206 };
1207 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1210 continue;
1211 }
1212 let dir = home.join("choice-actions");
1213 if std::fs::create_dir_all(&dir).is_err() {
1214 continue;
1215 }
1216 let Ok(lock) = std::fs::OpenOptions::new()
1217 .create(true)
1218 .truncate(false)
1219 .write(true)
1220 .open(dir.join(format!("{}.lock", q.id)))
1221 else {
1222 continue;
1223 };
1224 if crate::pr_head::try_lock_file(&lock).is_err() {
1225 continue;
1226 }
1227 let fresh = !home
1228 .join("choice-actions")
1229 .join(format!("{}.json", q.id))
1230 .exists();
1231 let result = async {
1232 let receipt = dir.join(format!("{}.json", q.id));
1233 if receipt.exists() {
1234 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1235 anyhow::ensure!(
1236 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1237 "previous maintenance attempt did not confirm completion; inspect its receipt"
1238 );
1239 return Ok::<_, anyhow::Error>(());
1240 }
1241 let record = |status: &str| -> anyhow::Result<()> {
1244 let tmp = receipt.with_extension("tmp");
1245 std::fs::write(
1246 &tmp,
1247 serde_json::to_vec(&serde_json::json!({
1248 "action": action, "status": status,
1249 }))?,
1250 )?;
1251 std::fs::rename(tmp, &receipt)?;
1252 Ok(())
1253 };
1254 let applied = async {
1255 match action {
1256 ask::ChoiceAction::ResumeRun { run } => {
1257 crate::resume::validate(home, run)?;
1258 record("started")?;
1259 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1260 crate::resume::validate_unqueued(home, run)?;
1261 crate::resume::start(home, run.clone()).await?;
1262 }
1263 }
1264 ask::ChoiceAction::ReleaseTask {
1265 task,
1266 held_at,
1267 reason,
1268 } => {
1269 anyhow::ensure!(
1270 queue.resolve_id(task)? == *task,
1271 "task action requires an exact task ID"
1272 );
1273 let _claim = queue.claim(task)?;
1274 let mut task = queue.get(task)?;
1275 if task.action_applied(&q.id) {
1276 return Ok(());
1277 }
1278 if q.node == crate::conduct::NODE
1283 && q.run == task.id
1284 && task.status == TaskStatus::Blocked
1285 && task.blocked_from == Some(TaskStatus::Held)
1286 && task.blocked_by.as_slice() == [q.id.as_str()]
1287 {
1288 task.unblock(&q.id);
1289 }
1290 anyhow::ensure!(
1291 task.status == TaskStatus::Held
1292 && task.held_at == Some(*held_at)
1293 && task.hold_reason.as_deref() == reason.as_deref()
1294 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1295 "task no longer has the hold this question names"
1296 );
1297 task.release_operator("choice", Some(&q.id));
1298 task.mark_action_applied(&q.id);
1299 queue.put(&mut task)?;
1300 }
1301 ask::ChoiceAction::AbandonRun { run } => {
1302 record("started")?;
1303 crate::driver::abandon_proven_dead(home, run)?;
1304 }
1305 ask::ChoiceAction::RerunChecks {
1306 repo,
1307 pr,
1308 head,
1309 runs,
1310 } => {
1311 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1312 .await?;
1313 }
1314 _ => unreachable!(),
1315 }
1316 Ok::<_, anyhow::Error>(())
1317 }
1318 .await;
1319 match applied {
1320 Ok(()) => record("applied"),
1321 Err(e) => {
1322 record(&format!("refused or failed: {e:#}"))?;
1323 Err(e)
1324 }
1325 }
1326 }
1327 .await;
1328 if home
1333 .join("choice-actions")
1334 .join(format!("{}.json", q.id))
1335 .exists()
1336 && let Some(listed) = task_of_question(&queue.list(), &q)
1337 && !listed.action_applied(&q.id)
1338 && let Ok(_claim) = queue.claim(&listed.id)
1339 && let Ok(mut origin) = queue.get(&listed.id)
1340 {
1341 if !origin.operator_held()
1342 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1343 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1344 {
1345 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1346 origin.hold_machine(Some(format!(
1347 "Maintenance choice {} settled; inspect its target before releasing this task",
1348 q.id
1349 )));
1350 } else {
1351 origin.unblock(&q.id);
1352 }
1353 }
1354 origin.mark_action_applied(&q.id);
1355 let _ = queue.put(&mut origin);
1356 }
1357 if let Err(e) = result
1358 && fresh
1359 {
1360 notices::raise_in(
1361 home,
1362 Notice::warn(
1363 &format!("action:{}", q.id),
1364 format!("Choice action {} was not completed: {e:#}", q.id),
1365 ),
1366 );
1367 }
1368 }
1369}
1370
1371pub(crate) async fn rerun_checks_once(
1378 home: &Path,
1379 request: &str,
1380 repo: &Path,
1381 pr: &str,
1382 head: &str,
1383 runs: &[String],
1384 started: impl FnOnce() -> anyhow::Result<()>,
1385) -> anyhow::Result<()> {
1386 use crate::release_watch::{GhForge, ReleaseForge};
1387 anyhow::ensure!(
1388 !runs.is_empty()
1389 && runs.len() <= 20
1390 && runs
1391 .iter()
1392 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1393 && !pr.starts_with('-'),
1394 "invalid rerun targets"
1395 );
1396 let snap = GhForge.snapshot(repo, pr).await?;
1397 anyhow::ensure!(
1398 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1399 "pull request head or lifecycle changed"
1400 );
1401 anyhow::ensure!(
1402 runs.iter().all(|r| snap
1403 .checks
1404 .iter()
1405 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1406 "named workflow is no longer failed on this pull request"
1407 );
1408 started()?;
1409 let dir = home.join("choice-actions");
1410 std::fs::create_dir_all(&dir)?;
1411 let rerun_lock = std::fs::OpenOptions::new()
1412 .create(true)
1413 .truncate(false)
1414 .write(true)
1415 .open(dir.join("reruns.lock"))?;
1416 crate::pr_head::try_lock_file(&rerun_lock)?;
1417 let ledger = dir.join("reruns.json");
1418 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1419 serde_json::from_slice(&std::fs::read(&ledger)?)?
1420 } else {
1421 Default::default()
1422 };
1423 for run in runs {
1424 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1425 if !recorded.insert(key) {
1426 continue;
1427 }
1428 let tmp = ledger.with_extension("tmp");
1429 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1430 std::fs::rename(tmp, &ledger)?;
1431 GhForge.rerun(repo, run).await?;
1432 }
1433 Ok(())
1434}
1435
1436pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1445 let tasks = queue.list();
1446 for q in questions.list() {
1447 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1449 continue;
1450 }
1451 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1452 continue;
1453 }
1454 let Some(listed) = task_of_question(&tasks, &q) else {
1455 if q.node == land::APPROVAL_NODE
1456 && q.resolution().as_deref().is_some_and(|choice| {
1457 choice == land::REPAIR || crate::merge_decision::stops(choice)
1458 })
1459 {
1460 notices::raise_in(
1461 home,
1462 Notice::warn(
1463 &format!("repair-owner:{}", q.id),
1464 format!(
1465 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1466 q.run, q.run
1467 ),
1468 ),
1469 );
1470 }
1471 continue;
1472 };
1473 if listed.action_applied(&q.id) {
1474 continue;
1475 }
1476 let Ok(_claim) = queue.claim(&listed.id) else {
1477 continue;
1478 };
1479 let Ok(mut task) = queue.get(&listed.id) else {
1480 continue;
1481 };
1482 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1485 let blockers: Vec<_> = task
1486 .blocked_by
1487 .iter()
1488 .filter(|id| {
1489 crate::queue::dependency(queue, questions, id)
1490 == crate::queue::Dependency::Question(q.id.clone())
1491 })
1492 .cloned()
1493 .collect();
1494 if !blockers.is_empty() {
1495 for id in blockers {
1496 task.unblock(&id);
1497 }
1498 let answer = match &q.answer {
1499 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1500 None => String::new(),
1501 };
1502 task.record_answer(q.summary.clone(), answer);
1503 }
1504 }
1505 let language = language_of(&task, Path::new("."));
1506 let decision = decide_action(&task, &q, phrases(&language), |id| {
1507 RunState::load_under(id, home)
1508 });
1509 let ran = matches!(
1510 decision,
1511 ActionDecision::Resume(_)
1512 | ActionDecision::Requeue
1513 | ActionDecision::Hold
1514 | ActionDecision::Done
1515 );
1516 if ran && asker_may_still_read(questions, &q, home) {
1517 continue;
1526 }
1527 let done = matches!(decision, ActionDecision::Done);
1528 match decision {
1529 ActionDecision::Skip => continue,
1530 ActionDecision::Resume(run) => {
1531 task.release_operator("choice", Some(&q.id));
1532 task.resume_override = Some(crate::queue::OperatorResume {
1533 question_id: q.id.clone(),
1534 at: Timestamp::now(),
1535 conductor_rehold: None,
1536 forced: true,
1537 pinned_run: Some(run),
1538 });
1539 }
1540 ActionDecision::Stale => {}
1541 ActionDecision::Requeue => {
1542 task.release_operator("choice-requeue", Some(&q.id));
1543 task.requeue();
1544 }
1545 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1546 ActionDecision::Done => task.succeed(),
1547 ActionDecision::Refuse(why) => {
1548 if task.operator_held() {
1549 task.hold_reason = Some(match task.hold_reason.take() {
1552 Some(reason) => format!("{reason}\n\n{why}"),
1553 None => why,
1554 });
1555 } else {
1556 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1557 task.hold_machine(Some(why));
1558 }
1559 notices::raise(
1560 Notice::warn(
1561 &format!("action:{}", q.id),
1562 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1563 )
1564 .link(Link::Task {
1565 id: task.id.clone(),
1566 }),
1567 );
1568 }
1569 }
1570 task.mark_action_applied(&q.id);
1571 record(queue, &mut task);
1574 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1575 supersede_prior_runs(&task, home);
1576 }
1577 let _ = questions.update(&q.id, |r| {
1580 r.answer_delivered = true;
1581 Ok(())
1582 });
1583 }
1584}
1585
1586fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1590 let now = Timestamp::now();
1591 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1592 return true;
1593 }
1594 q.node != crate::conduct::NODE
1595 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1596 s.seats_active()
1597 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1598 })
1599}
1600
1601fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1612 let tasks = queue.list();
1613 let by_id: std::collections::BTreeMap<&str, &Task> =
1614 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1615 let referenced: Vec<&str> = tasks
1616 .iter()
1617 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1618 .collect();
1619
1620 for mut question in questions.list() {
1621 if !question.status.open() || question.node != crate::conduct::NODE {
1622 continue;
1623 }
1624 if referenced
1629 .iter()
1630 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1631 {
1632 continue;
1633 }
1634 let Some(task) = by_id.get(question.run.as_str()) else {
1635 continue;
1636 };
1637 question.abandon(format!(
1638 "task {} no longer waits for this answer",
1639 task.short()
1640 ));
1641 if let Err(e) = questions.put(&mut question) {
1642 tracing::warn!(
1643 "could not retire question {} for task {}: {e:#}",
1644 question.short(),
1645 task.short()
1646 );
1647 }
1648 }
1649}
1650
1651#[derive(Debug, Clone, Copy)]
1658pub struct Verdict {
1659 pub status: RunStatus,
1661 pub left_pr: bool,
1663 pub quota_hit: bool,
1665 pub parked: bool,
1667 pub no_viable_candidates: bool,
1674}
1675
1676pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1733 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1734}
1735
1736fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1738 if verdict.parked {
1744 task.park(detail);
1745 return;
1746 }
1747 match verdict.status {
1748 RunStatus::Merged | RunStatus::Ready
1749 if task.pr_recovery.as_ref().is_some_and(|s| {
1750 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1751 }) =>
1752 {
1753 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1754 }
1755 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1756 RunStatus::AlreadyInBase => task.already_landed(detail),
1757 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1758 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1759 task.stall(detail)
1760 }
1761 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1762 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1763 RunStatus::Blocked => task.fail(detail, max_attempts),
1764 RunStatus::VerifiedNoop => task.handed_off(detail),
1765 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1766 }
1767}
1768
1769pub fn supersede_prior_runs(task: &Task, home: &Path) {
1818 let now = Timestamp::now();
1819 let last_run_succeeded = task
1826 .runs
1827 .last()
1828 .and_then(|id| RunState::load_under(id, home).ok())
1829 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1830 for id in task.superseded_attempts(last_run_succeeded) {
1831 let mut state = match RunState::load_under(id, home) {
1832 Ok(s) => s,
1833 Err(e) => {
1834 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1835 continue;
1836 }
1837 };
1838 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1839 continue;
1840 }
1841 let daemon_claims = is_working_on(home, id, now);
1842 if state.liveness(daemon_claims) == Liveness::Live {
1843 continue;
1844 }
1845 state.status = RunStatus::Superseded;
1846 if let Err(e) = state.save_under(home) {
1847 tracing::warn!("could not mark run {id} superseded: {e:#}");
1848 }
1849 }
1850}
1851
1852fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1874 for task in queue.list() {
1875 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1876 continue;
1877 }
1878 supersede_prior_runs(&task, home);
1879 }
1880}
1881
1882fn settle_and_diagnose(
1889 task: &mut Task,
1890 verdict: Verdict,
1891 detail: &str,
1892 max_attempts: usize,
1893 state: &RunState,
1894) {
1895 let p = phrases(&state.config.graph.language);
1896 if let Some(d) = &state.merge_decision {
1899 use crate::merge_decision as md;
1900 if task.runs.last().is_some_and(|id| id != &state.id) {
1901 return;
1902 }
1903 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1904 task.succeed();
1905 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1906 if state.liveness(false) != Liveness::Dead {
1907 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1908 return;
1909 }
1910 let replacement = md::Replacement {
1911 question: d.question.clone(),
1912 run: state.id.clone(),
1913 pr: d.target.pr.clone(),
1914 agent: agent.to_owned(),
1915 context: format!(
1916 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1917 d.target.pr,
1918 state.id,
1919 serde_json::to_string(&state.reviews).unwrap_or_default()
1920 ),
1921 };
1922 task.requeue();
1923 task.pr_recovery = None;
1924 task.review_of = None;
1925 task.replacement = Some(replacement);
1926 } else {
1927 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1928 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1929 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1930 }
1931 task.mark_action_applied(&d.question);
1932 if let Some(pass) = state.passes.last() {
1933 task.record_pass_at(
1934 &state.id,
1935 pass.index,
1936 Some(pass.started_at),
1937 task.attempts,
1938 None,
1939 );
1940 }
1941 return;
1942 }
1943 if let Some(replacement) = &task.replacement
1944 && !state.parked
1945 && state.resource_block.is_none()
1946 && (state.viable().is_empty()
1947 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1948 {
1949 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1950 return;
1951 }
1952 let attempts_before = task.attempts;
1953 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1959 task.notice_cause(
1960 &format!("resource:{}:{}", block.kind, block.node),
1961 None,
1962 None,
1963 );
1964 task.last_error = Some(detail.to_owned());
1965 task.diagnostic = None;
1966 task.attempts = task.attempts.saturating_sub(1);
1967 if block.kind == "disk pressure" {
1968 task.hold_machine(Some(format!(
1969 "{}{}",
1970 crate::disk::RUNNING_PRESSURE,
1971 block.detail
1972 )));
1973 } else {
1974 task.hold_machine(Some(format!(
1975 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1976 no agent was asked and no attempt was spent",
1977 crate::cache::RESOURCE_HOLD,
1978 block.kind,
1979 block.node
1980 )));
1981 }
1982 if let Some(pass) = state.passes.last() {
1983 task.record_pass_at(
1984 &state.id,
1985 pass.index,
1986 Some(pass.started_at),
1987 attempts_before,
1988 None,
1989 );
1990 }
1991 return;
1992 }
1993 let unresolved = task.hold_notice_cause.clone();
1994 if !verdict.parked
1995 && task
1996 .pr_recovery
1997 .as_ref()
1998 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
1999 && matches!(
2000 verdict.status,
2001 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2002 )
2003 {
2004 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2005 task.succeed();
2006 } else {
2007 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2008 }
2009 } else if !verdict.parked
2010 && task.pr_recovery.is_some()
2011 && matches!(
2012 verdict.status,
2013 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2014 )
2015 && !verdict.quota_hit
2016 {
2017 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2018 } else {
2019 settle_in(task, verdict, detail, max_attempts, p);
2020 }
2021 if !verdict.parked
2022 && !verdict.quota_hit
2023 && matches!(
2024 verdict.status,
2025 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2026 )
2027 {
2028 task.recovery_failed(state);
2029 }
2030 if state.prep_failure.is_some() {
2031 task.hold_notice_cause = unresolved;
2034 task.startup_notice_cause("startup-candidates");
2035 }
2036 if let Some(pass) = state.passes.last() {
2039 let parked = match &pass.end {
2040 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2041 _ => None,
2042 };
2043 task.record_pass_at(
2044 &state.id,
2045 pass.index,
2046 Some(pass.started_at),
2047 attempts_before,
2048 parked,
2049 );
2050 }
2051 if task.status == TaskStatus::Held {
2052 if state.prep_failure.is_none() {
2053 task.notice_cause(
2054 label(verdict.status),
2055 Some(&state.id),
2056 state.passes.last().map(|p| p.index),
2057 );
2058 }
2059 task.diagnostic = diagnostic(state);
2060 note_open_question(task, &state.id, p);
2061 }
2062}
2063
2064fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2079 let Some(home) = crate::run::try_home() else {
2080 return;
2081 };
2082 let open = Questions::at(home.join("questions")).open_for(run);
2083 let Some(q) = open.first() else {
2084 return;
2085 };
2086 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2087 decisions.sort_unstable();
2088 task.notice_cause(
2089 &format!("questions:{}", decisions.join(",")),
2090 Some(run),
2091 None,
2092 );
2093 let base = task.hold_reason.clone().unwrap_or_default();
2094 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2095}
2096
2097fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2108 match last_run {
2109 Some(state) => {
2110 let verdict = Verdict {
2111 status: state.status,
2112 left_pr: state.pr.is_some(),
2113 quota_hit: !state.quota.is_empty(),
2114 parked: state.parked,
2115 no_viable_candidates: state.viable().is_empty(),
2116 };
2117 let detail = format!(
2118 "{}{}",
2119 phrases(&state.config.graph.language).recovered_running,
2120 describe(&state)
2121 );
2122 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2123 }
2124 None => {
2125 let why = phrases(language).no_run_to_recover;
2128 task.last_error = Some(why.to_owned());
2129 task.notice_cause("missing-run", None, None);
2132 task.hold_machine(Some(why.to_owned()));
2133 }
2134 }
2135}
2136
2137fn reclaim_orphaned_running(
2159 queue: &Queue,
2160 max_attempts: usize,
2161 live_workers: &std::collections::BTreeSet<String>,
2162) -> Vec<String> {
2163 let mut reclaimed = Vec::new();
2164 for listed in queue.list() {
2165 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2166 continue;
2167 }
2168 let Ok(_claim) = queue.claim(&listed.id) else {
2169 continue;
2170 };
2171 let Ok(mut task) = queue.get(&listed.id) else {
2175 continue;
2176 };
2177 if task.status != TaskStatus::Running {
2178 continue;
2179 }
2180 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2181 if let Some(state) = &last_run
2191 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2192 {
2193 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2194 }
2195 let language = if last_run.is_none() {
2196 language_of(&task, Path::new("."))
2197 } else {
2198 String::new()
2199 };
2200 reclaim(&mut task, last_run, max_attempts, &language);
2201 if task.status == TaskStatus::Done {
2202 supersede_prior_runs(&task, &crate::run::home());
2203 }
2204 record(queue, &mut task);
2205 reclaimed.push(task.id.clone());
2206 }
2207 reclaimed
2208}
2209
2210fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2242 reclaim_abandoned_runs_with(
2243 home,
2244 now,
2245 crate::proc::pid_status,
2246 crate::proc::process_started_at,
2247 )
2248}
2249
2250fn reclaim_abandoned_runs_with<F, G>(
2256 home: &Path,
2257 now: Timestamp,
2258 query: F,
2259 identity: G,
2260) -> Vec<String>
2261where
2262 F: Fn(u32) -> Option<bool>,
2263 G: Fn(u32) -> Option<String>,
2264{
2265 let mut abandoned = Vec::new();
2266 for entry in std::fs::read_dir(home.join("runs"))
2267 .into_iter()
2268 .flatten()
2269 .flatten()
2270 {
2271 let id = entry.file_name().to_string_lossy().into_owned();
2272 if !crate::run::is_run_id(&id) {
2273 continue;
2274 }
2275 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2278 continue;
2279 };
2280 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2288 continue;
2289 };
2290 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2291 continue;
2292 };
2293 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2294 continue;
2295 }
2296 let daemon_claims = is_working_on(home, &id, now);
2308 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2309 != crate::run::Liveness::Dead
2310 {
2311 continue;
2312 }
2313 state.abandon("daemon");
2314 if let Err(e) = state.save_under(home) {
2315 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2316 continue;
2317 }
2318 if let Some(notice) = notices::run_ended(&state) {
2321 notices::raise_in(home, notice);
2322 }
2323 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2331 tracing::warn!("abandon questions for {id}: {e:#}");
2332 }
2333 abandoned.push(id);
2334 }
2335 abandoned
2336}
2337
2338#[derive(Debug, Clone, Serialize, Deserialize)]
2339struct TaskWorker {
2340 opts: Opts,
2341 queue: PathBuf,
2342 task: String,
2343 #[serde(default)]
2344 lane: PermitKind,
2345 #[serde(default)]
2346 standalone: bool,
2347}
2348
2349fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2350 Ok(crate::worker::active(home)?
2351 .into_iter()
2352 .filter(|job| job.kind == "task")
2353 .collect())
2354}
2355
2356fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2357 task_jobs(home)?
2358 .into_iter()
2359 .filter_map(
2360 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2361 Ok(request) if !request.standalone => Some(Ok(job)),
2362 Ok(_) => None,
2363 Err(error) => Some(Err(error.into())),
2364 },
2365 )
2366 .collect()
2367}
2368
2369#[must_use]
2371pub fn managed(home: &Path) -> bool {
2372 match crate::worker::active(home) {
2373 Err(error) => {
2374 tracing::error!("cannot determine scheduler ownership: {error:#}");
2375 true
2376 }
2377 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2378 Ok(_) => {
2379 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2380 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2381 }
2382 }
2383}
2384
2385pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2387 anyhow::ensure!(
2388 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2389 "the scheduler is stopping; wait for retirement before starting it again"
2390 );
2391 crate::worker::spawn(
2392 home,
2393 "scheduler",
2394 "scheduler",
2395 serde_json::to_value(opts.absolute()?)?,
2396 )
2397 .await
2398}
2399
2400fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2401 home.join("worker-parks").join(format!("{}.json", job.id))
2402}
2403
2404fn request_task_park(
2405 home: &Path,
2406 job: &crate::worker::Job,
2407 cause: crate::run::ParkCause,
2408) -> Result<()> {
2409 let path = park_request_path(home, job);
2410 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2411 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2412 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2413}
2414
2415pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2417 if control == crate::worker::Control::Park {
2418 for job in owned_task_jobs(home)? {
2419 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2420 }
2421 }
2422 crate::worker::request(home, "scheduler", control)
2423}
2424
2425pub async fn restart_detached(home: &Path) -> Result<()> {
2427 let active = crate::worker::active(home)?;
2428 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2429 if prior.is_none() && !managed(home) {
2430 return Ok(());
2431 }
2432 let opts: Opts = match prior {
2433 Some(job) => serde_json::from_value(job.payload)?,
2434 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2435 };
2436 request_control(home, crate::worker::Control::Stop)?;
2437 loop {
2438 let reserved = crate::worker::active(home)?
2439 .iter()
2440 .any(|job| job.kind == "scheduler");
2441 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2442 break;
2443 }
2444 tokio::time::sleep(Duration::from_millis(100)).await;
2445 }
2446 spawn_detached(home, &opts).await?;
2447 Ok(())
2448}
2449
2450pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2452 let opts: Opts = serde_json::from_value(payload)?;
2453 let _ = home;
2454 serve_until(opts, Stop::new()).await
2455}
2456
2457pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2459 let request: TaskWorker = serde_json::from_value(payload)?;
2460 let queue = Queue::at(request.queue);
2461 let key = format!("task:{}", request.task);
2462 let job = task_jobs(home)?
2463 .into_iter()
2464 .find(|job| job.key == key)
2465 .context("task worker reservation is missing")?;
2466 let park_path = park_request_path(home, &job);
2467 let _claim = loop {
2470 match queue.claim(&request.task) {
2471 Ok(claim) => break claim,
2472 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2475 return Err(e.context("claim the task for its worker"));
2476 }
2477 Err(_) => {
2478 sweep_stale_claims(&queue, STALE_CLAIM);
2479 tokio::time::sleep(Duration::from_millis(50)).await;
2480 }
2481 }
2482 };
2483 let mut task = queue.get(&request.task)?;
2484 if !task.status.runnable()
2485 || (!request.standalone
2486 && request.lane
2487 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2488 {
2489 return Ok(());
2490 }
2491 let stop = Stop::new();
2492 let watcher = {
2493 let home = home.to_path_buf();
2494 let stop = stop.clone();
2495 let park_path = park_path.clone();
2496 tokio::spawn(async move {
2497 loop {
2498 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2499 let cause = std::fs::read(&park_path)
2500 .ok()
2501 .and_then(|body| serde_json::from_slice(&body).ok())
2502 .unwrap_or(crate::run::ParkCause::Operator);
2503 stop.park(cause);
2504 return;
2505 }
2506 tokio::time::sleep(Duration::from_millis(200)).await;
2507 }
2508 })
2509 };
2510 let quota = if request.standalone {
2511 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2512 } else {
2513 let status = Arc::new(Mutex::new(Status::new()));
2514 Box::pin(attempt(
2515 &request.opts,
2516 &queue,
2517 &status,
2518 &stop,
2519 crate::graph::Pause::new(),
2520 &mut task,
2521 ))
2522 .await
2523 };
2524 watcher.abort();
2525 let _ = std::fs::remove_file(park_path);
2526 persist_cooldown(home, "a).await?;
2527 Ok(())
2528}
2529
2530pub async fn spawn_claimed(
2533 home: &Path,
2534 opts: &Opts,
2535 queue: &Queue,
2536 task: &Task,
2537 claim: crate::queue::Claim,
2538) -> Result<crate::worker::Job> {
2539 let request = TaskWorker {
2540 opts: opts.absolute()?,
2541 queue: std::path::absolute(queue.root())?,
2542 task: task.id.clone(),
2543 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2544 standalone: true,
2545 };
2546 let job = crate::worker::spawn(
2547 home,
2548 "task",
2549 &format!("task:{}", task.id),
2550 serde_json::to_value(request)?,
2551 )
2552 .await?;
2553 drop(claim);
2554 Ok(job)
2555}
2556
2557pub async fn run_detached_claimed(
2560 opts: &Opts,
2561 queue: &Queue,
2562 task: &mut Task,
2563 claim: crate::queue::Claim,
2564) -> Result<()> {
2565 let home = crate::run::home();
2566 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2567 loop {
2568 crate::worker::active(&home)?;
2571 let current = crate::worker::get(&home, &job.id)?;
2572 if matches!(
2573 current.status,
2574 crate::worker::Status::Finished | crate::worker::Status::Failed
2575 ) {
2576 *task = queue.get(&task.id)?;
2577 anyhow::ensure!(
2578 current.status != crate::worker::Status::Failed,
2579 "task worker failed: {}",
2580 current.result.as_deref().unwrap_or("unknown failure")
2581 );
2582 return Ok(());
2583 }
2584 tokio::time::sleep(Duration::from_millis(200)).await;
2585 }
2586}
2587
2588async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2589 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2590 let _lease = loop {
2591 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2592 break lease;
2593 }
2594 tokio::time::sleep(Duration::from_millis(10)).await;
2595 };
2596 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2597 let path = home.join("task-quota-cooldown.json");
2598 let temporary = path.with_extension("json.tmp");
2599 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2600 std::fs::rename(temporary, path)?;
2601 }
2602 }
2603 Ok(())
2604}
2605
2606fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2607 match std::fs::read(home.join("task-quota-cooldown.json")) {
2608 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2609 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2610 Err(error) => Err(error.into()),
2611 }
2612}
2613
2614pub async fn serve(opts: Opts) -> Result<()> {
2620 serve_until(opts, Stop::new()).await
2621}
2622
2623pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2634 let opts = opts.absolute()?;
2635 let home = crate::run::home();
2636 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2637 .context("another scheduler already owns this home")?;
2638 if let Some(reading) = read_status(&home)
2639 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2640 {
2641 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2642 }
2643 if !crate::worker::active(&home)?
2644 .iter()
2645 .any(|job| job.kind == "scheduler")
2646 {
2647 crate::worker::clear_control(&home, "scheduler")?;
2648 }
2649 std::fs::write(
2650 home.join("scheduler-options.json"),
2651 serde_json::to_vec(&opts)?,
2652 )?;
2653 let control = {
2654 let home = home.clone();
2655 let stop = stop.clone();
2656 tokio::spawn(async move {
2657 loop {
2658 match crate::worker::control(&home, "scheduler").ok().flatten() {
2659 Some(crate::worker::Control::Park) => {
2660 let _ = request_control(&home, crate::worker::Control::Park);
2661 stop.park(crate::run::ParkCause::Operator);
2662 break;
2663 }
2664 Some(crate::worker::Control::Stop) => {
2665 stop.stop();
2666 break;
2667 }
2668 None => {}
2669 }
2670 tokio::time::sleep(Duration::from_millis(100)).await;
2671 }
2672 })
2673 };
2674 let signal = {
2675 let stop = stop.clone();
2676 tokio::spawn(async move {
2677 if tokio::signal::ctrl_c().await.is_ok() {
2678 stop.stop();
2679 tracing::info!("scheduler shutdown requested; independent workers continue");
2680 }
2681 })
2682 };
2683
2684 let worktrees_root = opts
2685 .worktrees_root
2686 .clone()
2687 .unwrap_or_else(crate::run::default_worktree_root);
2688 let outcome = drive(
2689 &opts,
2690 &Queue::open(),
2691 &status_path(),
2692 &crate::run::home(),
2693 &worktrees_root,
2694 &stop,
2695 )
2696 .await;
2697
2698 signal.abort();
2699 control.abort();
2700 outcome
2701}
2702
2703async fn drive(
2716 opts: &Opts,
2717 queue: &Queue,
2718 status_file: &Path,
2719 home: &Path,
2720 worktrees_root: &Path,
2721 stop: &Stop,
2722) -> Result<()> {
2723 let recovery_cfg = prepare(&opts.repo, opts).ok();
2731 let daemon_cfg = recovery_cfg
2732 .as_ref()
2733 .map(|c| c.daemon.clone())
2734 .unwrap_or_default();
2735 let reconciled = crate::driver::reconcile(home);
2736 if !reconciled.is_empty() {
2737 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2738 }
2739 let mut initial_status = Status::new();
2740 let owned = owned_task_jobs(home)?;
2741 initial_status.current = current_work(home, Timestamp::now())
2742 .into_iter()
2743 .filter(|current| {
2744 owned
2745 .iter()
2746 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2747 })
2748 .collect();
2749 initial_status.idle = initial_status.current.is_empty();
2750 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2751 recovery_cfg.as_ref(),
2752 daemon_cfg.max_deputies,
2753 ));
2754 let status = Arc::new(Mutex::new(initial_status));
2755 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2756 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2757
2758 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2764
2765 tracing::info!(
2766 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2767 queue.root().display(),
2768 opts.poll.as_secs(),
2769 opts.max_attempts,
2770 concurrency,
2771 if daemon_cfg.pause_for_interrupts {
2772 ", interrupts enabled"
2773 } else {
2774 ""
2775 }
2776 );
2777
2778 let outcome = tokio::select! {
2779 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2780 max_concurrent: concurrency,
2781 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2782 recovery_cfg,
2783 }) => outcome,
2784 () = async {
2785 while !stop.stopped() {
2786 tokio::time::sleep(Duration::from_millis(25)).await;
2787 }
2788 } => {
2789 if stop.parking() {
2790 request_control(home, crate::worker::Control::Park)?;
2791 }
2792 Ok(())
2793 }
2794 };
2795
2796 stop.busy.store(0, Ordering::SeqCst);
2797 beat.abort();
2798 clear_status_at(status_file);
2799 outcome
2800}
2801
2802pub(crate) fn release_watch_settings_from_config(
2805 home: &Path,
2806 repo: &Path,
2807 cfg: &Config,
2808) -> (Vec<PathBuf>, u64) {
2809 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2810 .into_iter()
2811 .map(|r| r.path)
2812 .collect();
2813 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2816 .list()
2817 .into_iter()
2818 .map(|t| t.repo)
2819 .collect();
2820 let runs = home.join("runs");
2821 for id in crate::run::list_ids_in(&runs) {
2822 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2823 .ok()
2824 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2825 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2826 known.extend(repo);
2827 }
2828 for k in known {
2829 let k = k.canonicalize().unwrap_or(k);
2830 if k.join(".git").exists() && !paths.contains(&k) {
2831 paths.push(k);
2832 }
2833 }
2834 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2835 if !paths.contains(&own) {
2836 paths.push(own);
2837 }
2838 (paths, cfg.daemon.release_stall_minutes)
2839}
2840
2841const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2843
2844async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2850 loop {
2851 tokio::time::sleep(HEARTBEAT).await;
2852 let snapshot = {
2853 let mut guard = lock(&status);
2854 guard.updated_at = Timestamp::now();
2855 guard.clone()
2856 };
2857 if let Err(e) = write_status_to(&path, &snapshot) {
2858 tracing::warn!("could not refresh the daemon status file: {e:#}");
2861 }
2862 }
2863}
2864
2865#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2868enum LandResume {
2869 NotLanding,
2872 StillWaiting,
2877 Ready,
2881}
2882
2883fn land_resume_state(task: &Task) -> LandResume {
2887 let Some(run_id) = task.runs.last() else {
2888 return LandResume::NotLanding;
2889 };
2890 let Ok(state) = RunState::load(run_id) else {
2891 return LandResume::NotLanding;
2892 };
2893 if !state.parked {
2894 return LandResume::NotLanding;
2895 }
2896 if state.status == RunStatus::Landing
2897 && state.land_approval.as_ref().is_some_and(|binding| {
2898 ask::Questions::open()
2899 .get(&binding.question)
2900 .is_ok_and(|q| {
2901 q.run == state.id
2902 && q.resolution()
2903 .as_deref()
2904 .is_some_and(crate::merge_decision::stops)
2905 })
2906 })
2907 {
2908 return LandResume::Ready;
2909 }
2910 if !implementers_match(task, &state) {
2911 return LandResume::NotLanding;
2912 }
2913 let node = if state.status == RunStatus::Landing {
2917 land::APPROVAL_NODE
2918 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2919 crate::github_text::ASK_NODE
2920 } else {
2921 return LandResume::NotLanding;
2922 };
2923 let store = ask::Questions::open();
2924 let waiting = store
2925 .list()
2926 .into_iter()
2927 .filter(|q| &q.run == run_id && q.node == node)
2928 .max_by(|a, b| a.id.cmp(&b.id));
2929 let Some(mut q) = waiting else {
2930 return LandResume::Ready;
2931 };
2932 if !q.status.open() {
2933 if q.resolution().as_deref() == Some(land::REPAIR) {
2936 return if state.liveness(false) == Liveness::Live {
2937 LandResume::StillWaiting
2938 } else {
2939 LandResume::NotLanding
2940 };
2941 }
2942 return LandResume::Ready;
2943 }
2944 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2951 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2952 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2953 q.abandon(format!(
2954 "no answer within {}s of asking",
2955 timeout.as_secs().max(1)
2956 ));
2957 if store.put(&mut q).is_ok() {
2960 return LandResume::Ready;
2961 }
2962 }
2963 LandResume::StillWaiting
2964}
2965
2966const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2974
2975const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2987
2988#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3001enum Interrupt {
3002 Idle,
3004 Parking {
3016 parked: Vec<String>,
3017 interrupt_task: String,
3018 },
3019 Running {
3027 parked: Vec<String>,
3028 interrupt_task: String,
3029 },
3030 Resuming { parked: Vec<String> },
3037}
3038
3039fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3060 match state {
3061 Interrupt::Idle => {
3062 if in_flight.len() != 1 {
3074 return Interrupt::Idle;
3075 }
3076 match runnable.iter().find(|t| t.interrupt) {
3077 Some(t) => Interrupt::Parking {
3078 parked: in_flight.to_vec(),
3079 interrupt_task: t.id.clone(),
3080 },
3081 None => Interrupt::Idle,
3082 }
3083 }
3084 Interrupt::Parking {
3085 parked,
3086 interrupt_task,
3087 } => {
3088 if in_flight.iter().any(|id| parked.contains(id)) {
3089 Interrupt::Parking {
3091 parked,
3092 interrupt_task,
3093 }
3094 } else if in_flight.contains(&interrupt_task) {
3095 Interrupt::Running {
3096 parked,
3097 interrupt_task,
3098 }
3099 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3100 Interrupt::Parking {
3104 parked,
3105 interrupt_task,
3106 }
3107 } else {
3108 Interrupt::Resuming { parked }
3113 }
3114 }
3115 Interrupt::Running {
3116 parked,
3117 interrupt_task,
3118 } => {
3119 if in_flight.contains(&interrupt_task) {
3120 Interrupt::Running {
3121 parked,
3122 interrupt_task,
3123 }
3124 } else {
3125 Interrupt::Resuming { parked }
3131 }
3132 }
3133 Interrupt::Resuming { parked } => {
3134 if in_flight.iter().any(|id| parked.contains(id)) {
3135 Interrupt::Idle
3141 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3142 Interrupt::Resuming { parked }
3143 } else {
3144 Interrupt::Idle
3147 }
3148 }
3149 }
3150}
3151
3152fn advance_interrupt_tick(
3158 enabled: bool,
3159 state: Interrupt,
3160 in_flight: &[String],
3161 runnable: &[Task],
3162) -> Interrupt {
3163 if !enabled {
3164 return Interrupt::Idle;
3165 }
3166 advance_interrupt(state, in_flight, runnable)
3167}
3168
3169fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3174 match state {
3175 Interrupt::Idle => candidates,
3176 Interrupt::Parking {
3177 parked,
3178 interrupt_task,
3179 } => {
3180 if in_flight.iter().any(|id| parked.contains(id)) {
3181 Vec::new()
3182 } else {
3183 candidates
3184 .into_iter()
3185 .filter(|t| &t.id == interrupt_task)
3186 .collect()
3187 }
3188 }
3189 Interrupt::Running { .. } => Vec::new(),
3190 Interrupt::Resuming { parked } => candidates
3198 .into_iter()
3199 .find(|t| parked.contains(&t.id))
3200 .into_iter()
3201 .collect(),
3202 }
3203}
3204
3205struct DispatchLimits {
3209 max_concurrent: usize,
3212 pause_for_interrupts: bool,
3214 recovery_cfg: Option<Config>,
3215}
3216
3217#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3226enum PermitKind {
3227 None,
3232 Urgent,
3239 #[default]
3242 Ordinary,
3243}
3244
3245fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3248 if priority {
3249 PermitKind::None
3250 } else if urgent {
3251 PermitKind::Urgent
3252 } else {
3253 PermitKind::Ordinary
3254 }
3255}
3256
3257fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3258 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3259 (0usize, 0usize),
3260 |(ordinary, urgent), request| match request.lane {
3261 PermitKind::Ordinary => (ordinary + 1, urgent),
3262 PermitKind::Urgent => (ordinary, urgent + 1),
3263 PermitKind::None => (ordinary, urgent),
3264 },
3265 );
3266 (
3267 limit.saturating_sub(ordinary),
3268 1usize.saturating_sub(urgent),
3269 )
3270}
3271
3272struct DispatchClaim {
3273 task: Task,
3274 claim: crate::queue::Claim,
3275 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3276}
3277
3278fn claim_dispatch(
3281 queue: &Queue,
3282 candidate: &Task,
3283 cooling_down: bool,
3284 ordinary: &Arc<tokio::sync::Semaphore>,
3285 urgent: &Arc<tokio::sync::Semaphore>,
3286) -> Option<DispatchClaim> {
3287 let Ok(claim) = queue.claim(&candidate.id) else {
3288 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3289 return None;
3290 };
3291 let task = match queue.get(&candidate.id) {
3292 Ok(t) if t.status.runnable() => t,
3293 Ok(_) => return None,
3294 Err(e) => {
3295 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3296 return None;
3297 }
3298 };
3299 let resume = land_resume_state(&task);
3300 if resume == LandResume::StillWaiting {
3301 return None;
3302 }
3303 let priority = resume == LandResume::Ready;
3304 if !priority && cooling_down {
3305 return None;
3306 }
3307 let permit = match permit_kind(priority, task.urgent) {
3310 PermitKind::None => None,
3311 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3312 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3313 };
3314 let stamp = Timestamp::now();
3317 let mut task = task;
3318 task.dispatched_at = Some(stamp);
3319 if let Err(e) = queue.modify(&task.id, |t| {
3320 t.dispatched_at = Some(stamp);
3321 true
3322 }) {
3323 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3324 }
3325 Some(DispatchClaim {
3326 task,
3327 claim,
3328 permit,
3329 })
3330}
3331
3332#[derive(Serialize, Deserialize)]
3351struct ServiceWorker {
3352 opts: Opts,
3353 queue: PathBuf,
3354 worktrees_root: PathBuf,
3355 operation: String,
3356}
3357
3358fn load_conductor(home: &Path) -> Result<Conductor> {
3359 match std::fs::read(home.join("conductor-state.json")) {
3360 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3361 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3362 Err(error) => Err(error.into()),
3363 }
3364}
3365
3366fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3367 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3368 .into_iter()
3369 .map(|c| c.task)
3370 .collect();
3371 let stalled = stalled_tasks(queue, home, Timestamp::now());
3372 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3373 let finished = finished_tasks(queue)
3374 .into_iter()
3375 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3376 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3377 .collect();
3378 let queued = queued_tasks(queue)
3379 .into_iter()
3380 .filter(|t| !live.contains(&t.id))
3381 .collect();
3382 (queued, stalled, finished)
3383}
3384
3385async fn start_service(
3386 home: &Path,
3387 opts: &Opts,
3388 queue: &Queue,
3389 worktrees_root: &Path,
3390 operation: &str,
3391 cfg: Option<&Config>,
3392 busy: bool,
3393) -> Result<()> {
3394 let fetch_cfg = if operation == "fetch" {
3397 match prepare(&opts.repo, opts) {
3398 Ok(cfg) => Some(cfg),
3399 Err(error) => {
3400 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3401 return Ok(());
3402 }
3403 }
3404 } else {
3405 None
3406 };
3407 let cfg = fetch_cfg.as_ref().or(cfg);
3408 let ticks = home.join("service-ticks");
3409 std::fs::create_dir_all(&ticks)?;
3410 let receipt = ticks.join(format!("daemon-{operation}"));
3411 let last = std::fs::read(&receipt)
3412 .ok()
3413 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3414 let interval = match operation {
3415 "disk-pressure" => 30,
3416 "cache" => CACHE_CHECK_INTERVAL_SECS,
3417 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3418 _ => opts.poll.as_secs().max(1),
3419 };
3420 let now = Timestamp::now();
3421 if !cache_check_due(last, now, interval) {
3422 return Ok(());
3423 }
3424 let key = format!("service:daemon-{operation}");
3425 if crate::worker::active(home)?
3426 .iter()
3427 .any(|job| job.key == key)
3428 {
3429 return Ok(());
3430 }
3431 let eligible = match operation {
3432 "conductor" => {
3433 let (queued, stalled, finished) = conductor_work(queue, home);
3434 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3435 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3436 }
3437 "maintenance" => Questions::at(home.join("questions"))
3438 .list()
3439 .iter()
3440 .any(|q| {
3441 q.chosen_action().is_some_and(|action| action.maintenance())
3442 && !home
3443 .join("choice-actions")
3444 .join(format!("{}.json", q.id))
3445 .exists()
3446 }),
3447 "housekeeping" => !busy && runnable(queue).is_empty(),
3448 "cache" => {
3449 !busy
3450 && !runnable(queue).is_empty()
3451 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3452 }
3453 "disk-pressure" => busy,
3454 "fetch" => {
3455 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3456 }
3457 _ => false,
3458 };
3459 if !eligible {
3460 return Ok(());
3461 }
3462 let request = ServiceWorker {
3463 opts: opts.absolute()?,
3464 queue: std::path::absolute(queue.root())?,
3465 worktrees_root: std::path::absolute(worktrees_root)?,
3466 operation: operation.to_owned(),
3467 };
3468 if let Err(error) =
3469 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3470 {
3471 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3472 }
3473 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3474 Ok(())
3475}
3476
3477pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3480 let request: ServiceWorker = serde_json::from_value(payload)?;
3481 let opts = &request.opts;
3482 let queue = Queue::at(request.queue);
3483 let questions = Questions::at(home.join("questions"));
3484 match request.operation.as_str() {
3485 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3486 "conductor" => {
3487 let path = home.join("conductor-state.json");
3488 let mut conductor = load_conductor(home)?;
3489 let (queued, stalled, finished) = conductor_work(&queue, home);
3490 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3491 && conductor.worth_a_look(&queue, &stalled, &finished)
3492 {
3493 let cfg = prepare(&opts.repo, opts)?;
3494 conductor
3495 .maybe_run(
3496 &cfg,
3497 &opts.repo,
3498 &queue,
3499 &questions,
3500 home,
3501 &queued,
3502 &stalled,
3503 &finished,
3504 opts.max_attempts,
3505 )
3506 .await;
3507 let temporary = path.with_extension("json.tmp");
3508 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3509 std::fs::rename(temporary, path)?;
3510 }
3511 }
3512 "fetch" => {
3513 let cfg = prepare(&opts.repo, opts)?;
3514 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3515 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3516 tracing::debug!("fetch origins: {outcome:?}");
3517 }
3518 }
3519 "cache" => {
3520 if task_jobs(home)?.is_empty() {
3521 let mut checked = None;
3522 maybe_prune_cache_between_runs(
3523 &opts.repo,
3524 opts,
3525 home,
3526 &Stop::new(),
3527 &mut checked,
3528 Timestamp::now(),
3529 )
3530 .await;
3531 }
3532 }
3533 "disk-pressure" => {
3534 if let Ok(cfg) = prepare(&opts.repo, opts) {
3535 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3536 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3537 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3538 }
3539 for current in current_work(home, Timestamp::now()) {
3540 if let Ok(state) = RunState::load_under(¤t.run, home)
3541 && let Some(reason) = disk_gate(&state.repo, &state.config)
3542 {
3543 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3544 job.key.strip_prefix("task:") == Some(current.task.as_str())
3545 }) {
3546 request_task_park(
3547 home,
3548 &job,
3549 crate::run::ParkCause::Interrupt {
3550 reason: reason.clone(),
3551 },
3552 )?;
3553 }
3554 notices::raise_in(
3555 home,
3556 Notice::warn(
3557 "disk:pressure",
3558 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3559 ),
3560 );
3561 }
3562 }
3563 }
3564 }
3565 "housekeeping" => {
3566 if task_jobs(home)?.is_empty() {
3567 if let Ok(cfg) = prepare(&opts.repo, opts) {
3568 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3569 }
3570 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3571 resweep_superseded_attempts(&queue, home);
3572 triage_held(&queue, home, opts).await;
3573 }
3574 }
3575 operation => bail!("unknown daemon service {operation}"),
3576 }
3577 Ok(())
3578}
3579
3580fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3584 let key = format!("task:{}", request.task);
3585 let payload = serde_json::to_value(request)?;
3586 if crate::worker::active(home)?
3587 .iter()
3588 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3589 {
3590 tracing::warn!(
3591 "task {} acquired a competing worker reservation: {error:#}",
3592 crate::run::short_of(&request.task)
3593 );
3594 return Ok(());
3595 }
3596 tracing::error!("task worker startup stopped dispatch: {error:#}");
3597 notices::raise_in(
3598 home,
3599 Notice::error(
3600 "loop:worker-startup",
3601 "A task worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Startup details are kept in the local execution log.",
3602 ),
3603 );
3604 Err(error).with_context(|| {
3605 format!(
3606 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3607 crate::run::short_of(&request.task)
3608 )
3609 })
3610}
3611
3612async fn poll(
3613 opts: &Opts,
3614 queue: &Queue,
3615 status: &Arc<Mutex<Status>>,
3616 home: &Path,
3617 worktrees_root: &Path,
3618 stop: &Stop,
3619 limits: DispatchLimits,
3620) -> Result<()> {
3621 let mut attempted = Vec::new();
3622 let interrupt_path = home.join("scheduler-interrupt.json");
3623 let mut interrupt = match std::fs::read(&interrupt_path) {
3624 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3625 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3626 Err(error) => return Err(error.into()),
3627 };
3628 let mut previous = std::collections::BTreeSet::<String>::new();
3629 while !stop.stopped() {
3630 let jobs: Vec<_> = crate::worker::active(home)?
3631 .into_iter()
3632 .filter(|job| job.kind == "task")
3633 .collect();
3634 let requests: Vec<_> = jobs
3635 .iter()
3636 .map(|job| {
3637 serde_json::from_value::<TaskWorker>(job.payload.clone())
3638 .with_context(|| format!("read live task worker {}", job.id))
3639 })
3640 .collect::<Result<_>>()?;
3641 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3642 let owned: std::collections::BTreeSet<_> = requests
3643 .iter()
3644 .filter(|r| !r.standalone)
3645 .map(|r| r.task.clone())
3646 .collect();
3647 let current: Vec<_> = requests
3648 .iter()
3649 .filter(|request| !request.standalone)
3650 .map(|request| Current {
3651 task: request.task.clone(),
3652 run: queue
3653 .get(&request.task)
3654 .ok()
3655 .and_then(|task| task.runs.last().cloned())
3656 .unwrap_or_default(),
3657 })
3658 .collect();
3659 {
3660 let mut snapshot = lock(status);
3661 snapshot.polls += 1;
3662 snapshot.completed += previous.difference(&owned).count();
3663 snapshot.current = current;
3664 snapshot.idle = owned.is_empty();
3665 }
3666 previous = owned.clone();
3667 stop.busy.store(owned.len(), Ordering::SeqCst);
3668
3669 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3672 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3673 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3674 let mut permits = Vec::new();
3675 sweep_stale_claims(queue, STALE_CLAIM);
3676 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3677 reclaim_abandoned_runs(home, Timestamp::now());
3678 let questions = Questions::at(home.join("questions"));
3679
3680 crate::task_chat::sweep(
3683 queue,
3684 &crate::talk::Talks::at(home.join("talks")),
3685 &questions,
3686 home,
3687 );
3688 repair_choice_offers(&questions);
3689 resolve_blockers(queue, &questions);
3690 apply_task_choice_actions(queue, &questions, home);
3691 triage::apply_answers(queue, &questions);
3692 reconcile_task_questions(queue, &questions);
3693 let talks = crate::talk::Talks::at(home.join("talks"));
3694 crate::chat_notify::sweep(queue, &talks, home);
3695 crate::chat_watch::sweep(home, &talks);
3696 let mut turns = tokio::task::JoinSet::new();
3697 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3698 while turns.join_next().await.is_some() {}
3700
3701 crate::consult::start_services(
3702 home,
3703 &opts.repo,
3704 limits.recovery_cfg.as_ref(),
3705 opts.config.as_deref(),
3706 )
3707 .await?;
3708 for operation in [
3709 "conductor",
3710 "maintenance",
3711 "housekeeping",
3712 "disk-pressure",
3713 "cache",
3714 "fetch",
3715 ] {
3716 start_service(
3717 home,
3718 opts,
3719 queue,
3720 worktrees_root,
3721 operation,
3722 limits.recovery_cfg.as_ref(),
3723 !live.is_empty(),
3724 )
3725 .await?;
3726 }
3727 let latest = crate::worker::active(home)?;
3728 if latest.iter().any(|job| job.key == "service:daemon-cache")
3729 || latest
3730 .iter()
3731 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3732 {
3733 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3734 continue;
3735 }
3736 let candidates: Vec<Task> = runnable(queue)
3737 .into_iter()
3738 .filter(|task| !live.contains(&task.id))
3739 .filter(|task| !opts.once || !attempted.contains(&task.id))
3740 .collect();
3741 let in_flight: Vec<_> = owned.iter().cloned().collect();
3742 let prior_interrupt = std::mem::discriminant(&interrupt);
3743 interrupt = advance_interrupt_tick(
3744 limits.pause_for_interrupts,
3745 interrupt,
3746 &in_flight,
3747 &candidates,
3748 );
3749 if prior_interrupt != std::mem::discriminant(&interrupt) {
3752 let temporary = interrupt_path.with_extension("json.tmp");
3753 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3754 std::fs::rename(temporary, &interrupt_path)?;
3755 }
3756 if let Interrupt::Parking {
3757 parked,
3758 interrupt_task,
3759 } = &interrupt
3760 {
3761 for job in jobs.iter().filter(|job| {
3762 parked
3763 .iter()
3764 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3765 }) {
3766 if crate::worker::control(home, &job.key)?.is_none() {
3767 request_task_park(
3768 home,
3769 job,
3770 crate::run::ParkCause::Interrupt {
3771 reason: format!(
3772 "task {} asked to run first",
3773 crate::run::short_of(interrupt_task)
3774 ),
3775 },
3776 )?;
3777 }
3778 }
3779 }
3780 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3781 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3782 let mut started = false;
3783 let mut resync = false;
3784 for candidate in candidates {
3785 if stop.stopped() {
3786 break;
3787 }
3788 let Some(DispatchClaim {
3789 task,
3790 claim,
3791 permit,
3792 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3793 else {
3794 continue;
3795 };
3796 if crate::worker::active(home)?
3797 .iter()
3798 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3799 {
3800 drop(claim);
3801 resync = true;
3802 break;
3803 }
3804 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3805 let request = TaskWorker {
3806 opts: opts.absolute()?,
3807 queue: std::path::absolute(queue.root())?,
3808 task: task.id.clone(),
3809 lane,
3810 standalone: false,
3811 };
3812 if let Err(error) = crate::worker::spawn(
3815 home,
3816 "task",
3817 &format!("task:{}", task.id),
3818 serde_json::to_value(&request)?,
3819 )
3820 .await
3821 {
3822 task_worker_spawn_error(home, &request, error)?;
3823 drop(claim);
3824 resync = true;
3825 break;
3826 }
3827 drop(claim);
3828 permits.extend(permit);
3829 attempted.push(task.id);
3830 started = true;
3831 }
3832 if started || resync {
3833 continue;
3834 }
3835 if owned.is_empty() && opts.once {
3836 while crate::worker::active(home)?
3837 .iter()
3838 .any(|job| job.key == "service:daemon-housekeeping")
3839 {
3840 stop.idle(RECHECK_WHILE_BUSY).await;
3841 if stop.stopped() {
3842 return Ok(());
3843 }
3844 }
3845 break;
3846 }
3847 stop.idle(if live.is_empty() {
3848 opts.poll
3849 } else {
3850 RECHECK_WHILE_BUSY.min(opts.poll)
3851 })
3852 .await;
3853 }
3854 stop.busy.store(0, Ordering::SeqCst);
3857 Ok(())
3858}
3859
3860async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
3864 let busy = |error: &anyhow::Error| {
3865 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
3866 error.kind() == std::io::ErrorKind::WouldBlock
3867 || cfg!(windows) && error.raw_os_error() == Some(33)
3868 })
3869 };
3870 match crate::pr_head::owner_lock(home, id) {
3871 Ok(lock) => return Ok(lock),
3872 Err(error) if busy(&error) => {}
3873 Err(error) => return Err(error),
3874 }
3875 let wait = RunState::load_under(id, home)?
3876 .config
3877 .graph
3878 .timeout_resource_wait;
3879 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
3880}
3881
3882async fn public_resume_owner(
3886 home: &Path,
3887 runs: &[String],
3888 repo: &Path,
3889) -> Result<Option<RunState>> {
3890 let current_task = crate::worker::current_job(home)?
3894 .filter(|job| job.kind == "task" && job.pid == Some(std::process::id()))
3895 .map(|job| serde_json::from_value::<TaskWorker>(job.payload).map(|request| request.task))
3896 .transpose()?;
3897 let mut owner = None;
3898 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3899 let _lock = resume_owner_lock(home, id).await?;
3900 let state = RunState::load_under(id, home)?;
3901 if state.released()
3902 || state.retired_to.is_some()
3903 || state.pr_taken_over_by.is_some()
3904 || !crate::handover::landing_checkpoint(&state)
3905 {
3906 continue;
3907 }
3908 anyhow::ensure!(
3909 crate::git::toplevel(&state.repo).await?.canonicalize()?
3910 == crate::git::toplevel(repo).await?.canonicalize()?,
3911 "public owner repository differs; inspect run lineage"
3912 );
3913 anyhow::ensure!(
3914 owner.is_none(),
3915 "multiple retained public owners; inspect run lineage"
3916 );
3917 anyhow::ensure!(
3918 crate::handover::can_resume(&state),
3919 "public owner cannot resume"
3920 );
3921 let claimed = current_work(home, Timestamp::now())
3922 .iter()
3923 .any(|claim| claim.run == *id && current_task.as_deref() != Some(claim.task.as_str()));
3924 anyhow::ensure!(
3925 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3926 "public owner's driver is live or unknown; wait before retrying"
3927 );
3928 owner = Some(state);
3929 }
3930 Ok(owner)
3931}
3932
3933async fn attempt(
3939 opts: &Opts,
3940 queue: &Queue,
3941 status: &Arc<Mutex<Status>>,
3942 stop: &Stop,
3943 interrupt_pause: crate::graph::Pause,
3944 task: &mut Task,
3945) -> Vec<QuotaLoss> {
3946 let repo = repo_for(task, &opts.repo);
3947 tracing::info!(
3948 "task {} — {} (repo {})",
3949 task.short(),
3950 task.title,
3951 repo.display()
3952 );
3953
3954 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3955 Ok(owner) => owner,
3956 Err(e) => {
3957 task.hold_machine(Some(format!(
3958 "Cannot verify existing public owner: {e:#}; no attempt started."
3959 )));
3960 record(queue, task);
3961 return Vec::new();
3962 }
3963 };
3964 let resume_runs = public_owner
3965 .as_ref()
3966 .map(|s| vec![s.id.clone()])
3967 .unwrap_or_else(|| task.runs.clone());
3968
3969 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
3973 && previous
3974 .merge_decision
3975 .as_ref()
3976 .is_some_and(|decision| !decision.closed)
3977 && previous.status != RunStatus::Landing
3978 && !crate::merge_decision::resumable_hold(&previous)
3979 {
3980 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3981 record(queue, task);
3982 return Vec::new();
3983 }
3984 let stop_owner = resume_runs
3985 .last()
3986 .and_then(|id| RunState::load(id).ok())
3987 .filter(|state| {
3988 crate::merge_decision::resumable_hold(state)
3989 || state.status == RunStatus::Landing
3990 && state.land_approval.as_ref().is_some_and(|binding| {
3991 Questions::open().get(&binding.question).is_ok_and(|q| {
3992 q.run == state.id
3993 && q.resolution()
3994 .as_deref()
3995 .is_some_and(crate::merge_decision::stops)
3996 })
3997 })
3998 });
3999 if let Some(owner) = stop_owner {
4000 let mut runner = match Runner::resume(&owner.id) {
4001 Ok(runner) => runner,
4002 Err(e) => {
4003 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4004 record(queue, task);
4005 return Vec::new();
4006 }
4007 };
4008 runner.on_pause(stop.pause());
4009 runner.watch_interrupt(interrupt_pause);
4010 let quota_before = runner.state.quota.clone();
4011 task.start(owner.id.clone());
4012 record(queue, task);
4013 lock(status).current.push(Current {
4014 task: task.id.clone(),
4015 run: owner.id,
4016 });
4017 runner.state.next_pass_task_entry = true;
4018 let result = runner.execute().await;
4019 return finish_attempt(
4020 opts.max_attempts,
4021 queue,
4022 task,
4023 &runner.state,
4024 "a_before,
4025 result,
4026 );
4027 }
4028
4029 if let Some(owner) = &public_owner
4030 && (task.fresh_start || !may_resume_run(owner, task))
4031 {
4032 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4033 record(queue, task);
4034 return Vec::new();
4035 }
4036
4037 let unfinished = (!task.fresh_start)
4038 .then(|| unfinished_run(&resume_runs, task.short()))
4039 .flatten()
4040 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4041 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4042 if !task.fresh_start
4043 && task.pr_recovery.is_none()
4044 && task.review_branch.is_none()
4045 && task.resume_override.is_none()
4046 && previous
4047 .as_ref()
4048 .is_some_and(|run| run.retired_to.is_some())
4049 && unfinished.is_none()
4050 {
4051 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4052 record(queue, task);
4053 return Vec::new();
4054 }
4055
4056 let saved_recovery = previous.as_ref().filter(|run| {
4057 task.replacement.is_none()
4058 && task.pr_recovery.as_ref().is_some_and(|selected| {
4059 selected.check_task(&task.id, &task.instruction).is_ok()
4060 && selected_starter(selected, Some(run), unfinished.as_deref())
4061 == Starter::Resume(run.id.clone())
4062 })
4063 });
4064
4065 let explicit = match &task.overrides {
4069 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4070 Some(_) => opts.config.as_deref(),
4071 None => opts.config.as_deref(),
4072 };
4073 let (mut config, layers) = if let Some(run) = saved_recovery {
4074 let mut saved = run.clone();
4077 if let Err(e) = saved.configure_recovery_resume() {
4078 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4079 record(queue, task);
4080 return Vec::new();
4081 }
4082 (saved.config, Vec::new())
4083 } else {
4084 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4085 Ok(loaded) => loaded,
4086 Err(e) => {
4087 let reason = format!("[config] {e:#}");
4088 record_config_failure(task, &e);
4089 task.last_error = Some(reason.clone());
4090 task.hold_machine(Some(reason));
4091 record(queue, task);
4092 return Vec::new();
4093 }
4094 };
4095 if let Err(e) = apply_options(&mut config, opts, task) {
4096 task.hold_manual(Some(format!(
4097 "[config] {e:#}; correct the merge override before releasing the task"
4098 )));
4099 record(queue, task);
4100 return Vec::new();
4101 }
4102 if let Some(selected) = &task.pr_recovery
4103 && let Err(e) = selected.configure(&mut config)
4104 {
4105 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4106 record(queue, task);
4107 return Vec::new();
4108 }
4109 (config, layers)
4110 };
4111 let config_evidence = crate::pr_recovery::ConfigEvidence {
4112 layers,
4113 overrides: task.overrides.clone(),
4114 daemon_merge: task
4115 .overrides
4116 .as_ref()
4117 .is_none_or(|o| o.inherit_daemon_options)
4118 .then(|| opts.merge.clone())
4119 .flatten(),
4120 };
4121 apply_task_implementers(&mut config, task);
4122 if let Some(replacement) = &task.replacement {
4123 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4124 old.liveness(false) == Liveness::Dead
4125 && old.merge_decision.as_ref().is_some_and(|d| {
4126 d.closed
4127 && d.question == replacement.question
4128 && d.replacement_run.as_ref().is_none_or(|run| {
4129 task.runs.last() == Some(run)
4130 && RunState::load(run).is_ok_and(|current| {
4131 current.replaces.as_ref().is_some_and(|pin| {
4132 pin.question == replacement.question
4133 && pin.agent == replacement.agent
4134 })
4135 })
4136 })
4137 })
4138 });
4139 if !safe {
4140 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4141 record(queue, task);
4142 return Vec::new();
4143 }
4144 match config.agent(&replacement.agent) {
4145 Ok(agent) if crate::agent::installed(agent) => {}
4146 _ => {
4147 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4148 record(queue, task);
4149 return Vec::new();
4150 }
4151 }
4152 config.graph.implementers = 1;
4153 config.graph.land_approval = true;
4154 config.merge.mode = MergeMode::Pr;
4155 config.graph.selected_implementer = Some(replacement.agent.clone());
4156 }
4157 let p = phrases(&config.graph.language);
4158 let start_failed = p.could_not_start;
4159 if let Ok(base) =
4164 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4165 .await
4166 && let Ok(head) =
4167 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4168 {
4169 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4170 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4171 let source = crate::git::rev_parse(&repo, branch)
4172 .await
4173 .unwrap_or_default();
4174 inputs.push_str(&format!(";review:{branch}:{source}"));
4175 }
4176 if let Some(reason) = task.recovery_exhausted(
4177 &head,
4178 &inputs,
4179 task.attempt_limit
4180 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4181 ) {
4182 task.hold_unrequested_retry(Some(reason));
4183 record(queue, task);
4184 return Vec::new();
4185 }
4186 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4187 }
4188
4189 if let Some(reason) = disk_gate(&repo, &config) {
4197 task.notice_cause("disk-space", None, None);
4198 task.last_error = Some(reason.clone());
4199 task.hold_machine(Some(reason.clone()));
4200 record(queue, task);
4201 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4202 notices::raise(
4205 Notice::warn(
4206 &format!("disk:{}", repo.display()),
4207 "A task was held for want of free disk space; free some, then release it from the queue.",
4208 )
4209 .link(Link::Task {
4210 id: task.id.clone(),
4211 }),
4212 );
4213 return Vec::new();
4214 }
4215
4216 let review_branch = take_review_branch(task, previous.as_ref());
4241 let branch_exists = match &review_branch {
4242 Some(branch) => crate::git::branch_exists(&repo, branch)
4243 .await
4244 .unwrap_or(false),
4245 None => false,
4246 };
4247 let proposed = match (&review_branch, &unfinished, &task.review_of) {
4251 (None, None, Some(branch)) => Starter::Review(branch.clone()),
4252 _ => choose_starter(
4253 review_branch.as_deref(),
4254 branch_exists,
4255 unfinished.as_deref(),
4256 ),
4257 };
4258 let retry_selected = task.replacement.is_some()
4259 && task.resumption.is_some()
4260 && !task.fresh_start
4261 && previous.as_ref().is_some_and(|run| {
4262 run.replaces.is_some()
4263 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4264 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4265 });
4266 let stop_pin = task
4267 .resume_override
4268 .as_ref()
4269 .filter(|pin| {
4270 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4271 q.node == land::APPROVAL_NODE
4272 && q.resolution()
4273 .as_deref()
4274 .is_some_and(crate::merge_decision::stops)
4275 })
4276 })
4277 .and_then(|pin| pin.pinned_run.clone());
4278 let starter = if let Some(owner) = &public_owner {
4279 Starter::Resume(owner.id.clone())
4280 } else if let Some(run) = stop_pin {
4281 Starter::Resume(run)
4284 } else if retry_selected {
4285 Starter::Resume(previous.as_ref().unwrap().id.clone())
4286 } else if let Some(selected) = &task.pr_recovery {
4287 if selected.check_task(&task.id, &task.instruction).is_ok() {
4288 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4289 } else {
4290 Starter::Review(selected.branch.clone())
4291 }
4292 } else {
4293 starter_with_implementers(task, previous.as_ref(), proposed)
4294 };
4295 if let Some(pin) = &task.resume_override
4298 && let Some(run) = &pin.pinned_run
4299 && starter != Starter::Resume(run.clone())
4300 {
4301 let run = run.clone();
4302 task.notice_cause("resume-owner", Some(&run), None);
4303 task.hold_machine(Some(format!(
4304 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4305 task.id
4306 )));
4307 record(queue, task);
4308 return Vec::new();
4309 }
4310 if task.replacement.is_some()
4311 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4312 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4313 {
4314 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4315 record(queue, task);
4316 return Vec::new();
4317 }
4318 if let Some(reason) = retained_attempt_hold(
4319 task,
4320 &starter,
4321 config.disk.retained_attempt_limit,
4322 RunState::load,
4323 ) {
4324 if let Starter::Review(branch) = &starter {
4327 task.review_branch = Some(branch.clone());
4328 }
4329 task.hold_unrequested_retry(Some(reason));
4330 record(queue, task);
4331 return Vec::new();
4332 }
4333 let attachments = match task_attachments(queue, task) {
4338 Ok(a) => a,
4339 Err(e) => {
4340 task.attempts += 1;
4341 task.fail_startup_target(
4342 "startup-attachments",
4343 e.name.clone(),
4344 format!("{start_failed}{e:#}"),
4345 opts.max_attempts,
4346 );
4347 record(queue, task);
4348 return Vec::new();
4349 }
4350 };
4351 let startup_pause = stop.pause();
4352 let startup_interrupt = interrupt_pause.clone();
4353 let wait = crate::repo_lock::WaitPolicy::new(
4354 Duration::from_secs(config.graph.timeout_resource_wait),
4355 move || startup_pause.parked() || startup_interrupt.parked(),
4356 );
4357 let started = wait.scope(Box::pin(async { match &starter {
4360 Starter::Review(_) if task.pr_recovery.is_some() => {
4361 let selected = task.pr_recovery.clone().expect("selected recovery");
4362 match selected.check_task(&task.id, &task.instruction) {
4363 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4364 .await
4365 .map(|mut r| {
4366 r.state.attachments = attachments.clone();
4367 r
4368 }),
4369 Err(e) => Err(e),
4370 }
4371 }
4372 Starter::Review(branch) => {
4373 tracing::info!(
4374 "task {} reopens `{branch}` as a review-only pass",
4375 task.short()
4376 );
4377 let takeover = crate::handover::Takeover {
4380 earlier: task.earlier_attempts().to_vec(),
4381 home: crate::run::home(),
4382 choice: take_divergence_answer(branch, &config.merge.remote, task),
4383 task_instruction: Some(instruction_for(task)),
4384 };
4385 Runner::review_taking_over(
4386 &repo,
4387 branch,
4388 config,
4389 Some(takeover),
4390 crate::run::Origin::queue(&task.id),
4391 )
4392 .await
4393 }
4394 Starter::Resume(id) => {
4395 tracing::info!("resuming run {id} rather than competing again");
4396 Runner::resume(id).and_then(|mut r| {
4397 if retry_selected {
4398 for candidate in &mut r.state.candidates {
4401 candidate.failed = None;
4402 candidate.empty = false;
4403 candidate.commits = 0;
4404 }
4405 r.state.status = RunStatus::Implementing;
4406 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4407 }
4408 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4409 r.state.sources = task.sources.clone();
4410 }
4411 if r.state.prep_failure.is_some() {
4412 let approval = task
4413 .resumption
4414 .as_ref()
4415 .context("preparation retry requires an explicit task release")?;
4416 r.retry_preparation(approval.at)?;
4417 }
4418 if let Some(instruction) =
4419 prepare_instruction(&starter, Some(&r.state.instruction), task)
4420 {
4421 r.state.instruction = instruction;
4422 }
4423 r.state.attachments = attachments.clone();
4424 if task.replacement.is_some() {
4425 r.state.config.merge.mode = MergeMode::Pr;
4426 r.state.config.graph.land_approval = true;
4427 }
4428 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4432 .overrides
4433 .as_ref()
4434 .and_then(|o| o.merge.as_deref())
4435 .and_then(|m| merge_mode(m).ok())
4436 {
4437 r.state.config.merge.mode = mode;
4438 }
4439 Ok(r)
4440 })
4441 }
4442 Starter::Start => {
4443 if let Some(branch) = &review_branch {
4444 tracing::warn!(
4445 "conductor chose review for task {} but branch `{branch}` no longer \
4446 exists; requeuing as a fresh competition instead",
4447 task.short()
4448 );
4449 }
4450 let instruction = prepare_instruction(&starter, None, task)
4451 .unwrap_or_else(|| task.instruction.clone());
4452 Runner::start_naming(
4453 &repo,
4454 instruction,
4455 &task.title,
4456 config,
4457 crate::run::Origin::queue(&task.id),
4458 )
4459 .await
4460 .map(|mut r| {
4461 r.state.attachments = attachments.clone();
4462 r.state.sources = task.sources.clone();
4463 r
4464 })
4465 }
4466 }})).await;
4467 let mut runner = match started {
4468 Ok(r) => r,
4469 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4470 task.last_error = Some(e.to_string());
4471 if stop.parking() || interrupt_pause.parked() {
4472 let attempts = task.attempts;
4473 task.park(e.to_string());
4474 task.attempts = attempts;
4476 } else {
4477 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4478 }
4479 record(queue, task);
4480 return Vec::new();
4481 }
4482 Err(e) if task.pr_recovery.is_some() => {
4486 task.startup_notice_cause("startup-public-head");
4487 task.attempts += 1;
4488 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4489 task.link_run(&partial.run);
4490 task.record_pass(&partial.run, 0, task.attempts, None);
4491 }
4492 task.last_error = Some(format!("{start_failed}{e:#}"));
4493 task.hold_unrequested_retry(Some(format!(
4494 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4495 )));
4496 record(queue, task);
4497 return Vec::new();
4498 }
4499 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4500 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4501 Some(r) => (p.handover_refused)(r),
4502 None => format!("{e:#}"),
4503 };
4504 task.startup_notice_cause("startup-handover");
4505 let reason = format!("{start_failed}{detail}");
4506 task.last_error = Some(reason.clone());
4507 let branch = match &starter {
4510 Starter::Review(branch) => Some(branch.clone()),
4511 _ => None,
4512 };
4513 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4517 record(queue, task);
4518 tracing::warn!(
4519 "holding {} for a branch it cannot take over: {e:#}",
4520 task.short()
4521 );
4522 return Vec::new();
4523 }
4524 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4529 let d = e
4530 .downcast_ref::<crate::reconcile::Diverged>()
4531 .expect("checked by the guard");
4532 let mut q = ask::Question::new(
4533 task.id.clone(),
4534 crate::reconcile::NODE.to_owned(),
4535 crate::reconcile::SEAT.to_owned(),
4536 d.summary(),
4537 d.detail(),
4538 d.choices(),
4539 );
4540 match Questions::open().put(&mut q) {
4541 Ok(()) => {
4542 task.last_error = Some(format!("{e:#}"));
4543 task.review_branch = Some(d.branch.clone());
4546 task.review_instruction = Some(task.instruction.clone());
4547 task.block(vec![q.id.clone()], Some(d.summary()));
4548 }
4549 Err(put) => {
4550 tracing::warn!("could not file the divergence question: {put:#}");
4551 task.attempts += 1;
4552 task.fail_startup(
4553 "startup-question-store",
4554 format!("{start_failed}{e:#}"),
4555 opts.max_attempts,
4556 );
4557 }
4558 }
4559 record(queue, task);
4560 return Vec::new();
4561 }
4562 Err(e) => {
4563 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4569 && let Starter::Review(branch) = &starter
4570 {
4571 task.review_branch = Some(branch.clone());
4572 task.review_instruction = Some(task.instruction.clone());
4573 task.last_error = Some(format!("{start_failed}{e:#}"));
4574 task.status = crate::queue::TaskStatus::Failed;
4575 record(queue, task);
4576 return Vec::new();
4577 }
4578 task.attempts += 1;
4579 task.last_error = Some(format!("{start_failed}{e:#}"));
4580 task.startup_notice_cause("startup-source");
4581 task.hold_unrequested_retry(Some(format!(
4582 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4583 )));
4584 record(queue, task);
4585 return Vec::new();
4586 }
4587 };
4588 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4589 runner.state.recovery_config = Some(config_evidence);
4590 }
4591 if let Some(replacement) = &task.replacement {
4592 if matches!(starter, Starter::Start) {
4593 runner.state.instruction.push_str(&format!(
4594 "\n\n# Replacement review context\n{}",
4595 replacement.context
4596 ));
4597 }
4598 runner.state.replaces = Some(replacement.clone());
4599 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4600 if let Ok(mut old) = RunState::load(&replacement.run)
4601 && let Some(d) = &mut old.merge_decision
4602 {
4603 d.replacement_run = Some(runner.state.id.clone());
4604 old.event(
4605 "replacement",
4606 format!(
4607 "replacement run {} owns the next attempt; old PR remains closed",
4608 runner.state.id
4609 ),
4610 );
4611 if let Err(e) = old.save() {
4612 task.hold_manual(Some(format!(
4613 "Could not link replacement history: {e}; no agents dispatched"
4614 )));
4615 record(queue, task);
4616 return Vec::new();
4617 }
4618 }
4619 }
4620 runner.state.task_instruction = Some(task.instruction.clone());
4621 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4624 if runner.state.origin_chat.is_none() {
4625 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4626 }
4627 runner.on_pause(stop.pause());
4629 runner.watch_interrupt(interrupt_pause);
4633
4634 let run = runner.state.id.clone();
4637 task.start(run.clone());
4638 record(queue, task);
4639 lock(status).current.push(Current {
4640 task: task.id.clone(),
4641 run,
4642 });
4643
4644 let quota_before = runner.state.quota.clone();
4647 runner.state.next_pass_task_entry = true;
4648 let result = runner.execute().await;
4649 finish_attempt(
4650 opts.max_attempts,
4651 queue,
4652 task,
4653 &runner.state,
4654 "a_before,
4655 result,
4656 )
4657}
4658
4659pub fn finish_attempt(
4664 max_attempts: usize,
4665 queue: &Queue,
4666 task: &mut Task,
4667 state: &RunState,
4668 quota_before: &[QuotaLoss],
4669 result: Result<()>,
4670) -> Vec<QuotaLoss> {
4671 let detail = match result {
4672 Ok(()) => describe(state),
4673 Err(e) => format!("{e:#}"),
4674 };
4675 let fresh = losses_this_attempt(quota_before, &state.quota);
4676 let verdict = Verdict {
4677 status: state.status,
4678 left_pr: state.pr.is_some(),
4681 quota_hit: !fresh.is_empty(),
4687 parked: state.parked,
4691 no_viable_candidates: state.viable().is_empty(),
4694 };
4695 if let Some(replacement) = &state.replaces
4696 && let Some(pr) = &state.pr
4697 && let Ok(mut old) = RunState::load(&replacement.run)
4698 && let Some(d) = &mut old.merge_decision
4699 {
4700 d.replacement_pr = Some(pr.url.clone());
4701 let _ = old.save();
4702 }
4703 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4704 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4705 task.hold_unrequested_retry(Some(format!(
4706 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4707 task.short()
4708 )));
4709 }
4710 if task.status == TaskStatus::Done {
4711 supersede_prior_runs(task, &crate::run::home());
4712 }
4713 record(queue, task);
4714 tracing::info!(
4715 "task {} is {} after run {} ({})",
4716 task.short(),
4717 task.status.as_str(),
4718 state.short(),
4719 label(state.status)
4720 );
4721 fresh
4722}
4723
4724pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4728 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4731 && !awaiting_resume_with(task, RunState::load)
4732 && withheld_text_wait(task).is_none()
4736 && !(task.fresh_start && task.replacement.is_some())
4737 {
4738 let why = task.last_error.clone().map_or_else(
4739 || "the run did not finish".to_owned(),
4740 |e| format!("the run did not finish: {e}"),
4741 );
4742 task.hold_unrequested_retry(Some(format!(
4743 "{why}. It was started by hand, so it is not retried \
4744 automatically; `magi task release` retries it."
4745 )));
4746 record(queue, task);
4747 }
4748}
4749
4750#[must_use]
4754pub fn loop_would_resume(run: &str) -> bool {
4755 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4756}
4757
4758#[must_use]
4768pub fn foreign_loop(
4769 reading: Option<&Reading>,
4770 now: Timestamp,
4771 own_pid: u32,
4772) -> Option<Option<u32>> {
4773 let reading = reading.filter(|r| r.running(now))?;
4774 match reading.pid {
4775 Some(pid) if pid == own_pid => None,
4776 pid => Some(pid),
4777 }
4778}
4779
4780fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4781 reading.worker_protocol != Some(1)
4782 && (foreign_loop(Some(reading), now, own_pid).is_some()
4783 || reading
4784 .pid
4785 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4786}
4787
4788pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4799 run_claimed_until(opts, queue, task, &Stop::new()).await;
4800}
4801
4802async fn run_claimed_until(
4803 opts: &Opts,
4804 queue: &Queue,
4805 task: &mut Task,
4806 stop: &Stop,
4807) -> Vec<QuotaLoss> {
4808 let status = Arc::new(Mutex::new(Status::new()));
4809 let mut quota = Box::pin(attempt(
4812 opts,
4813 queue,
4814 &status,
4815 stop,
4816 crate::graph::Pause::new(),
4817 task,
4818 ))
4819 .await;
4820 let mut resumes = 0;
4829 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
4830 let Some(timeout) = withheld_text_wait(task) else {
4831 break;
4832 };
4833 eprintln!(
4834 "waiting for the owner's answer on a withheld pull request text \
4835 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4836 );
4837 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4840 let ready = loop {
4841 match land_resume_state(task) {
4842 LandResume::StillWaiting
4843 if !stop.parking() && std::time::Instant::now() < limit =>
4844 {
4845 tokio::time::sleep(WITHHELD_POLL).await;
4846 }
4847 LandResume::Ready => break true,
4848 _ => break false,
4849 }
4850 };
4851 if !ready || stop.parking() {
4852 break;
4853 }
4854 resumes += 1;
4855 quota.extend(
4856 Box::pin(attempt(
4857 opts,
4858 queue,
4859 &status,
4860 stop,
4861 crate::graph::Pause::new(),
4862 task,
4863 ))
4864 .await,
4865 );
4866 }
4867 hold_if_runnable(queue, task);
4868 quota
4869}
4870
4871const MAX_WITHHELD_RESUMES: u32 = 3;
4874
4875const WITHHELD_POLL: Duration = Duration::from_secs(15);
4877
4878fn withheld_text_wait(task: &Task) -> Option<u64> {
4881 let id = task.runs.last()?;
4882 let s = RunState::load(id).ok()?;
4883 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4884 .then_some(s.config.graph.answer_timeout)
4885}
4886
4887fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4897 after
4898 .iter()
4899 .filter(|q| !before.contains(q))
4900 .cloned()
4901 .collect()
4902}
4903
4904fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4907 if quota.is_empty() {
4908 return None;
4909 }
4910 let with_hint = quota.iter().find(|q| q.reset.is_some());
4911 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4912 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4913 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4914 Some(
4915 now.checked_add(jiff::SignedDuration::from_secs(secs))
4916 .unwrap_or(Timestamp::MAX),
4917 )
4918}
4919
4920fn implementers_match(task: &Task, state: &RunState) -> bool {
4922 task.replacement.as_ref().map_or_else(
4923 || {
4924 task.implementers()
4925 .is_none_or(|n| n == state.config.graph.implementers)
4926 },
4927 |replacement| {
4928 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4929 && state.config.graph.implementers == 1
4930 },
4931 )
4932}
4933
4934fn starter_with_implementers(
4936 task: &Task,
4937 previous: Option<&RunState>,
4938 proposed: Starter,
4939) -> Starter {
4940 if previous.is_some_and(|s| !implementers_match(task, s)) {
4941 task.review_of
4942 .as_ref()
4943 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4944 } else {
4945 proposed
4946 }
4947}
4948
4949fn apply_task_implementers(config: &mut Config, task: &Task) {
4951 if let Some(n) = task.implementers() {
4952 config.graph.implementers = n;
4953 }
4954}
4955
4956#[cfg(test)]
4961fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4962 let explicit = match &task.overrides {
4963 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4964 _ => opts.config.as_deref(),
4965 };
4966 let (mut config, _) = Config::discover(repo, explicit)?;
4967 apply_options(&mut config, opts, task)?;
4968 Ok(config)
4969}
4970
4971fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
4972 if task
4973 .overrides
4974 .as_ref()
4975 .is_none_or(|o| o.inherit_daemon_options)
4976 && let Some(mode) = opts.merge.as_deref()
4977 {
4978 config.merge.mode = merge_mode(mode)?;
4979 }
4980 if let Some(o) = &task.overrides {
4981 o.apply(config);
4982 }
4983 Ok(())
4984}
4985
4986fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
4988 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
4989 if let Some(mode) = &opts.merge {
4990 config.merge.mode = merge_mode(mode)?;
4991 }
4992 Ok(config)
4993}
4994
4995async fn maybe_prune_cache_between_runs(
5000 repo: &Path,
5001 opts: &Opts,
5002 home: &Path,
5003 stop: &Stop,
5004 last_checked: &mut Option<Timestamp>,
5005 now: Timestamp,
5006) {
5007 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5008 return;
5009 }
5010 *last_checked = Some(now);
5011 let cfg = match prepare(repo, opts) {
5012 Ok(cfg) => cfg,
5013 Err(e) => {
5014 tracing::warn!("cache check: no config: {e:#}");
5015 return;
5016 }
5017 };
5018 match clean::prune_cache_if_over_limit(&cfg, home) {
5019 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5020 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5021 pruned.files,
5022 pruned.freed
5023 ),
5024 Ok(_) => {}
5025 Err(e) => {
5026 tracing::warn!("housekeep: prune cache: {e:#}");
5027 notices::raise_in(
5028 home,
5029 Notice::warn(
5030 "housekeep:cache",
5031 "Pruning the shared build cache failed; disk usage may keep growing.",
5032 ),
5033 );
5034 }
5035 }
5036}
5037
5038fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5042 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5043}
5044
5045async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5064 let cfg = match prepare(repo, opts) {
5065 Ok(cfg) => cfg,
5066 Err(e) => {
5067 tracing::warn!("housekeep: no config: {e:#}");
5068 return;
5069 }
5070 };
5071 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5080 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5081 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5086 let mut extra = Vec::new();
5087 if out.unreadable > 0 {
5088 extra.push(format!("{} unreadable", out.unreadable));
5089 }
5090 if out.orphaned_worktrees > 0 {
5091 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5092 }
5093 let detail = if extra.is_empty() {
5094 String::new()
5095 } else {
5096 format!(" ({})", extra.join(", "))
5097 };
5098 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5099 }
5100 if out.external_merges_recorded > 0 {
5101 tracing::info!(
5102 "housekeep: recorded {} run(s) as merged externally",
5103 out.external_merges_recorded
5104 );
5105 }
5106 if out.stale_pr_states_repaired > 0 {
5107 tracing::info!(
5108 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5109 out.stale_pr_states_repaired
5110 );
5111 }
5112 if out.cache_files > 0 {
5113 tracing::info!(
5114 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5115 out.cache_files,
5116 out.cache_freed
5117 );
5118 }
5119 if out.questions_abandoned > 0 {
5120 tracing::info!(
5121 "housekeep: abandoned {} question(s) left open by a finished run",
5122 out.questions_abandoned
5123 );
5124 }
5125}
5126
5127async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5136 let questions = Questions::at(home.join("questions"));
5137 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5138 if report.is_empty() {
5139 return;
5140 }
5141 if !report.quarantined.is_empty() {
5142 tracing::info!(
5143 "triage: held {} blocked task(s) whose blocked-on task or \
5144 question no longer exists: {}",
5145 report.quarantined.len(),
5146 report.quarantined.join(", ")
5147 );
5148 }
5149 if !report.resumed.is_empty() {
5150 tracing::info!(
5151 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5152 report.resumed.len(),
5153 report.resumed.join(", ")
5154 );
5155 }
5156 if !report.asked.is_empty() {
5157 tracing::info!(
5158 "triage: asked about {} held task(s): {}",
5159 report.asked.len(),
5160 report.asked.join(", ")
5161 );
5162 }
5163 if !report.answered.is_empty() {
5164 tracing::info!(
5165 "triage: applied {} operator answer(s): {}",
5166 report.answered.len(),
5167 report.answered.join(", ")
5168 );
5169 }
5170}
5171
5172pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5179 disk_gate_with(repo, config, crate::disk::free_bytes)
5180}
5181
5182fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5186 repo: &Path,
5187 config: &Config,
5188 free_bytes: F,
5189) -> Option<String> {
5190 let min = config.disk.min_free_bytes;
5191 if min == 0 {
5192 return None;
5193 }
5194 let mut paths = vec![repo.to_owned()];
5195 if let Some(cache) = config.cache_dir() {
5196 paths.push(cache);
5197 }
5198 if let Some(root) = &config.graph.worktree_root {
5199 paths.push(root.clone());
5200 }
5201 for path in paths {
5202 let probe = if path == repo {
5203 path.as_path()
5204 } else {
5205 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5206 };
5207 let reason = match free_bytes(probe) {
5208 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5209 Err(e) => Some(crate::disk::unmeasured_in(
5210 probe,
5211 &e,
5212 &config.graph.language,
5213 )),
5214 };
5215 if reason.is_some() {
5216 return reason;
5217 }
5218 }
5219 None
5220}
5221
5222const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5229
5230const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5234
5235fn quota_wait(
5244 reset_at: Option<Timestamp>,
5245 now: Timestamp,
5246 fallback: Duration,
5247 cap: Duration,
5248) -> Duration {
5249 match reset_at {
5250 Some(at) if at > now => {
5251 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5252 Duration::from_secs(secs).min(cap)
5253 }
5254 _ => fallback,
5255 }
5256}
5257
5258fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5272 parse_reset_hint_zoned(text, now)
5273 .or_else(|| parse_reset_hint_dated(text))
5274 .or_else(|| parse_reset_hint_relative(text, recorded))
5275}
5276
5277fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5281 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5282 let mut rest = rest.trim();
5283 if rest.is_empty() {
5284 return None;
5285 }
5286 let mut total: i64 = 0;
5287 let mut matched = false;
5288 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5289 if let Some((digits, tail)) = rest.split_once(unit)
5290 && !digits.is_empty()
5291 && digits.bytes().all(|b| b.is_ascii_digit())
5292 {
5293 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5294 rest = tail;
5295 matched = true;
5296 }
5297 }
5298 if !rest.is_empty() || !matched {
5299 return None;
5300 }
5301 recorded
5302 .checked_add(jiff::SignedDuration::from_secs(total))
5303 .ok()
5304}
5305
5306fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5310 let clock = clock.trim().to_lowercase();
5311 let (digits, pm) = clock
5312 .strip_suffix("am")
5313 .map(|d| (d, false))
5314 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5315 let (h, m) = digits.trim().split_once(':')?;
5316 let mut hour: i8 = h.trim().parse().ok()?;
5317 let minute: i8 = m.trim().parse().ok()?;
5318 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5319 return None;
5320 }
5321 if pm && hour != 12 {
5322 hour += 12;
5323 } else if !pm && hour == 12 {
5324 hour = 0;
5325 }
5326 Some((hour, minute))
5327}
5328
5329fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5334 let open = text.find('(')?;
5335 let close = text.rfind(')')?;
5336 if close <= open {
5337 return None;
5338 }
5339 let zone = text[open + 1..close].trim();
5340 let (hour, minute) = parse_12h_clock(&text[..open])?;
5341 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5342 let candidate = now
5343 .to_zoned(tz)
5344 .with()
5345 .hour(hour)
5346 .minute(minute)
5347 .second(0)
5348 .millisecond(0)
5349 .microsecond(0)
5350 .nanosecond(0)
5351 .build()
5352 .ok()?;
5353 let mut at = candidate.timestamp();
5354 if at <= now {
5355 at += jiff::SignedDuration::from_hours(24);
5356 }
5357 Some(at)
5358}
5359
5360fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5369 let words: Vec<&str> = text.split_whitespace().collect();
5370 if words.len() < 5 {
5371 return None;
5372 }
5373 (0..=words.len() - 5)
5374 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5375}
5376
5377fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5383 if trailing.is_some_and(|next| next.starts_with('(')) {
5384 return None;
5385 }
5386 let month = month_number(window[0])?;
5387 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5388 let day_digits = ["st", "nd", "rd", "th"]
5389 .iter()
5390 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5391 let day: i8 = day_digits.parse().ok()?;
5392 let year_token = window[2];
5393 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5394 return None;
5395 }
5396 let year: i16 = year_token.parse().ok()?;
5397 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5401 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5402 let date = jiff::civil::Date::new(year, month, day).ok()?;
5403 let candidate = date
5404 .at(hour, minute, 0, 0)
5405 .to_zoned(jiff::tz::TimeZone::UTC)
5406 .ok()?;
5407 Some(candidate.timestamp())
5408}
5409
5410fn month_number(name: &str) -> Option<i8> {
5413 const NAMES: [&str; 12] = [
5414 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5415 ];
5416 let lower = name.to_lowercase();
5417 NAMES
5418 .iter()
5419 .position(|n| *n == lower.as_str())
5420 .map(|i| i as i8 + 1)
5421}
5422
5423#[cfg(test)]
5425fn exhausted_review_budget(state: &RunState) -> bool {
5426 !crate::handover::can_resume(state)
5429 && state.status == RunStatus::Blocked
5430 && !state.released()
5431 && state.retired_to.is_none()
5432}
5433
5434fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5471 unfinished_run_with(runs, short, RunState::load)
5472}
5473
5474pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5477where
5478 F: FnMut(&str) -> Result<RunState>,
5479{
5480 let mut id = runs.last()?.clone();
5481 let mut visited = std::collections::BTreeSet::new();
5482 let mut repo = None;
5483 loop {
5484 if !runs.contains(&id) || !visited.insert(id.clone()) {
5485 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5486 return None;
5487 }
5488 let state = match load(&id) {
5489 Ok(state) if state.id == id => state,
5490 Ok(_) => return None,
5491 Err(e) => {
5492 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5493 return None;
5494 }
5495 };
5496 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5497 return None;
5498 }
5499 repo = Some(state.repo.clone());
5500 if let Some(authority) = state.retired_to {
5501 id = authority;
5502 continue;
5503 }
5504 return (crate::handover::can_resume(&state)
5505 && state.liveness(false) != crate::run::Liveness::Live)
5506 .then_some(id);
5507 }
5508}
5509
5510fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5518where
5519 F: FnOnce(&str) -> Result<RunState>,
5520{
5521 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5523 || task.fresh_start
5524 || task.review_branch.is_some()
5525 {
5526 return false;
5527 }
5528 let Some(id) = task.runs.last() else {
5529 return false;
5530 };
5531 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5532}
5533
5534#[derive(Debug, Clone, PartialEq, Eq)]
5537enum Starter {
5538 Review(String),
5541 Resume(String),
5543 Start,
5545}
5546
5547fn retained_attempt_hold(
5551 task: &Task,
5552 starter: &Starter,
5553 limit: usize,
5554 mut load: impl FnMut(&str) -> Result<RunState>,
5555) -> Option<String> {
5556 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5557 return None;
5558 }
5559 let mut retained = 0;
5560 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5561 let state = match load(id) {
5562 Ok(state) if state.id == *id => state,
5563 _ => {
5564 return Some(format!(
5565 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5566 ));
5567 }
5568 };
5569 if state.retired_to.is_some() || state.released() {
5570 continue;
5571 }
5572 retained += 1;
5575 }
5576 (retained >= limit).then(|| format!(
5577 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5578 ))
5579}
5580
5581fn selected_starter(
5583 selected: &crate::pr_recovery::Selection,
5584 previous: Option<&RunState>,
5585 unfinished: Option<&str>,
5586) -> Starter {
5587 if let Some(id) = unfinished
5588 && previous.is_some_and(|r| {
5589 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5590 })
5591 {
5592 Starter::Resume(id.to_owned())
5593 } else {
5594 Starter::Review(selected.branch.clone())
5595 }
5596}
5597
5598fn take_divergence_answer(
5602 branch: &str,
5603 remote: &str,
5604 task: &mut Task,
5605) -> Option<crate::reconcile::Choice> {
5606 let summary = crate::reconcile::summary_for(branch, remote);
5607 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5608 (a.question == summary)
5609 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5610 .flatten()
5611 .map(|c| (i, c))
5612 })?;
5613 task.answers.remove(idx);
5614 Some(choice)
5615}
5616
5617fn choose_starter(
5629 review_branch: Option<&str>,
5630 branch_exists: bool,
5631 unfinished: Option<&str>,
5632) -> Starter {
5633 match review_branch {
5634 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5635 Some(_) => Starter::Start,
5636 None => match unfinished {
5637 Some(id) => Starter::Resume(id.to_owned()),
5638 None => Starter::Start,
5639 },
5640 }
5641}
5642
5643fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5645 let branch = task.review_branch.take();
5646 let instruction = task.review_instruction.take();
5647 branch.filter(|branch| match &instruction {
5648 Some(authored) => *authored == task.instruction,
5649 None => previous.map_or_else(
5652 || task.review_of.as_ref() == Some(branch),
5653 |run| compatible_run_instruction(run, task),
5654 ),
5655 })
5656}
5657
5658fn may_resume_run(run: &RunState, task: &Task) -> bool {
5661 task.resume_override
5662 .as_ref()
5663 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5664 || compatible_run_instruction(run, task)
5665}
5666
5667pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5669 if let Some(authored) = &run.task_instruction {
5670 return compatible_instruction(authored, task);
5671 }
5672 if compatible_instruction(&run.instruction, task) {
5673 return true;
5674 }
5675 if run.reviewed_commits.is_some() {
5678 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5679 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5680 }
5681 if let Some(branch) = &task.review_of {
5682 let canonical = format!(
5683 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5684 );
5685 let generated = format!(
5686 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5687 );
5688 return task.instruction == canonical && run.instruction.starts_with(&generated);
5689 }
5690 }
5691 false
5692}
5693
5694pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5696 strip_answers_block(old, task) == task.instruction
5697}
5698
5699fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5702 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5703 return fallback.to_path_buf();
5704 }
5705 task.repo.clone()
5706}
5707
5708const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5712
5713fn answers_block(task: &Task, count: usize) -> String {
5715 let mut s = ANSWERS_HEADER.to_owned();
5716 for a in &task.answers[..count] {
5717 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5718 }
5719 s
5720}
5721
5722fn resumption_block(r: &crate::queue::Resumption) -> String {
5725 format!(
5726 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5727 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5728 r.generation, r.at, r.instruction, r.answers_through,
5729 )
5730}
5731
5732fn append_answers(base: &str, task: &Task) -> String {
5734 let mut s = base.to_owned();
5735 if !task.answers.is_empty() {
5736 s.push_str(&answers_block(task, task.answers.len()));
5737 }
5738 if let Some(r) = &task.resumption {
5739 s.push_str(&resumption_block(r));
5740 }
5741 s
5742}
5743
5744fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5748 let mut instruction = instruction;
5749 for approval in task
5750 .resumption
5751 .iter()
5752 .chain(task.resumption_history.iter().rev())
5753 {
5754 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5755 instruction = base;
5756 break;
5757 }
5758 }
5759 for count in (1..=task.answers.len()).rev() {
5760 let block = answers_block(task, count);
5761 if let Some(base) = instruction.strip_suffix(&block) {
5762 return base;
5763 }
5764 }
5765 instruction
5766}
5767
5768fn instruction_for(task: &Task) -> String {
5776 append_answers(&task.instruction, task)
5777}
5778
5779fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5791 append_answers(strip_answers_block(old_instruction, task), task)
5792}
5793
5794fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5796 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5797 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5798 if let Some(cause) = task.hold_notice_cause.as_mut() {
5799 cause.target = failure.map(|f| f.target.clone());
5800 }
5801}
5802
5803#[derive(Debug, thiserror::Error)]
5804#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5805struct MissingAttachment {
5806 name: String,
5807 path: PathBuf,
5808}
5809
5810fn task_attachments(
5812 queue: &Queue,
5813 task: &Task,
5814) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5815 let paths = queue.attachment_paths(task);
5816 for (name, path) in task.attachments.iter().zip(&paths) {
5817 if !path.is_file() {
5818 return Err(MissingAttachment {
5819 name: name.clone(),
5820 path: path.clone(),
5821 });
5822 }
5823 }
5824 Ok(paths)
5825}
5826
5827fn prepare_instruction(
5838 starter: &Starter,
5839 old_instruction: Option<&str>,
5840 task: &Task,
5841) -> Option<String> {
5842 match starter {
5843 Starter::Start => Some(instruction_for(task)),
5844 Starter::Resume(_) => Some(resumed_instruction(
5845 old_instruction.expect("a resumed run always has a prior instruction"),
5846 task,
5847 )),
5848 Starter::Review(_) => None,
5849 }
5850}
5851
5852fn record(queue: &Queue, task: &mut Task) {
5856 if let Err(e) = queue.put(task) {
5857 tracing::error!("could not record task {}: {e:#}", task.short());
5858 let notice = Notice::error(
5859 "loop:record",
5860 "The loop could not save a task's state; check the disk.",
5861 )
5862 .link(Link::Task {
5863 id: task.id.clone(),
5864 });
5865 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5867 Some(home) => notices::raise_in(home, notice),
5868 None => notices::raise(notice),
5869 }
5870 }
5871}
5872
5873fn runnable(queue: &Queue) -> Vec<Task> {
5879 let mut tasks: Vec<Task> = queue
5880 .list()
5881 .into_iter()
5882 .filter(|t| t.status.runnable())
5883 .collect();
5884 tasks.sort_unstable_by(crate::queue::dispatch_order);
5885 tasks
5886}
5887
5888fn describe(state: &RunState) -> String {
5902 let p = phrases(&state.config.graph.language);
5903 let mut detail = if state.status == RunStatus::Stalled {
5904 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5905 seats.sort_unstable();
5906 seats.dedup();
5907 if seats.is_empty() {
5908 p.quorum_lost.to_owned()
5909 } else {
5910 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5911 }
5912 } else {
5913 format!("{}{}", p.run_ended, state.status.display_label())
5914 };
5915 if let Some(last) = state.events.last() {
5916 let unanswered = state
5920 .reviews
5921 .last()
5922 .filter(|r| {
5923 state.status == RunStatus::Blocked
5924 && last.node == "review"
5925 && r.incomplete()
5926 && r.blocking == 0
5927 && r.round == state.config.graph.review_rounds
5928 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
5929 })
5930 .map(|r| (r.expected - r.answered, r.round));
5931 match unanswered {
5932 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
5933 detail.push_str(&format!(
5934 " ({}: {})",
5935 last.node,
5936 (p.reviewers_never_answered)(missing, rounds)
5937 ));
5938 }
5939 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
5940 }
5941 }
5942 detail.push_str(&format!(" [run {}]", state.id));
5943 detail
5944}
5945
5946const DIAGNOSTIC_MAX: usize = 4_000;
5952
5953const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
5958
5959fn diagnostic(state: &RunState) -> Option<String> {
5973 let mut parts: Vec<String> = Vec::new();
5974
5975 for o in state.gate.iter().filter(|o| !o.ok()) {
5977 parts.push(format!(
5978 "gate `{}` failed ({}):\n{}",
5979 o.command,
5980 notices::exit_status(o.code),
5981 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
5982 ));
5983 }
5984
5985 if let Some(last) = state
5988 .events
5989 .iter()
5990 .rev()
5991 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
5992 {
5993 parts.push(last.message.clone());
5994 }
5995
5996 if state.viable().is_empty() {
6003 for c in &state.candidates {
6004 if let Some(evidence) = &c.verified_noop {
6005 parts.push(format!(
6006 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6007 c.label
6008 ));
6009 } else if !c.summary.trim().is_empty() {
6010 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6011 } else if let Some(why) = &c.failed {
6012 parts.push(format!("candidate {}: {why}", c.label));
6013 }
6014 }
6015 }
6016
6017 if parts.is_empty() {
6018 return None;
6019 }
6020 Some(crate::run::tail(
6025 &parts.join("\n\n"),
6026 DIAGNOSTIC_MAX.saturating_sub(100),
6027 ))
6028}
6029
6030fn label(status: RunStatus) -> &'static str {
6038 status.as_str()
6039}
6040
6041pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6043 match mode {
6044 "none" => Ok(MergeMode::None),
6045 "local" => Ok(MergeMode::Local),
6046 "pr" => Ok(MergeMode::Pr),
6047 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6048 }
6049}
6050
6051fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6056 mutex
6057 .lock()
6058 .unwrap_or_else(std::sync::PoisonError::into_inner)
6059}
6060
6061#[cfg(test)]
6062mod tests {
6063 use super::*;
6064 use crate::queue::{Source, TaskStatus};
6065 use crate::run::{Candidate, CommandOutcome};
6066 use pretty_assertions::assert_eq;
6067
6068 fn task() -> Task {
6069 Task::new(
6070 "add retries".to_owned(),
6071 "add retries".to_owned(),
6072 PathBuf::from("/repo"),
6073 Source::Human,
6074 )
6075 }
6076
6077 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6078 let job = crate::worker::Job {
6079 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6080 sequence: 1,
6081 kind: "task".into(),
6082 key: format!("task:{}", request.task),
6083 payload: serde_json::to_value(request).unwrap(),
6084 generation: "a".repeat(64),
6085 version: None,
6086 status: crate::worker::Status::Starting,
6087 result: None,
6088 pid: None,
6089 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6090 finished_at: None,
6091 };
6092 let dir = home.join("workers/jobs").join(&job.id);
6093 std::fs::create_dir_all(&dir).unwrap();
6094 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6095 job
6096 }
6097
6098 #[test]
6099 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6100 let home = tempfile::tempdir().unwrap();
6101 let request = TaskWorker {
6102 opts: Opts::default(),
6103 queue: home.path().join("queue"),
6104 task: "startup".into(),
6105 lane: PermitKind::Ordinary,
6106 standalone: false,
6107 };
6108 let job = starting_task_reservation(home.path(), &request);
6109 task_worker_spawn_error(
6110 home.path(),
6111 &request,
6112 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6113 )
6114 .expect_err("our own reservation must not turn a failed spawn into a resync");
6115 let active = crate::worker::active(home.path()).unwrap();
6116 assert_eq!(active.len(), 1);
6117 assert_eq!(active[0].id, job.id);
6118 assert_eq!(active[0].status, crate::worker::Status::Starting);
6119 assert!(
6120 crate::worker::control(home.path(), &job.key)
6121 .unwrap()
6122 .is_none()
6123 );
6124 let notices = notices::Notices::at(home.path().join("notifications")).list();
6125 assert_eq!(notices.len(), 1);
6126 assert_eq!(notices[0].key, "loop:worker-startup");
6127 assert!(!notices[0].message.contains("private-error-detail"));
6128 }
6129
6130 #[test]
6131 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6132 let home = tempfile::tempdir().unwrap();
6133 let mut request = TaskWorker {
6134 opts: Opts::default(),
6135 queue: home.path().join("queue"),
6136 task: "handoff".into(),
6137 lane: PermitKind::Ordinary,
6138 standalone: true,
6139 };
6140 let job = starting_task_reservation(home.path(), &request);
6141 request.standalone = false;
6142 task_worker_spawn_error(
6143 home.path(),
6144 &request,
6145 anyhow::anyhow!("operation key is already reserved for a different request"),
6146 )
6147 .expect("a different standalone request is the handoff race, not our startup failure");
6148 let active = crate::worker::active(home.path()).unwrap();
6149 assert_eq!(active.len(), 1);
6150 assert_eq!(active[0].id, job.id);
6151 assert_eq!(active[0].payload, job.payload);
6152 assert!(
6153 notices::Notices::at(home.path().join("notifications"))
6154 .list()
6155 .is_empty()
6156 );
6157 }
6158
6159 #[test]
6160 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6161 let dir = tempfile::tempdir().unwrap();
6162 let queue = Queue::at(dir.path().join("queue"));
6163 let mut task = task();
6164 task.status = TaskStatus::Running;
6165 task.attempts = 1;
6166 queue.put(&mut task).unwrap();
6167 let live = std::collections::BTreeSet::from([task.id.clone()]);
6168 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6169 drop(queue.claim(&task.id).unwrap());
6170 let saved = queue.get(&task.id).unwrap();
6171 assert_eq!(saved.status, TaskStatus::Running);
6172 assert_eq!(saved.attempts, 1);
6173 assert!(saved.last_error.is_none());
6174 }
6175
6176 #[test]
6177 fn standalone_workers_do_not_consume_scheduler_lanes() {
6178 let request = |task: &str, lane, standalone| TaskWorker {
6179 opts: Opts::default(),
6180 queue: PathBuf::from("/queue"),
6181 task: task.to_owned(),
6182 lane,
6183 standalone,
6184 };
6185 let mut jobs = vec![
6186 request("ordinary", PermitKind::Ordinary, false),
6187 request("standalone-urgent", PermitKind::Urgent, true),
6188 request("standalone-ordinary", PermitKind::Ordinary, true),
6189 request("decision", PermitKind::None, false),
6190 ];
6191 assert_eq!(available_slots(&jobs, 2), (1, 1));
6192 jobs.push(request("urgent", PermitKind::Urgent, false));
6193 assert_eq!(available_slots(&jobs, 2), (1, 0));
6194 assert_eq!(available_slots(&jobs, 0), (0, 0));
6195 }
6196
6197 #[test]
6198 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6199 let now = Timestamp::now();
6200 let mut reading = Reading {
6201 pid: Some(std::process::id()),
6202 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6203 ..Reading::default()
6204 };
6205 assert!(!reading.running(now));
6206 assert!(legacy_scheduler_running(&reading, now, 0));
6207 reading.worker_protocol = Some(1);
6208 assert!(!legacy_scheduler_running(&reading, now, 0));
6209 }
6210
6211 #[test]
6212 fn selected_public_head_overrides_historical_resume_and_pin() {
6213 let mut t = task();
6214 let mut old = run_state(RunStatus::Blocked);
6215 let selection = crate::pr_recovery::Selection {
6216 url: "url".into(),
6217 branch: format!("magi/task-{}", t.id),
6218 head: "a".repeat(40),
6219 source_head: None,
6220 source_run: None,
6221 source_proof: None,
6222 disposition: Some(crate::pr_recovery::Disposition::Review),
6223 instruction: t.instruction.clone(),
6224 approved_at: Timestamp::now(),
6225 };
6226 t.resume_override = Some(crate::queue::OperatorResume {
6227 pinned_run: Some(old.id.clone()),
6228 question_id: "choice".into(),
6229 at: Timestamp::now(),
6230 conductor_rehold: None,
6231 forced: false,
6232 });
6233 assert_eq!(
6234 selected_starter(&selection, Some(&old), Some(&old.id)),
6235 Starter::Review(selection.branch.clone())
6236 );
6237 old.pr_recovery = Some(selection.clone());
6238 assert_eq!(
6239 selected_starter(&selection, Some(&old), Some(&old.id)),
6240 Starter::Review(selection.branch.clone())
6241 );
6242 old.reviewed_commits = Some(vec!["implementation".into()]);
6243 assert_eq!(
6244 selected_starter(&selection, Some(&old), Some(&old.id)),
6245 Starter::Resume(old.id.clone())
6246 );
6247 let mut changed = selection.clone();
6248 changed.head = "b".repeat(40);
6249 assert_eq!(
6250 selected_starter(&changed, Some(&old), Some(&old.id)),
6251 Starter::Review(changed.branch)
6252 );
6253 }
6254
6255 #[test]
6256 fn candidate_transplant_failure_is_one_actionable_hold() {
6257 let dir = tempfile::tempdir().unwrap();
6258 let q = Queue::at(dir.path().join("queue"));
6259 let mut t = task();
6260 let mut state = run_state(RunStatus::Blocked);
6261 state.event("prep", "cannot apply referenced commit: conflict");
6262 state.prep_failure = Some("conflict".into());
6263 t.start(state.id.clone());
6264 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6265 assert_eq!(t.attempts, 1);
6266 assert_eq!(t.status, TaskStatus::Held);
6267 assert!(t.operator_held());
6268 assert!(!t.status.runnable());
6269 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6270 let first = t.chat_notice_key().unwrap();
6271 t.chat_notices.push(first.clone());
6272 t.release_operator("test", None);
6273 state.id = "another-unprepared-run".into();
6274 t.start(state.id.clone());
6275 finish_attempt(
6276 3,
6277 &q,
6278 &mut t,
6279 &state,
6280 &[],
6281 Err(anyhow::anyhow!("still conflicts")),
6282 );
6283 assert_eq!(
6284 t.chat_notice_key().as_ref(),
6285 Some(&first),
6286 "no work happened"
6287 );
6288 t.release_operator("test", None);
6289 state.prep_failure = None;
6290 state.status = RunStatus::Failed;
6291 state.begin_pass();
6292 t.start(state.id.clone());
6293 finish_attempt(
6294 1,
6295 &q,
6296 &mut t,
6297 &state,
6298 &[],
6299 Err(anyhow::anyhow!("new gate failure")),
6300 );
6301 let failure = t.chat_notice_key().unwrap();
6302 assert_ne!(failure, first, "approved recovery reached a real failure");
6303 t.release_operator("test", None);
6304 state.begin_pass();
6305 t.start(state.id.clone());
6306 finish_attempt(
6307 1,
6308 &q,
6309 &mut t,
6310 &state,
6311 &[],
6312 Err(anyhow::anyhow!("new failure on resume")),
6313 );
6314 assert_ne!(
6315 t.chat_notice_key().as_ref(),
6316 Some(&failure),
6317 "a new settled pass"
6318 );
6319 }
6320
6321 #[test]
6322 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6323 for source in ["cli", "web"] {
6324 let mut t = task();
6325 let mut run = run_state(RunStatus::Blocked);
6326 run.instruction = "Run A original criteria".into();
6327 run.task_instruction = Some(run.instruction.clone());
6328 t.start(run.id.clone());
6329 t.release_operator("choice", Some("resume-question"));
6330 t.resume_override = Some(crate::queue::OperatorResume {
6331 question_id: "resume-question".into(),
6332 at: Timestamp::now(),
6333 conductor_rehold: None,
6334 forced: true,
6335 pinned_run: Some(run.id.clone()),
6336 });
6337 t.hold_machine(Some("not enough free space to start a run".into()));
6338 let mut unchanged = t.clone();
6339 unchanged
6340 .edit("title changed".into(), unchanged.instruction.clone())
6341 .unwrap();
6342 unchanged.release_operator(source, None);
6343 assert_eq!(
6344 unchanged
6345 .resume_override
6346 .as_ref()
6347 .unwrap()
6348 .pinned_run
6349 .as_deref(),
6350 Some(run.id.as_str())
6351 );
6352 assert!(!compatible_run_instruction(&run, &unchanged));
6353 assert!(may_resume_run(&run, &unchanged));
6354
6355 t.edit(
6356 "approved recovery".into(),
6357 "Reuse measured PR head B; fix Clippy; review".into(),
6358 )
6359 .unwrap();
6360 assert!(t.resume_override.is_none());
6361 t.release_operator(source, None);
6362 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6363 assert!(reloaded.resume_override.is_none());
6364 assert!(!may_resume_run(&run, &reloaded));
6365 let unfinished =
6366 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6367 .filter(|_| may_resume_run(&run, &reloaded));
6368 assert_eq!(
6369 choose_starter(None, false, unfinished.as_deref()),
6370 Starter::Start
6371 );
6372 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6373 assert_eq!(
6374 reloaded.resumption_history[0].question_id.as_deref(),
6375 Some("resume-question")
6376 );
6377
6378 unchanged.instruction = "Reuse measured PR head B".into();
6381 unchanged.hold_machine(Some("disk hold".into()));
6382 unchanged.release_operator(source, None);
6383 assert!(unchanged.resume_override.is_none());
6384 assert!(!may_resume_run(&run, &unchanged));
6385 }
6386 }
6387
6388 #[test]
6389 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6390 let mut t = task();
6391 let mut run = run_state(RunStatus::Blocked);
6392 t.review_of = Some("approved-head".into());
6393 t.instruction =
6394 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6395 .into();
6396 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6397 run.reviewed_commits = Some(vec!["Commit log".into()]);
6398 t.start(run.id.clone());
6399 t.hold_machine(Some("interrupted review".into()));
6400 t.release_operator("cli", None);
6401 assert!(
6402 compatible_run_instruction(&run, &t),
6403 "legacy explicit review remains resumable"
6404 );
6405 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6406 .filter(|_| compatible_run_instruction(&run, &t));
6407 assert_eq!(
6408 choose_starter(None, false, unfinished.as_deref()),
6409 Starter::Resume(run.id.clone())
6410 );
6411 run.task_instruction = Some(t.instruction.clone());
6412 assert!(compatible_run_instruction(&run, &t));
6413 let reloaded: RunState =
6414 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6415 assert!(compatible_run_instruction(&reloaded, &t));
6416 t.edit(
6417 "new approved head".into(),
6418 "Reuse measured PR head B".into(),
6419 )
6420 .unwrap();
6421 assert!(!compatible_run_instruction(&run, &t));
6422
6423 t.review_of = None;
6425 t.instruction = "Fix Clippy on approved head".into();
6426 run.task_instruction = None;
6427 run.instruction = format!(
6428 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6429 t.instruction
6430 );
6431 assert!(compatible_run_instruction(&run, &t));
6432 t.instruction.push('\n');
6433 assert!(
6434 compatible_run_instruction(&run, &t),
6435 "legacy generated review trimmed the original criteria"
6436 );
6437 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6438 let mut unchanged = t.clone();
6439 assert_eq!(
6440 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6441 Some("approved-head")
6442 );
6443 t.edit("new plan".into(), "Reuse measured PR head B".into())
6444 .unwrap();
6445 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6446 assert!(!compatible_run_instruction(&run, &t));
6447 t.review_branch = Some("approved-head".into());
6448 t.review_instruction = None;
6449 assert!(
6450 take_review_branch(&mut t, Some(&run)).is_none(),
6451 "legacy retained candidate cannot replace a revised approved head"
6452 );
6453 }
6454
6455 #[test]
6456 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6457 crate::run::pin_test_home();
6458 let dir = tempfile::tempdir().unwrap();
6459 let queue = Queue::at(dir.path().join("queue"));
6460 for resume in [true, false] {
6461 let state = run_state(RunStatus::Blocked);
6462 let mut t = task();
6463 t.start(state.id.clone());
6464 let action = if resume {
6465 ask::ChoiceAction::Resume {
6466 run: state.id.clone(),
6467 }
6468 } else {
6469 ask::ChoiceAction::Requeue
6470 };
6471 let mut q = action_question(&state.id, action);
6472 q.status = ask::QuestionStatus::Open;
6473 q.answer = None;
6474 q.answered_at = None;
6475 t.fail("run stopped with a question", 3);
6476 hold_if_runnable(&queue, &mut t);
6477 assert!(t.operator_decided_at.is_none());
6478 assert!(t.operator_held());
6479 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6480 let expected = if resume {
6481 ActionDecision::Resume(state.id.clone())
6482 } else {
6483 ActionDecision::Requeue
6484 };
6485 assert_eq!(
6486 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6487 expected
6488 );
6489 t.hold_manual(Some("later explicit hold".into()));
6490 assert_eq!(
6491 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6492 ActionDecision::Stale
6493 );
6494 }
6495 }
6496
6497 #[test]
6498 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6499 let mut t = task();
6500 t.answers.push(crate::queue::AnsweredQuestion {
6501 question: "Inspect the remote?".into(),
6502 answer: "Inspect then wait".into(),
6503 });
6504 let old = instruction_for(&t);
6505 assert!(compatible_instruction(&old, &t));
6506 t.instruction = "Reuse approved PR head and review".into();
6507 t.release_operator("cli", None);
6508 assert!(!compatible_instruction(&old, &t));
6509 let approved = instruction_for(&t);
6510 assert!(approved.contains("waiting premises are superseded"));
6511 assert!(compatible_instruction(&approved, &t));
6512 assert_eq!(resumed_instruction(&approved, &t), approved);
6513 t.release_operator("web", None);
6514 assert!(compatible_instruction(&approved, &t));
6515 assert_eq!(t.resumption_history.len(), 1);
6516 t.start("run".into());
6517 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6518 }
6519
6520 #[test]
6524 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
6525 let mut state = RunState::new(
6526 PathBuf::from("/repo"),
6527 "main".to_owned(),
6528 "abc1234".to_owned(),
6529 "task".to_owned(),
6530 Config::default(),
6531 );
6532 state.status = RunStatus::Blocked;
6533 state.resource_block = Some(crate::run::ResourceBlock {
6534 node: "graft".to_owned(),
6535 kind: "shared build cache".to_owned(),
6536 detail: "timed out".to_owned(),
6537 at: jiff::Timestamp::now(),
6538 });
6539 let mut t = task();
6540 t.start("20260902-000000-aaaa".to_owned());
6541 assert_eq!(t.attempts, 1);
6542 let verdict = Verdict {
6543 status: RunStatus::Blocked,
6544 left_pr: false,
6545 quota_hit: false,
6546 parked: false,
6547 no_viable_candidates: false,
6548 };
6549 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6550 assert_eq!(t.status, TaskStatus::Held);
6551 assert_eq!(t.attempts, 0, "the attempt is refunded");
6552 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6553 assert!(
6554 !t.operator_held(),
6555 "a machine hold, released without a human"
6556 );
6557
6558 let mut selected = task();
6559 selected.start("20260902-000000-cccc".into());
6560 selected.replacement = Some(crate::merge_decision::Replacement {
6561 question: "fixture".into(),
6562 run: "earlier".into(),
6563 pr: "https://example.invalid/pr/1".into(),
6564 agent: "selected".into(),
6565 context: "fixture".into(),
6566 });
6567 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6568 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6569 assert_eq!(selected.attempts, 0);
6570 assert!(
6571 !selected.operator_held(),
6572 "a selected agent also retains a machine resource hold"
6573 );
6574 assert!(
6575 selected
6576 .hold_reason
6577 .as_deref()
6578 .unwrap()
6579 .starts_with("[resource] ")
6580 );
6581
6582 let mut p = task();
6584 p.start("20260902-000000-bbbb".to_owned());
6585 settle_and_diagnose(
6586 &mut p,
6587 Verdict {
6588 parked: true,
6589 ..verdict
6590 },
6591 "parked",
6592 2,
6593 &state,
6594 );
6595 assert_eq!(p.status, TaskStatus::Parked);
6596 }
6597
6598 #[test]
6599 fn record_failure_notice_links_the_task() {
6600 let dir = tempfile::tempdir().unwrap();
6601 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6604 let queue = Queue::at(dir.path().join("queue"));
6605 let mut t = interrupt_task("rec-fail-1");
6606 record(&queue, &mut t);
6607
6608 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6609 assert_eq!(found.len(), 1);
6610 assert_eq!(found[0].key, "loop:record");
6611 assert_eq!(
6612 found[0].link,
6613 Some(Link::Task {
6614 id: "rec-fail-1".to_owned()
6615 })
6616 );
6617 }
6618
6619 fn interrupt_task(id: &str) -> Task {
6622 let mut t = task();
6623 t.id = id.to_owned();
6624 t.interrupt = true;
6625 t
6626 }
6627
6628 fn task_with_id(id: &str) -> Task {
6630 let mut t = task();
6631 t.id = id.to_owned();
6632 t
6633 }
6634
6635 fn urgent_task(id: &str) -> Task {
6637 let mut t = task();
6638 t.id = id.to_owned();
6639 t.urgent = true;
6640 t
6641 }
6642
6643 #[test]
6644 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6645 let dir = tempfile::tempdir().unwrap();
6646 let queue = Queue::at(dir.path().join("queue"));
6647 let mut parked = task_with_id("parked");
6648 parked.start("run".into());
6649 parked.park("upgrade");
6650 parked.record_pass_at(
6651 "run",
6652 0,
6653 Some(Timestamp::now()),
6654 1,
6655 Some(crate::run::ParkCause::Upgrade {
6656 from: "old".into(),
6657 to: None,
6658 }),
6659 );
6660 let mut fresh = task_with_id("fresh");
6661 fresh.priority = 100;
6662 let mut urgent_task = urgent_task("urgent");
6663 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6664 queue.put(t).unwrap();
6665 }
6666 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6667 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6668 let candidates = runnable(&queue);
6669 assert_eq!(candidates[0].id, parked.id);
6670 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6671 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6672 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6673 drop(first);
6674 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6675 let next = runnable(&queue)
6676 .into_iter()
6677 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6678 .unwrap();
6679 assert_eq!(next.task.id, fresh.id);
6680 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6681 drop(claimed_elsewhere);
6682 drop(next);
6683 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6684 }
6685
6686 #[test]
6687 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6688 let dir = tempfile::tempdir().unwrap();
6689 let queue = Queue::at(dir.path().join("queue"));
6690 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6691 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6692 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6693 let mut first = task();
6694 queue.put(&mut first).unwrap();
6695 let stale_plain = queue.get(&first.id).unwrap();
6696 first.set_urgent(true).unwrap();
6697 queue.put(&mut first).unwrap();
6698 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6699 let mut second = urgent_task("second");
6700 queue.put(&mut second).unwrap();
6701 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6702 drop(queue.claim(&second.id).unwrap());
6703 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6704 drop(dispatched);
6705 let stale_urgent = queue.get(&first.id).unwrap();
6706 first.set_urgent(false).unwrap();
6707 queue.put(&mut first).unwrap();
6708 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6709 assert_eq!(urgent.available_permits(), 1);
6710 drop(queue.claim(&first.id).unwrap());
6711 drop(occupied);
6712 }
6713
6714 #[test]
6715 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6716 let dir = tempfile::tempdir().unwrap();
6717 let queue = Queue::at(dir.path().join("queue"));
6718 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6719 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6720 let mut t = task();
6721 queue.put(&mut t).unwrap();
6722 for enabled in [true, false] {
6723 let stale = queue.get(&t.id).unwrap();
6724 t.set_urgent(enabled).unwrap();
6725 queue.put(&mut t).unwrap();
6726 assert_ne!(stale.urgent, enabled);
6727 let DispatchClaim {
6728 task: fresh,
6729 claim,
6730 permit,
6731 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6732 assert_eq!(fresh.urgent, enabled);
6733 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6734 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6735 assert!(queue.claim(&t.id).is_err());
6736 assert_eq!(fresh.attempts, stale.attempts);
6737 drop((claim, permit));
6738 assert_eq!(ordinary.available_permits(), 1);
6739 assert_eq!(urgent.available_permits(), 1);
6740 }
6741 t.set_urgent(true).unwrap();
6742 queue.put(&mut t).unwrap();
6743 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6744 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6745 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6746 drop(queue.claim(&t.id).unwrap());
6747 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6748 drop(occupied);
6749 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6750 drop(queue.claim(&t.id).unwrap());
6751 let held = queue.claim(&t.id).unwrap();
6752 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6753 drop(held);
6754 t.status = TaskStatus::Held;
6755 queue.put(&mut t).unwrap();
6756 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6757 drop(queue.claim(&t.id).unwrap());
6758 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6759 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6760 drop(queue.claim(&t.id).unwrap());
6761 assert_eq!(urgent.available_permits(), 1);
6762 }
6763
6764 #[test]
6769 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6770 assert_eq!(permit_kind(true, false), PermitKind::None);
6771 assert_eq!(permit_kind(true, true), PermitKind::None);
6772 }
6773
6774 #[test]
6779 fn permit_kind_separates_urgent_from_ordinary() {
6780 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6781 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6782 }
6783
6784 #[test]
6791 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6792 let cfg = Config::default();
6793 let repo = Path::new("/any/repo/path");
6794
6795 let reason =
6796 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
6797 assert!(reason.contains("1024"), "{reason}");
6798 assert!(
6799 reason.contains(&cfg.disk.min_free_bytes.to_string()),
6800 "{reason}"
6801 );
6802
6803 assert_eq!(
6804 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
6805 None,
6806 "exactly at the floor is open"
6807 );
6808 assert_eq!(
6809 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
6810 None,
6811 "comfortably above the floor is open"
6812 );
6813 }
6814
6815 #[test]
6816 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
6817 let mut cfg = Config::default();
6818 cfg.disk.min_free_bytes = 0;
6819 let repo = Path::new("/any/repo/path");
6820 assert_eq!(
6821 disk_gate_with(repo, &cfg, |_| Ok(0)),
6822 None,
6823 "a zero floor never measures at all"
6824 );
6825 }
6826
6827 #[test]
6828 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
6829 let cfg = Config::default();
6830 let repo = Path::new("/any/repo/path");
6831 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
6832 .expect("a measurement failure must close the gate, not open it");
6833 assert!(reason.contains("could not measure"), "{reason}");
6834 }
6835
6836 #[test]
6837 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
6838 let ordinary = task();
6839 let next = advance_interrupt(
6840 Interrupt::Idle,
6841 std::slice::from_ref(&ordinary.id),
6842 std::slice::from_ref(&ordinary),
6843 );
6844 assert_eq!(next, Interrupt::Idle);
6845 }
6846
6847 #[test]
6848 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
6849 let marked = interrupt_task("marked");
6852 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6853 assert_eq!(next, Interrupt::Idle);
6854 }
6855
6856 #[test]
6857 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
6858 let marked = interrupt_task("marked");
6859 let next = advance_interrupt(
6860 Interrupt::Idle,
6861 &["running".to_owned()],
6862 std::slice::from_ref(&marked),
6863 );
6864 assert_eq!(
6865 next,
6866 Interrupt::Parking {
6867 parked: vec!["running".to_owned()],
6868 interrupt_task: "marked".to_owned(),
6869 }
6870 );
6871 }
6872
6873 #[test]
6882 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
6883 let marked = interrupt_task("marked");
6884
6885 let two = advance_interrupt(
6886 Interrupt::Idle,
6887 &["a".to_owned(), "b".to_owned()],
6888 std::slice::from_ref(&marked),
6889 );
6890 assert_eq!(two, Interrupt::Idle);
6891
6892 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6893 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
6894 }
6895
6896 #[test]
6897 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
6898 let state = Interrupt::Parking {
6899 parked: vec!["running".to_owned()],
6900 interrupt_task: "marked".to_owned(),
6901 };
6902 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
6904 assert_eq!(still_going, state);
6905
6906 let stopped_but_not_yet_dispatched =
6910 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
6911 assert_eq!(stopped_but_not_yet_dispatched, state);
6912
6913 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
6915 assert_eq!(
6916 dispatched,
6917 Interrupt::Running {
6918 parked: vec!["running".to_owned()],
6919 interrupt_task: "marked".to_owned(),
6920 }
6921 );
6922 }
6923
6924 #[test]
6925 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
6926 let state = Interrupt::Running {
6927 parked: vec!["running".to_owned()],
6928 interrupt_task: "marked".to_owned(),
6929 };
6930 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
6931 assert_eq!(still_running, state);
6932
6933 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
6940 assert_eq!(
6941 ended,
6942 Interrupt::Resuming {
6943 parked: vec!["running".to_owned()]
6944 }
6945 );
6946 }
6947
6948 #[test]
6949 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
6950 let state = Interrupt::Resuming {
6951 parked: vec!["running".to_owned()],
6952 };
6953 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
6954 assert_eq!(still_waiting, state);
6955
6956 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
6957 assert_eq!(dispatched, Interrupt::Idle);
6958 }
6959
6960 #[test]
6966 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
6967 {
6968 let state = Interrupt::Parking {
6969 parked: vec!["running".to_owned()],
6970 interrupt_task: "marked".to_owned(),
6971 };
6972 let next = advance_interrupt(state, &[], &[]);
6975 assert_eq!(
6976 next,
6977 Interrupt::Resuming {
6978 parked: vec!["running".to_owned()]
6979 },
6980 "abandoning the interrupt must not abandon the resume it owes"
6981 );
6982 }
6983
6984 #[test]
6987 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
6988 let state = Interrupt::Resuming {
6989 parked: vec!["running".to_owned()],
6990 };
6991 let next = advance_interrupt(state, &[], &[]);
6992 assert_eq!(
6993 next,
6994 Interrupt::Idle,
6995 "nothing is left to wait for; the loop must not stay wedged"
6996 );
6997 }
6998
6999 #[test]
7000 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7001 let marked = interrupt_task("marked");
7002 let next = advance_interrupt_tick(
7003 false,
7004 Interrupt::Idle,
7005 &["running".to_owned()],
7006 std::slice::from_ref(&marked),
7007 );
7008 assert_eq!(
7009 next,
7010 Interrupt::Idle,
7011 "an unmarked, unconfigured daemon must behave exactly as before"
7012 );
7013 }
7014
7015 #[test]
7016 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7017 let state = Interrupt::Parking {
7018 parked: vec!["running".to_owned()],
7019 interrupt_task: "marked".to_owned(),
7020 };
7021 let candidates = vec![interrupt_task("marked"), task()];
7022 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7023 assert!(
7024 allowed.is_empty(),
7025 "nothing may dispatch - not even the interrupt task itself - \
7026 until the parked run has actually stopped"
7027 );
7028 }
7029
7030 #[test]
7044 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7045 for state in [
7046 Interrupt::Parking {
7047 parked: vec!["running".to_owned()],
7048 interrupt_task: "marked".to_owned(),
7049 },
7050 Interrupt::Running {
7051 parked: vec!["running".to_owned()],
7052 interrupt_task: "marked".to_owned(),
7053 },
7054 Interrupt::Resuming {
7055 parked: vec!["running".to_owned()],
7056 },
7057 ] {
7058 let candidates = vec![
7059 interrupt_task("marked"),
7060 urgent_task("hot"),
7061 task_with_id("ordinary"),
7062 ];
7063 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7064 assert!(
7065 !allowed.iter().any(|t| t.id == "hot"),
7066 "an urgent candidate must wait out the same gate as anything \
7067 else while the run it would run alongside has not actually \
7068 left flight, for state {state:?}: {allowed:?}"
7069 );
7070 }
7071 }
7072
7073 #[test]
7079 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7080 let state = Interrupt::Resuming {
7081 parked: vec!["hot".to_owned()],
7082 };
7083 let candidates = vec![urgent_task("hot"), task()];
7084 let allowed = interrupt_gate(&state, &[], candidates);
7085 assert_eq!(
7086 allowed.iter().filter(|t| t.id == "hot").count(),
7087 1,
7088 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7089 );
7090 }
7091
7092 #[test]
7093 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7094 let state = Interrupt::Parking {
7095 parked: vec!["running".to_owned()],
7096 interrupt_task: "marked".to_owned(),
7097 };
7098 let other = task();
7099 let candidates = vec![interrupt_task("marked"), other.clone()];
7100 let allowed = interrupt_gate(&state, &[], candidates);
7101 assert_eq!(allowed.len(), 1);
7102 assert_eq!(allowed[0].id, "marked");
7103 }
7104
7105 #[test]
7106 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7107 let state = Interrupt::Running {
7108 parked: vec!["running".to_owned()],
7109 interrupt_task: "marked".to_owned(),
7110 };
7111 let candidates = vec![task(), task()];
7112 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7113 assert!(allowed.is_empty());
7114 }
7115
7116 #[test]
7123 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7124 let state = Interrupt::Resuming {
7125 parked: vec!["a".to_owned(), "c".to_owned()],
7126 };
7127 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7128 let allowed = interrupt_gate(&state, &[], candidates);
7129 assert_eq!(
7130 allowed.len(),
7131 1,
7132 "at most one candidate may be offered while resuming: {allowed:?}"
7133 );
7134 assert_eq!(allowed[0].id, "a");
7135 }
7136
7137 #[test]
7138 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7139 let state = Interrupt::Resuming {
7140 parked: vec!["a".to_owned()],
7141 };
7142 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7143 assert!(allowed.is_empty());
7144 }
7145
7146 #[test]
7152 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7153 let running = task(); let marked = interrupt_task("marked");
7155
7156 let mut state = Interrupt::Idle;
7157 let in_flight = vec![running.id.clone()];
7159 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7160 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7161 assert!(gated.is_empty(), "still waiting on `running` to park");
7162
7163 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7165 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7166 assert_eq!(
7167 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7168 vec!["marked"],
7169 "only the interrupt task may be offered to the dispatcher now"
7170 );
7171
7172 state = advance_interrupt_tick(
7174 true,
7175 state,
7176 &["marked".to_owned()],
7177 std::slice::from_ref(&running),
7178 );
7179 let gated = interrupt_gate(
7180 &state,
7181 &["marked".to_owned()],
7182 vec![marked.clone(), running.clone()],
7183 );
7184 assert!(
7185 gated.is_empty(),
7186 "the parked run must not be offered back while the interrupt \
7187 task is still running"
7188 );
7189
7190 let other = task_with_id("other");
7194 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7195 assert_eq!(
7196 state,
7197 Interrupt::Resuming {
7198 parked: vec![running.id.clone()]
7199 }
7200 );
7201 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7202 assert_eq!(
7203 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7204 vec![running.id.as_str()],
7205 "exactly the parked run resumes - not the unrelated task, even \
7206 though it was offered first"
7207 );
7208
7209 state = advance_interrupt_tick(
7213 true,
7214 state,
7215 std::slice::from_ref(&running.id),
7216 std::slice::from_ref(&other),
7217 );
7218 assert_eq!(state, Interrupt::Idle);
7219 let gated = interrupt_gate(
7220 &state,
7221 std::slice::from_ref(&running.id),
7222 vec![other.clone()],
7223 );
7224 assert_eq!(
7225 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7226 vec![other.id.as_str()],
7227 "ordinary dispatch is unrestricted again"
7228 );
7229 }
7230
7231 #[test]
7232 fn every_run_status_settles_the_task_it_came_from() {
7233 let table = [
7235 (RunStatus::Merged, TaskStatus::Done, 1),
7236 (RunStatus::Ready, TaskStatus::Done, 1),
7237 (RunStatus::Stalled, TaskStatus::Failed, 0),
7238 (RunStatus::Blocked, TaskStatus::Failed, 1),
7239 (RunStatus::Failed, TaskStatus::Failed, 1),
7240 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7241 (RunStatus::Prep, TaskStatus::Failed, 1),
7242 (RunStatus::Implementing, TaskStatus::Failed, 1),
7243 (RunStatus::Judging, TaskStatus::Failed, 1),
7244 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7245 (RunStatus::Voting, TaskStatus::Failed, 1),
7246 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7247 (RunStatus::Gating, TaskStatus::Failed, 1),
7248 ];
7249 for (run, want, attempts) in table {
7250 let mut t = task();
7251 t.start("20260902-000000-aaaa".to_owned());
7252 settle(
7253 &mut t,
7254 Verdict {
7255 status: run,
7256 left_pr: false,
7257 parked: false,
7258 quota_hit: matches!(run, RunStatus::Stalled),
7259 no_viable_candidates: false,
7260 },
7261 "why",
7262 2,
7263 );
7264 assert_eq!(t.status, want, "task status after {}", label(run));
7265 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7266 }
7267 }
7268
7269 #[test]
7270 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7271 let mut stalled = task();
7272 stalled.start("20260902-000000-aaaa".to_owned());
7273 settle(
7274 &mut stalled,
7275 Verdict {
7276 status: RunStatus::Stalled,
7277 left_pr: false,
7278 parked: false,
7279 quota_hit: true,
7280 no_viable_candidates: false,
7281 },
7282 "quota",
7283 1,
7284 );
7285 assert_eq!(stalled.attempts, 0);
7286 assert!(
7287 stalled.status.runnable(),
7288 "a machine problem must leave the task in line"
7289 );
7290
7291 let mut blocked = task();
7292 blocked.start("20260902-000000-aaaa".to_owned());
7293 settle(
7294 &mut blocked,
7295 Verdict {
7296 status: RunStatus::Blocked,
7297 left_pr: false,
7298 parked: false,
7299 quota_hit: false,
7300 no_viable_candidates: false,
7301 },
7302 "findings open",
7303 1,
7304 );
7305 assert_eq!(blocked.attempts, 1);
7306 assert_eq!(
7307 blocked.status,
7308 TaskStatus::Held,
7309 "the last attempt hands the task to a human"
7310 );
7311 }
7312
7313 #[test]
7314 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7315 let mut delivered = task();
7318 delivered.start("20260903-080619-01c2".to_owned());
7319 settle(
7320 &mut delivered,
7321 Verdict {
7322 status: RunStatus::Blocked,
7323 left_pr: true,
7324 parked: false,
7325 quota_hit: false,
7326 no_viable_candidates: false,
7327 },
7328 "no check status",
7329 4,
7330 );
7331 assert_eq!(
7332 delivered.status,
7333 TaskStatus::Held,
7334 "a pull request waiting on CI or a person is not a retryable failure"
7335 );
7336 assert!(
7337 !delivered.status.runnable(),
7338 "the loop must not pick this task up again"
7339 );
7340 assert_eq!(
7341 delivered.last_error.as_deref(),
7342 Some("no check status"),
7343 "the operator needs to be told what the gate was waiting for"
7344 );
7345
7346 let mut empty_handed = task();
7349 empty_handed.start("20260903-080619-01c2".to_owned());
7350 settle(
7351 &mut empty_handed,
7352 Verdict {
7353 status: RunStatus::Blocked,
7354 left_pr: false,
7355 parked: false,
7356 quota_hit: false,
7357 no_viable_candidates: false,
7358 },
7359 "findings open",
7360 4,
7361 );
7362 assert_eq!(empty_handed.status, TaskStatus::Failed);
7363 assert!(empty_handed.status.runnable());
7364 }
7365
7366 #[test]
7367 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7368 let mut noop = task();
7374 noop.start("20260912-131304-391f".to_owned());
7375 settle(
7376 &mut noop,
7377 Verdict {
7378 status: RunStatus::VerifiedNoop,
7379 left_pr: false,
7380 parked: false,
7381 quota_hit: false,
7382 no_viable_candidates: true,
7383 },
7384 "candidate A: already fixed by b32cfc4, on main",
7385 4,
7386 );
7387 assert_eq!(
7388 noop.status,
7389 TaskStatus::Held,
7390 "an unverified claim is a request for a human, not a failure"
7391 );
7392 assert!(
7393 !noop.status.runnable(),
7394 "the loop must not requeue this on the same unverified claim"
7395 );
7396 assert_eq!(noop.attempts, 1);
7401 }
7402
7403 #[test]
7404 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7405 let mut parked = task();
7410 parked.start("20260903-183634-2d98".to_owned());
7411 settle(
7412 &mut parked,
7413 Verdict {
7414 status: RunStatus::Implementing,
7415 left_pr: false,
7416 quota_hit: false,
7417 parked: true,
7418 no_viable_candidates: false,
7419 },
7420 "parked after `implementing`",
7421 2,
7422 );
7423 assert_eq!(parked.attempts, 0, "a park is refunded");
7424 assert!(
7425 parked.status.runnable(),
7426 "and the task stays in line so the next loop resumes its run"
7427 );
7428 assert_eq!(parked.status, TaskStatus::Parked);
7429 assert_eq!(
7430 parked.park_reason.as_deref(),
7431 Some("parked after `implementing`"),
7432 "the card says where it stopped"
7433 );
7434 assert_eq!(parked.last_error, None, "nothing failed");
7435
7436 let mut broken = task();
7440 broken.start("20260903-183634-2d98".to_owned());
7441 settle(
7442 &mut broken,
7443 Verdict {
7444 status: RunStatus::Implementing,
7445 left_pr: false,
7446 quota_hit: false,
7447 parked: false,
7448 no_viable_candidates: false,
7449 },
7450 "returned mid-flight",
7451 2,
7452 );
7453 assert_eq!(broken.attempts, 1);
7454 }
7455
7456 #[test]
7457 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7458 let mut flaky = task();
7463 flaky.start("20260903-123023-e633".to_owned());
7464 settle(
7465 &mut flaky,
7466 Verdict {
7467 status: RunStatus::Stalled,
7468 left_pr: false,
7469 parked: false,
7470 quota_hit: false,
7471 no_viable_candidates: false,
7472 },
7473 "verdict rests on 1 of 3 judges",
7474 2,
7475 );
7476 assert_eq!(
7477 flaky.attempts, 1,
7478 "flakiness spends an attempt, so `max_attempts` still bounds it"
7479 );
7480 assert!(flaky.status.runnable(), "and it is still worth retrying");
7481
7482 let mut limited = task();
7484 limited.start("20260903-123023-e633".to_owned());
7485 settle(
7486 &mut limited,
7487 Verdict {
7488 status: RunStatus::Stalled,
7489 left_pr: false,
7490 parked: false,
7491 quota_hit: true,
7492 no_viable_candidates: false,
7493 },
7494 "judge-2, judge-3 out of quota",
7495 2,
7496 );
7497 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7498 assert!(limited.status.runnable());
7499
7500 let mut worn = task();
7503 for _ in 0..2 {
7504 worn.release();
7505 }
7506 worn.start("20260903-123023-e633".to_owned());
7507 worn.attempts = 2;
7508 settle(
7509 &mut worn,
7510 Verdict {
7511 status: RunStatus::Stalled,
7512 left_pr: false,
7513 parked: false,
7514 quota_hit: false,
7515 no_viable_candidates: false,
7516 },
7517 "no quorum again",
7518 2,
7519 );
7520 assert_eq!(worn.status, TaskStatus::Held);
7521 assert!(!worn.status.runnable());
7522 }
7523
7524 #[test]
7525 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
7526 let mut wiped_out = task();
7533 wiped_out.start("20260907-025000-a1b2".to_owned());
7534 settle(
7535 &mut wiped_out,
7536 Verdict {
7537 status: RunStatus::Failed,
7538 left_pr: false,
7539 parked: false,
7540 quota_hit: true,
7541 no_viable_candidates: true,
7542 },
7543 "no candidate produced a change; nothing to judge",
7544 2,
7545 );
7546 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7547 assert!(
7548 wiped_out.status.runnable(),
7549 "a machine problem must leave the task in line"
7550 );
7551
7552 let mut partial_progress = task();
7558 partial_progress.start("20260907-025500-c3d4".to_owned());
7559 settle(
7560 &mut partial_progress,
7561 Verdict {
7562 status: RunStatus::Failed,
7563 left_pr: false,
7564 parked: false,
7565 quota_hit: true,
7566 no_viable_candidates: false,
7567 },
7568 "gate failed on the winning candidate",
7569 2,
7570 );
7571 assert_eq!(
7572 partial_progress.attempts, 1,
7573 "a candidate that actually produced a change spends the attempt \
7574 even though some other seat hit its quota"
7575 );
7576 assert!(partial_progress.status.runnable());
7577 }
7578
7579 #[test]
7580 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7581 let mut t = task();
7587 t.start("20260907-025000-a1b2".to_owned());
7588 let mut state = run_state(RunStatus::Failed);
7589 state.quota.push(QuotaLoss {
7590 seat: "cand-a".to_owned(),
7591 node: "implement".to_owned(),
7592 at: Timestamp::now(),
7593 reset: None,
7594 });
7595 assert!(
7596 state.viable().is_empty(),
7597 "no candidate was added, so nothing is viable"
7598 );
7599 reclaim(&mut t, Some(state), 2, "en");
7600 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7601 assert!(t.status.runnable());
7602 }
7603
7604 #[test]
7605 fn a_held_task_is_never_offered_to_the_loop() {
7606 let dir = tempfile::tempdir().unwrap();
7607 let queue = Queue::at(dir.path().to_path_buf());
7608 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7609 let mut t = task();
7610 t.id = format!("2026090{n}-000000-000{n}");
7611 t.priority = priority;
7612 queue.put(&mut t).unwrap();
7613 }
7614 let mut held = task();
7615 held.id = "20260909-000000-9999".to_owned();
7616 held.priority = 99;
7617 held.hold_machine(None);
7618 queue.put(&mut held).unwrap();
7619
7620 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7621 assert_eq!(order.len(), 3);
7622 assert!(!order.contains(&held.id));
7623 assert_eq!(
7624 order.first().cloned(),
7625 queue.next_runnable().map(|t| t.id),
7626 "the loop's first candidate is exactly what the queue offers"
7627 );
7628 assert_eq!(
7629 order,
7630 vec![
7631 "20260902-000000-0002".to_owned(),
7632 "20260903-000000-0003".to_owned(),
7633 "20260901-000000-0001".to_owned(),
7634 ],
7635 "priority first, then oldest, so nothing starves"
7636 );
7637 }
7638
7639 #[test]
7640 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7641 let dir = tempfile::tempdir().unwrap();
7642 let queue = Queue::at(dir.path().to_path_buf());
7643 let mut old = task();
7644 old.id = "20260101-000000-old0".to_owned();
7645 queue.put(&mut old).unwrap();
7646 let mut fresh = task();
7647 fresh.id = "20260101-000000-new0".to_owned();
7648 queue.put(&mut fresh).unwrap();
7649
7650 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7654 std::thread::sleep(Duration::from_millis(60));
7655 let live = queue.claim(&fresh.id).unwrap();
7656
7657 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7658 assert_eq!(swept, vec![old.id.clone()]);
7659 assert!(
7660 queue.claim(&old.id).is_ok(),
7661 "an unparseable lock older than the threshold is swept"
7662 );
7663 assert!(
7664 queue.claim(&fresh.id).is_err(),
7665 "a live pid protects its lock regardless of age"
7666 );
7667 drop(live);
7668 }
7669
7670 #[test]
7671 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7672 let dir = tempfile::tempdir().unwrap();
7675 let queue = Queue::at(dir.path().to_path_buf());
7676 let mut t = task();
7677 t.id = "20260101-000000-live".to_owned();
7678 queue.put(&mut t).unwrap();
7679
7680 let claim = queue.claim(&t.id).unwrap();
7681 std::thread::sleep(Duration::from_millis(60));
7682
7683 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7684 assert!(
7685 swept.is_empty(),
7686 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7687 );
7688 assert!(
7689 queue.claim(&t.id).is_err(),
7690 "the lock still protects its task"
7691 );
7692 drop(claim);
7693 }
7694
7695 fn injected_dead_pid() -> u32 {
7698 std::process::id().checked_add(1).unwrap_or(1)
7699 }
7700
7701 #[test]
7702 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7703 let dir = tempfile::tempdir().unwrap();
7704 let queue = Queue::at(dir.path().to_path_buf());
7705 let mut t = task();
7706 t.id = "20260101-000000-dead".to_owned();
7707 queue.put(&mut t).unwrap();
7708 let dead_pid = injected_dead_pid();
7709
7710 std::fs::write(
7715 dir.path().join(format!("{}.lock", t.id)),
7716 dead_pid.to_string(),
7717 )
7718 .unwrap();
7719
7720 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7721 pid != dead_pid
7722 });
7723 assert_eq!(
7724 swept,
7725 vec![t.id.clone()],
7726 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7727 );
7728 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7729 }
7730
7731 #[test]
7732 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7733 let dir = tempfile::tempdir().unwrap();
7734 let queue = Queue::at(dir.path().to_path_buf());
7735 let mut t = task();
7736 t.id = "20260101-000000-late".to_owned();
7737 queue.put(&mut t).unwrap();
7738 let dead_pid = injected_dead_pid();
7739
7740 assert!(
7743 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7744 "nothing has claimed the task yet"
7745 );
7746
7747 std::fs::write(
7750 dir.path().join(format!("{}.lock", t.id)),
7751 dead_pid.to_string(),
7752 )
7753 .unwrap();
7754
7755 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7759 pid != dead_pid
7760 });
7761 assert_eq!(swept, vec![t.id.clone()]);
7762 }
7763
7764 #[test]
7765 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7766 crate::run::pin_test_home();
7771 let dir = tempfile::tempdir().unwrap();
7772 let queue = Queue::at(dir.path().to_path_buf());
7773 let mut t = task();
7774 t.id = "20260101-000000-crsh".to_owned();
7775 t.status = TaskStatus::Running;
7776 t.attempts = 1;
7777 t.runs.push("20260904-000000-4043".to_owned());
7781 queue.put(&mut t).unwrap();
7782 let dead_pid = injected_dead_pid();
7783
7784 std::fs::write(
7787 dir.path().join(format!("{}.lock", t.id)),
7788 dead_pid.to_string(),
7789 )
7790 .unwrap();
7791
7792 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
7798 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
7799
7800 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7801 pid != dead_pid
7802 });
7803 assert_eq!(swept, vec![t.id.clone()]);
7804
7805 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
7806 assert_eq!(reclaimed, vec![t.id.clone()]);
7807 let after = queue.get(&t.id).unwrap();
7808 assert_eq!(
7809 after.status,
7810 TaskStatus::Held,
7811 "no run.json to recover from, so a human is asked"
7812 );
7813 assert_eq!(
7814 after.runs,
7815 vec!["20260904-000000-4043".to_owned()],
7816 "the crashed run's id is kept as evidence, not discarded"
7817 );
7818 }
7819
7820 #[test]
7821 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
7822 let dir = tempfile::tempdir().unwrap();
7823 let queue = Queue::at(dir.path().to_path_buf());
7824 let mut t = task();
7825 t.id = "20260101-000000-unknown".to_owned();
7826 queue.put(&mut t).unwrap();
7827 let dead_pid = injected_dead_pid();
7828 std::fs::write(
7829 dir.path().join(format!("{}.lock", t.id)),
7830 dead_pid.to_string(),
7831 )
7832 .unwrap();
7833
7834 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
7835 assert!(swept.is_empty(), "an unknown pid must keep its lock");
7836 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
7837 }
7838
7839 fn run_state_in(status: RunStatus, language: &str) -> RunState {
7840 let mut s = run_state(status);
7841 s.config.graph.language = language.to_owned();
7842 s
7843 }
7844
7845 fn unstarted_verdict(status: RunStatus) -> Verdict {
7846 Verdict {
7847 status,
7848 left_pr: false,
7849 quota_hit: false,
7850 parked: false,
7851 no_viable_candidates: false,
7852 }
7853 }
7854
7855 #[test]
7856 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
7857 let mut t = task();
7858 t.attempts = 1;
7859 settle_in(
7860 &mut t,
7861 unstarted_verdict(RunStatus::AlreadyInBase),
7862 "already in main",
7863 1,
7864 phrases("en"),
7865 );
7866 assert_eq!(t.status, TaskStatus::Done);
7867 assert_eq!(t.attempts, 0);
7868 }
7869
7870 #[test]
7871 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
7872 let reason = |language: &str| {
7873 let mut t = task();
7874 settle_in(
7875 &mut t,
7876 unstarted_verdict(RunStatus::Judging),
7877 "boom",
7878 1,
7879 phrases(language),
7880 );
7881 t.last_error.or(t.hold_reason).unwrap_or_default()
7882 };
7883 assert!(
7884 reason("en").starts_with("the graph stopped at `"),
7885 "{}",
7886 reason("en")
7887 );
7888 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
7889 assert!(reason("日本語").contains("boom"));
7890 assert_eq!(reason("fr"), reason("en"));
7891 }
7892
7893 #[test]
7894 fn describe_follows_the_run_language_and_keeps_the_run_id() {
7895 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
7896 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
7897 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
7898 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
7899 assert!(ja.contains("[run "), "{ja}");
7900 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
7901 assert!(ended.starts_with("run 終了: "), "{ended}");
7902 let mut de = run_state_in(RunStatus::Failed, "de");
7903 let mut en = run_state_in(RunStatus::Failed, "en");
7904 de.id = "same".to_owned();
7905 en.id = "same".to_owned();
7906 assert_eq!(describe(&de), describe(&en));
7907 }
7908
7909 #[test]
7910 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
7911 use crate::handover::Refused;
7912 let cases = [
7913 Refused::Foreign {
7914 branch: "b".into(),
7915 path: "/w/x".into(),
7916 why: "made by hand".into(),
7917 },
7918 Refused::Unsafe {
7919 branch: "b".into(),
7920 path: "/w/x".into(),
7921 why: "its worktree has uncommitted changes (a.rs)".into(),
7922 },
7923 Refused::ReleaseFailed {
7924 branch: "b".into(),
7925 path: "/w/x".into(),
7926 run: "ab12".into(),
7927 },
7928 ];
7929 for r in &cases {
7930 let en = (phrases("en").handover_refused)(r);
7931 assert_eq!(en, r.to_string());
7932 let ja = (phrases("ja").handover_refused)(r);
7933 assert!(
7934 !ja.contains("is checked out") && !ja.contains("try again"),
7935 "{ja}"
7936 );
7937 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
7938 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
7939 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
7940 }
7941 }
7942 assert!(phrases("en").handover_hint.contains("release the task"));
7943 assert!(phrases("ja").handover_hint.contains("解放"));
7944 }
7945
7946 #[test]
7947 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
7948 let build = |lang: &str| {
7949 let mut s = run_state_in(RunStatus::Blocked, lang);
7950 s.id = "same".to_owned();
7951 s.config.graph.review_rounds = 3;
7952 let mut r = review_round(3);
7953 r.expected = 3;
7954 r.answered = 1;
7955 s.reviews.push(r);
7956 s.event(
7957 "review",
7958 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
7959 );
7960 s
7961 };
7962 let en = describe(&build("en"));
7963 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
7964 let ja = describe(&build("ja"));
7965 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
7966 assert!(!ja.contains("never answered"), "{ja}");
7967 assert!(ja.contains("[run "), "{ja}");
7968
7969 let mut failed = build("ja");
7972 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
7973 command: "cargo test".to_owned(),
7974 code: Some(1),
7975 output_tail: String::new(),
7976 duration_ms: 0,
7977 resource_blocked: false,
7978 });
7979 failed.event("review", "stopped; e2e failed: cargo test");
7980 let ja = describe(&failed);
7981 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
7982 assert!(!ja.contains("席のレビュアー"), "{ja}");
7983 }
7984
7985 #[test]
7986 fn refusals_and_recovery_prose_follow_the_language() {
7987 let t = held_task_with("r1");
7988 let q = action_question(
7989 "r1",
7990 ask::ChoiceAction::Resume {
7991 run: "r1".to_owned(),
7992 },
7993 );
7994 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
7995 ActionDecision::Refuse(s) => s,
7996 other => panic!("{other:?}"),
7997 };
7998 assert!(refuse(phrases("en")).contains("could not be read: gone"));
7999 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8000 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8001
8002 let mut held = task();
8003 reclaim(&mut held, None, 2, "ja");
8004 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8005 let mut held = task();
8006 reclaim(&mut held, None, 2, "xx");
8007 assert!(held.hold_reason.unwrap().contains("held for a human"));
8008 }
8009
8010 fn run_state(status: RunStatus) -> RunState {
8011 let mut state = RunState::new(
8012 PathBuf::from("/repo"),
8013 "main".to_owned(),
8014 "abc1234def".to_owned(),
8015 "add retries".to_owned(),
8016 Config::default(),
8017 );
8018 state.status = status;
8019 state
8020 }
8021
8022 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8023 Candidate {
8024 index: 0,
8025 label,
8026 agent: "claude".to_owned(),
8027 branch: format!("magi/x/{label}"),
8028 worktree: PathBuf::from("/repo"),
8029 summary: summary.to_owned(),
8030 stat: String::new(),
8031 files: 0,
8032 commits: usize::from(!empty),
8033 empty,
8034 failed: failed.map(str::to_owned),
8035 verified_noop: None,
8036 duration_ms: 0,
8037 folded: false,
8038 }
8039 }
8040
8041 #[test]
8042 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8043 let mut state = run_state(RunStatus::Blocked);
8044 state.gate = vec![
8045 CommandOutcome {
8046 command: "cargo make check".to_owned(),
8047 code: Some(0),
8048 output_tail: "ok".to_owned(),
8049 duration_ms: 0,
8050 resource_blocked: false,
8051 },
8052 CommandOutcome {
8053 command: "cargo test".to_owned(),
8054 code: Some(101),
8055 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8056 duration_ms: 0,
8057 resource_blocked: false,
8058 },
8059 ];
8060 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8061 assert!(d.contains("cargo test"), "{d}");
8062 assert!(
8063 !d.contains("cargo make check"),
8064 "a passing check is not a diagnostic: {d}"
8065 );
8066 assert!(d.contains("assertion failed"), "{d}");
8067 }
8068
8069 #[test]
8070 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8071 let mut state = run_state(RunStatus::Blocked);
8072 state.event(
8073 "land",
8074 "stopped: the fixer produced no commit while 2 check(s) were failing \
8075 (build, lint); stopping instead of looping on an unchanged tree",
8076 );
8077 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8078 assert!(d.contains("build"), "{d}");
8079 assert!(d.contains("lint"), "{d}");
8080 assert!(d.contains("fixer produced no commit"), "{d}");
8081 }
8082
8083 #[test]
8084 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8085 let state = run_state(RunStatus::VerifiedNoop);
8090 let d = describe(&state);
8091 assert!(
8092 d.contains("agent-verified no-op"),
8093 "expected the display label, not the wire spelling: {d}"
8094 );
8095 assert!(!d.contains("verified_noop"), "{d}");
8096 }
8097
8098 #[test]
8099 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8100 let mut state = run_state(RunStatus::Failed);
8106 state.candidates = vec![candidate(
8107 'A',
8108 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8109 true,
8110 None,
8111 )];
8112 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8113 assert!(d.contains("candidate A"), "{d}");
8114 assert!(d.contains("tagged v1.2.3"), "{d}");
8115 }
8116
8117 #[test]
8118 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8119 let mut state = run_state(RunStatus::Failed);
8120 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8121 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8122 assert!(d.contains("candidate A"), "{d}");
8123 assert!(d.contains("agent timed out"), "{d}");
8124 }
8125
8126 #[test]
8127 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8128 let mut state = run_state(RunStatus::Failed);
8131 state.candidates = vec![candidate('A', "did the work", false, None)];
8132 assert!(diagnostic(&state).is_none());
8133 }
8134
8135 #[test]
8136 fn diagnostic_is_bounded_however_much_a_run_printed() {
8137 let mut state = run_state(RunStatus::Blocked);
8138 state.gate = vec![
8139 CommandOutcome {
8140 command: "cargo test".to_owned(),
8141 code: Some(101),
8142 output_tail: "x".repeat(50_000),
8143 duration_ms: 0,
8144 resource_blocked: false,
8145 },
8146 CommandOutcome {
8147 command: "cargo clippy".to_owned(),
8148 code: Some(1),
8149 output_tail: "y".repeat(50_000),
8150 duration_ms: 0,
8151 resource_blocked: false,
8152 },
8153 ];
8154 state.candidates = vec![
8155 candidate('A', &"z".repeat(50_000), true, None),
8156 candidate('B', &"w".repeat(50_000), true, None),
8157 ];
8158 let d = diagnostic(&state).expect("plenty here to diagnose");
8159 assert!(
8160 d.len() <= DIAGNOSTIC_MAX,
8161 "diagnostic grew to {} bytes, unbounded",
8162 d.len()
8163 );
8164 }
8165
8166 #[test]
8167 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8168 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8169 let mut t = task();
8170 let mut state = run_state(status);
8171 t.pr_recovery = Some(crate::pr_recovery::Selection {
8172 url: "url".into(),
8173 branch: format!("magi/task-{}", t.id),
8174 head: "a".repeat(40),
8175 source_head: None,
8176 source_run: None,
8177 source_proof: None,
8178 disposition: Some(crate::pr_recovery::Disposition::Land),
8179 instruction: t.instruction.clone(),
8180 approved_at: Timestamp::now(),
8181 });
8182 state.pr_recovery = t.pr_recovery.clone();
8183 t.start(state.id.clone());
8184 settle_and_diagnose(
8185 &mut t,
8186 Verdict {
8187 status,
8188 left_pr: false,
8189 quota_hit: false,
8190 parked: false,
8191 no_viable_candidates: false,
8192 },
8193 "delivery stopped",
8194 5,
8195 &state,
8196 );
8197 assert_eq!(t.status, TaskStatus::Held);
8198 let reason = t.hold_reason.as_deref().unwrap();
8199 if status == RunStatus::Ready {
8200 assert!(reason.contains("Selected PR delivery is incomplete"));
8201 assert!(t.recovery_budget.is_none());
8202 } else {
8203 assert!(reason.contains("Selected recovery stopped"));
8204 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8205 }
8206 assert!(t.pr_recovery.is_some());
8207 }
8208 }
8209
8210 #[test]
8211 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8212 let mut state = run_state(RunStatus::Blocked);
8213 state.gate = vec![CommandOutcome {
8214 command: "cargo test".to_owned(),
8215 code: Some(101),
8216 output_tail: "assertion failed".to_owned(),
8217 duration_ms: 0,
8218 resource_blocked: false,
8219 }];
8220 let verdict = Verdict {
8221 status: RunStatus::Blocked,
8222 left_pr: false,
8223 quota_hit: false,
8224 parked: false,
8225 no_viable_candidates: false,
8226 };
8227
8228 let mut t = task();
8231 t.start("run-1".to_owned());
8232 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8233 assert_eq!(t.status, TaskStatus::Failed);
8234 assert!(t.diagnostic.is_none());
8235
8236 t.start("run-2".to_owned());
8239 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8240 assert_eq!(t.status, TaskStatus::Held);
8241 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8242 assert!(d.contains("cargo test"), "{d}");
8243 }
8244
8245 #[test]
8246 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8247 crate::run::pin_test_home();
8252 let home = crate::run::home();
8253 let state = run_state(RunStatus::VerifiedNoop);
8254 let mut q = ask::Question::new(
8255 state.id.clone(),
8256 "implement".to_owned(),
8257 "impl-A".to_owned(),
8258 "is this really a no-op?".to_owned(),
8259 String::new(),
8260 Vec::new(),
8261 );
8262 Questions::at(home.join("questions")).put(&mut q).unwrap();
8263
8264 let verdict = Verdict {
8265 status: RunStatus::VerifiedNoop,
8266 left_pr: false,
8267 quota_hit: false,
8268 parked: false,
8269 no_viable_candidates: false,
8270 };
8271 let mut t = task();
8272 t.start(state.id.clone());
8273 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8274
8275 assert_eq!(t.status, TaskStatus::Held);
8276 let reason = t.hold_reason.expect("a held task must record why");
8277 assert!(
8278 reason.starts_with("run ended agent-verified no-op"),
8279 "the original settle reason must survive unchanged: {reason}"
8280 );
8281 assert!(
8282 reason.contains(q.short()),
8283 "the open question's id must be named so the notice is actionable: {reason}"
8284 );
8285 }
8286
8287 #[test]
8288 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8289 crate::run::pin_test_home();
8290 let state = run_state(RunStatus::VerifiedNoop);
8291
8292 let verdict = Verdict {
8293 status: RunStatus::VerifiedNoop,
8294 left_pr: false,
8295 quota_hit: false,
8296 parked: false,
8297 no_viable_candidates: false,
8298 };
8299 let mut t = task();
8300 t.start(state.id.clone());
8301 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8302
8303 assert_eq!(t.status, TaskStatus::Held);
8304 assert_eq!(
8305 t.hold_reason.as_deref(),
8306 Some("run ended agent-verified no-op"),
8307 "nothing to append when the question was already answered or never asked"
8308 );
8309 }
8310
8311 #[test]
8312 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8313 crate::run::pin_test_home();
8314 let mut first = run_state(RunStatus::Blocked);
8315 first.id = "20260101-000000-sup1".to_owned();
8316 first.save().unwrap();
8317 let mut second = run_state(RunStatus::Merged);
8318 second.id = "20260101-000000-sup2".to_owned();
8319 second.save().unwrap();
8320
8321 let mut t = task();
8322 t.runs = vec![first.id.clone(), second.id.clone()];
8323 t.status = TaskStatus::Done;
8324
8325 supersede_prior_runs(&t, &crate::run::home());
8326
8327 assert_eq!(
8328 RunState::load(&first.id).unwrap().status,
8329 RunStatus::Superseded,
8330 "the first attempt's Blocked no longer needs anyone's attention"
8331 );
8332 assert_eq!(
8333 RunState::load(&second.id).unwrap().status,
8334 RunStatus::Merged,
8335 "the run that actually succeeded is left exactly as it was"
8336 );
8337 }
8338
8339 #[test]
8340 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8341 crate::run::pin_test_home();
8347 let mut first = run_state(RunStatus::Blocked);
8348 first.id = "20260101-000000-sup9".to_owned();
8349 first.driver_pid = Some(std::process::id());
8352 first.driver_started_at = Some(
8353 crate::proc::process_started_at(std::process::id())
8354 .expect("this test process's own start time must be queryable"),
8355 );
8356 first.save().unwrap();
8357 let mut second = run_state(RunStatus::Merged);
8358 second.id = "20260101-000000-supa".to_owned();
8359 second.save().unwrap();
8360
8361 let mut t = task();
8362 t.runs = vec![first.id.clone(), second.id.clone()];
8363 t.status = TaskStatus::Done;
8364
8365 supersede_prior_runs(&t, &crate::run::home());
8366
8367 assert_eq!(
8368 RunState::load(&first.id).unwrap().status,
8369 RunStatus::Blocked,
8370 "a live driver_pid means something is still actually working this run, \
8371 even though no daemon claims it - rewriting under it would just be \
8372 undone the next time that process saves"
8373 );
8374 }
8375
8376 #[test]
8377 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8378 let dir = tempfile::tempdir().unwrap();
8384 let home = dir.path().to_path_buf();
8385 let queue = Queue::at(dir.path().join("queue"));
8386
8387 let mut first = run_state(RunStatus::Blocked);
8388 first.id = "20260101-000000-supd".to_owned();
8389 first.driver_pid = Some(std::process::id());
8390 first.driver_started_at = Some(
8391 crate::proc::process_started_at(std::process::id())
8392 .expect("this test process's own start time must be queryable"),
8393 );
8394 first.save_under(&home).unwrap();
8395 let mut second = run_state(RunStatus::Merged);
8396 second.id = "20260101-000000-supe".to_owned();
8397 second.save_under(&home).unwrap();
8398
8399 let mut t = task();
8400 t.runs = vec![first.id.clone(), second.id.clone()];
8401 t.status = TaskStatus::Done;
8402 queue.put(&mut t).unwrap();
8403
8404 resweep_superseded_attempts(&queue, &home);
8405 assert_eq!(
8406 RunState::load_under(&first.id, &home).unwrap().status,
8407 RunStatus::Blocked,
8408 "still live on the first pass, so still untouched"
8409 );
8410
8411 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8417 stale.driver_started_at = Some("1".to_owned());
8418 stale.save_under(&home).unwrap();
8419
8420 resweep_superseded_attempts(&queue, &home);
8421 assert_eq!(
8422 RunState::load_under(&first.id, &home).unwrap().status,
8423 RunStatus::Superseded,
8424 "the second pass catches up what the first one correctly skipped"
8425 );
8426 }
8427
8428 #[test]
8429 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8430 crate::run::pin_test_home();
8431 let mut first = run_state(RunStatus::Blocked);
8432 first.id = "20260101-000000-sup3".to_owned();
8433 first.save().unwrap();
8434 let mut second = run_state(RunStatus::Blocked);
8435 second.id = "20260101-000000-sup4".to_owned();
8436 second.save().unwrap();
8437
8438 let mut t = task();
8439 t.runs = vec![first.id.clone(), second.id.clone()];
8440 t.status = TaskStatus::Failed;
8444
8445 supersede_prior_runs(&t, &crate::run::home());
8446
8447 assert_eq!(
8448 RunState::load(&first.id).unwrap().status,
8449 RunStatus::Blocked
8450 );
8451 assert_eq!(
8452 RunState::load(&second.id).unwrap().status,
8453 RunStatus::Blocked
8454 );
8455 }
8456
8457 #[test]
8458 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
8459 crate::run::pin_test_home();
8463 let mut first = run_state(RunStatus::Blocked);
8464 first.id = "20260101-000000-sup5".to_owned();
8465 first.save().unwrap();
8466
8467 let mut t = task();
8468 t.runs = vec![first.id.clone()];
8469 t.status = TaskStatus::Done;
8470
8471 supersede_prior_runs(&t, &crate::run::home());
8472
8473 assert_eq!(
8474 RunState::load(&first.id).unwrap().status,
8475 RunStatus::Blocked,
8476 "a single-attempt task has no earlier run to supersede"
8477 );
8478 }
8479
8480 #[test]
8481 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
8482 crate::run::pin_test_home();
8489 let mut first = run_state(RunStatus::Blocked);
8490 first.id = "20260101-000000-supb".to_owned();
8491 first.save().unwrap();
8492 let mut second = run_state(RunStatus::Failed);
8493 second.id = "20260101-000000-supc".to_owned();
8494 second.save().unwrap();
8495
8496 let mut t = task();
8497 t.runs = vec![first.id.clone(), second.id.clone()];
8498 t.status = TaskStatus::Done;
8499
8500 supersede_prior_runs(&t, &crate::run::home());
8501
8502 assert_eq!(
8503 RunState::load(&first.id).unwrap().status,
8504 RunStatus::Blocked,
8505 "the task's last attempt never landed, so there is nothing here \
8506 actually superseding it"
8507 );
8508 }
8509
8510 #[test]
8511 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
8512 crate::run::pin_test_home();
8516 let mut failed = run_state(RunStatus::Failed);
8517 failed.id = "20260101-000000-sup6".to_owned();
8518 failed.save().unwrap();
8519 let mut noop = run_state(RunStatus::VerifiedNoop);
8520 noop.id = "20260101-000000-sup7".to_owned();
8521 noop.save().unwrap();
8522 let mut winner = run_state(RunStatus::Ready);
8523 winner.id = "20260101-000000-sup8".to_owned();
8524 winner.save().unwrap();
8525
8526 let mut t = task();
8527 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8528 t.status = TaskStatus::Done;
8529
8530 supersede_prior_runs(&t, &crate::run::home());
8531
8532 assert_eq!(
8533 RunState::load(&failed.id).unwrap().status,
8534 RunStatus::Failed
8535 );
8536 assert_eq!(
8537 RunState::load(&noop.id).unwrap().status,
8538 RunStatus::VerifiedNoop
8539 );
8540 }
8541
8542 fn approval_question(run: &str) -> ask::Question {
8543 ask::Question::new(
8544 run.to_owned(),
8545 land::APPROVAL_NODE.to_owned(),
8546 "land".to_owned(),
8547 "merge?".to_owned(),
8548 String::new(),
8549 vec!["merge".to_owned(), "hold".to_owned()],
8550 )
8551 }
8552
8553 #[test]
8554 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8555 crate::run::pin_test_home();
8556 let mut state = run_state(RunStatus::Landing);
8557 state.id = "20260101-000000-fre1".to_owned();
8558 state.parked = true;
8559 state.save().unwrap();
8560 ask::Questions::open()
8561 .put(&mut approval_question(&state.id))
8562 .unwrap();
8563
8564 let mut t = task();
8565 t.runs.push(state.id.clone());
8566 assert_eq!(
8567 land_resume_state(&t),
8568 LandResume::StillWaiting,
8569 "nobody has answered and the timeout has not passed"
8570 );
8571 }
8572
8573 #[test]
8574 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8575 crate::run::pin_test_home();
8576 let mut state = run_state(RunStatus::Gating);
8577 state.id = "20260101-000000-gt01".to_owned();
8578 state.parked = true;
8579 state.config.graph.answer_timeout = 60;
8580 state.github_text = Some(crate::run::GithubTextAsk {
8581 rewrite: None,
8582 fingerprint: "f".into(),
8583 title: true,
8584 body: false,
8585 categories: vec!["title-language".into()],
8586 question: None,
8587 asks: 1,
8588 resolved: false,
8589 chosen_title: None,
8590 choice: None,
8591 });
8592 state.save().unwrap();
8593 let store = ask::Questions::open();
8594 let mut q = ask::Question::new(
8595 state.id.clone(),
8596 crate::github_text::ASK_NODE.to_owned(),
8597 crate::github_text::ASK_SEAT.to_owned(),
8598 "s".into(),
8599 String::new(),
8600 vec!["use fallback".into()],
8601 );
8602 store.put(&mut q).unwrap();
8603 let mut t = task();
8604 t.runs.push(state.id.clone());
8605 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8606 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8607 store.put(&mut q).unwrap();
8608 assert_eq!(land_resume_state(&t), LandResume::Ready);
8609 assert!(!store.get(&q.id).unwrap().status.open());
8610 state.github_text.as_mut().unwrap().resolved = true;
8612 state.save().unwrap();
8613 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8614 }
8615
8616 #[test]
8617 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8618 crate::run::pin_test_home();
8623 let mut state = run_state(RunStatus::Landing);
8624 state.id = "20260101-000000-exp1".to_owned();
8625 state.parked = true;
8626 state.config.graph.answer_timeout = 60;
8627 state.save().unwrap();
8628
8629 let store = ask::Questions::open();
8630 let mut q = approval_question(&state.id);
8631 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8632 store.put(&mut q).unwrap();
8633
8634 let mut t = task();
8635 t.runs.push(state.id.clone());
8636 assert_eq!(
8637 land_resume_state(&t),
8638 LandResume::Ready,
8639 "an expired question must not be waited on forever"
8640 );
8641
8642 let after = store.get(&q.id).unwrap();
8643 assert!(
8644 !after.status.open(),
8645 "the question is abandoned, not silently ignored"
8646 );
8647 assert!(
8648 after.resolution().is_none(),
8649 "an abandoned question is not read as a decision"
8650 );
8651 }
8652
8653 #[test]
8654 fn reclaim_settles_a_running_task_against_its_last_run() {
8655 let mut t = task();
8656 t.start("20260904-000000-4043".to_owned());
8657 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8658 assert_eq!(
8659 t.status,
8660 TaskStatus::Done,
8661 "a run that actually finished must not stay `running` forever"
8662 );
8663 }
8664
8665 #[test]
8666 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8667 let mut t = task();
8668 t.start("20260904-000000-4043".to_owned());
8669 t.last_error = Some("earlier trouble".to_owned());
8670 let mut state = run_state(RunStatus::Judging);
8671 state.parked = true;
8672 reclaim(&mut t, Some(state), 1, "en");
8673 assert_eq!(t.status, TaskStatus::Parked);
8674 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8675 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8676 assert!(t.park_reason.is_some());
8677 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8678 assert!(!t.fresh_start);
8679 assert!(t.status.runnable());
8680 }
8681
8682 #[test]
8683 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8684 let mut t = task();
8685 t.start("20260904-000000-4043".to_owned());
8686 t.last_error = Some("earlier trouble".to_owned());
8687 settle(
8688 &mut t,
8689 Verdict {
8690 status: RunStatus::Judging,
8691 left_pr: false,
8692 quota_hit: false,
8693 parked: true,
8694 no_viable_candidates: false,
8695 },
8696 "parked after `judging`",
8697 1,
8698 );
8699 assert_eq!(t.status, TaskStatus::Parked);
8700 assert_eq!(t.attempts, 0);
8701 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8702 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8703 assert_eq!(t.schema, crate::queue::SCHEMA);
8704 assert!(
8705 crate::notices::task_held(&t).is_none(),
8706 "a park is not news"
8707 );
8708 t.start("20260904-000000-4043".to_owned());
8709 assert_eq!(t.park_reason, None);
8710 }
8711
8712 #[test]
8713 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8714 let mut t = task();
8718 t.start("20260904-000000-4043".to_owned());
8719 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8720 assert_eq!(t.status, TaskStatus::Failed);
8721 assert!(t.status.runnable());
8722 }
8723
8724 #[test]
8725 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8726 let mut t = task();
8727 t.start("20260904-000000-4043".to_owned());
8728 assert_eq!(t.attempts, 1);
8729 let mut run = run_state(RunStatus::Implementing);
8730 run.id = "20260904-000000-4043".to_owned();
8731 run.begin_pass();
8732 let cause = crate::run::ParkCause::Upgrade {
8733 from: "0.121.0".to_owned(),
8734 to: Some("0.121.1".to_owned()),
8735 };
8736 run.end_pass_parked(cause.clone());
8737 run.parked = true;
8738 let verdict = Verdict {
8739 status: run.status,
8740 left_pr: false,
8741 quota_hit: false,
8742 parked: true,
8743 no_viable_candidates: false,
8744 };
8745 let mut orphan = t.clone();
8748 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8749 assert_eq!(orphan.status, TaskStatus::Parked);
8750 assert_eq!(orphan.attempts, 0);
8751 assert_eq!(
8752 orphan.upgrade_resume_key(),
8753 Some((run.passes[0].started_at, orphan.id.as_str()))
8754 );
8755 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8756 assert_eq!(t.attempts, 0);
8757 assert_eq!(t.passes.len(), 1);
8758 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8759 assert_eq!(t.passes[0].parked, Some(cause));
8760 assert_eq!(t.passes[0].pass_index, 0);
8761 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8762 assert!(t.upgrade_resume_key().is_some());
8763
8764 t.start("20260904-000000-4043".to_owned());
8767 run.begin_pass();
8768 run.status = RunStatus::Failed;
8769 let verdict = Verdict {
8770 status: RunStatus::Failed,
8771 left_pr: false,
8772 quota_hit: false,
8773 parked: false,
8774 no_viable_candidates: false,
8775 };
8776 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8777 assert_eq!(t.passes.len(), 2);
8778 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8779 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8780 assert_eq!(t.passes[1].pass_index, 1);
8781 assert_eq!(t.attempts, 1);
8782 }
8783
8784 #[test]
8785 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8786 let mut t = task();
8787 t.start("20260904-000000-4043".to_owned());
8788 reclaim(&mut t, None, 2, "en");
8789 assert_eq!(t.status, TaskStatus::Held);
8790 assert!(
8791 t.last_error
8792 .as_deref()
8793 .is_some_and(|e| e.contains("running")),
8794 "the operator needs to know why this task was held"
8795 );
8796 }
8797
8798 #[test]
8799 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
8800 let dir = tempfile::tempdir().unwrap();
8801 let queue = Queue::at(dir.path().to_path_buf());
8802
8803 let mut orphaned = task();
8805 orphaned.id = "20260904-000000-orph".to_owned();
8806 orphaned.status = TaskStatus::Running;
8807 orphaned.attempts = 1;
8808 queue.put(&mut orphaned).unwrap();
8809
8810 let mut alive = task();
8811 alive.id = "20260904-000000-live".to_owned();
8812 alive.status = TaskStatus::Running;
8813 alive.attempts = 1;
8814 queue.put(&mut alive).unwrap();
8815 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
8816
8817 let mut queued = task();
8818 queued.id = "20260904-000000-wait".to_owned();
8819 queue.put(&mut queued).unwrap();
8820
8821 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8822 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
8823
8824 assert_eq!(
8825 queue.get(&orphaned.id).unwrap().status,
8826 TaskStatus::Held,
8827 "nothing was driving it and there was no run to recover"
8828 );
8829 assert_eq!(
8830 queue.get(&alive.id).unwrap().status,
8831 TaskStatus::Running,
8832 "a live claim must protect the task it belongs to"
8833 );
8834 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
8835 }
8836
8837 fn read_run_under(home: &Path, id: &str) -> RunState {
8843 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
8844 serde_json::from_str(&body).unwrap()
8845 }
8846
8847 #[test]
8848 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
8849 let dir = tempfile::tempdir().unwrap();
8850 let home = dir.path().to_path_buf();
8851 let now = Timestamp::now();
8852 let overrun_seat = || crate::run::ActiveSeat {
8853 node: "implement".to_owned(),
8854 started_at: now - jiff::SignedDuration::new(21_000, 0),
8855 timeout_secs: 3_600,
8856 attempt: 0,
8857 task: None,
8858 command: None,
8859 index: None,
8860 total: None,
8861 };
8862
8863 let mut dead = run_state(RunStatus::Implementing);
8864 dead.id = "20260101-000000-dead".to_owned();
8865 dead.active.insert("impl-A".to_owned(), overrun_seat());
8866 dead.driver_pid = Some(4242);
8869 dead.save_under(&home).unwrap();
8870
8871 let mut alive = run_state(RunStatus::Implementing);
8874 alive.id = "20260101-000000-aliv".to_owned();
8875 alive.active.insert("impl-A".to_owned(), overrun_seat());
8876 alive.save_under(&home).unwrap();
8877 let mut status = Status::new();
8878 status.current = vec![Current {
8879 task: "20260101-000000-task".to_owned(),
8880 run: alive.id.clone(),
8881 }];
8882 write_status_to(&home.join("daemon.json"), &status).unwrap();
8883
8884 let questions = Questions::at(home.join("questions"));
8888 let mut q = ask::Question::new(
8889 dead.id.clone(),
8890 "implement".to_owned(),
8891 "impl-A".to_owned(),
8892 "Which storage backend?".to_owned(),
8893 String::new(),
8894 vec!["SQLite".to_owned(), "Redis".to_owned()],
8895 );
8896 questions.put(&mut q).unwrap();
8897
8898 let abandoned = reclaim_abandoned_runs_with(
8899 &home,
8900 now,
8901 |pid| if pid == 4242 { Some(false) } else { None },
8902 |_| panic!("a query answering Dead outright needs no identity corroboration"),
8903 );
8904 assert_eq!(abandoned, vec![dead.id.clone()]);
8905
8906 let reloaded = read_run_under(&home, &dead.id);
8907 assert_eq!(reloaded.status, RunStatus::Failed);
8908 assert!(reloaded.active.is_empty());
8909 assert!(
8910 !questions.get(&q.id).unwrap().status.open(),
8911 "the failed run's own open question must be settled in the same pass"
8912 );
8913
8914 let still_alive = read_run_under(&home, &alive.id);
8915 assert_eq!(
8916 still_alive.status,
8917 RunStatus::Implementing,
8918 "a live daemon's claim protects it"
8919 );
8920 assert!(!still_alive.active.is_empty());
8921 }
8922
8923 #[test]
8933 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
8934 let dir = tempfile::tempdir().unwrap();
8935 let home = dir.path().to_path_buf();
8936 let now = Timestamp::now();
8937
8938 let mut manual = run_state(RunStatus::Reviewing);
8939 manual.id = "20260101-000000-manl".to_owned();
8940 manual.active.insert(
8941 "review-1".to_owned(),
8942 crate::run::ActiveSeat {
8943 node: "review".to_owned(),
8944 started_at: now - jiff::SignedDuration::new(21_000, 0),
8945 timeout_secs: 3_600,
8946 attempt: 0,
8947 task: None,
8948 command: None,
8949 index: None,
8950 total: None,
8951 },
8952 );
8953 manual.driver_pid = Some(4242);
8957 manual.driver_started_at = Some("1790000000".to_owned());
8958 manual.save_under(&home).unwrap();
8959
8960 let abandoned = reclaim_abandoned_runs_with(
8961 &home,
8962 now,
8963 |pid| if pid == 4242 { Some(true) } else { None },
8964 |pid| {
8965 if pid == 4242 {
8966 Some("1790000000".to_owned())
8967 } else {
8968 None
8969 }
8970 },
8971 );
8972 assert!(
8973 abandoned.is_empty(),
8974 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
8975 );
8976
8977 let reloaded = read_run_under(&home, &manual.id);
8978 assert_eq!(reloaded.status, RunStatus::Reviewing);
8979 assert!(!reloaded.active.is_empty());
8980 }
8981
8982 #[test]
8983 fn an_already_claimed_task_is_skipped_rather_than_failed() {
8984 let dir = tempfile::tempdir().unwrap();
8985 let queue = Queue::at(dir.path().to_path_buf());
8986 let mut only = task();
8987 queue.put(&mut only).unwrap();
8988
8989 let _elsewhere = queue.claim(&only.id).unwrap();
8990 let candidates = runnable(&queue);
8991 assert_eq!(candidates.len(), 1, "the task is still runnable");
8992 assert!(
8993 queue.claim(&candidates[0].id).is_err(),
8994 "the loop cannot take a claim somebody else holds"
8995 );
8996
8997 let after = queue.get(&only.id).unwrap();
8998 assert_eq!(after.status, TaskStatus::Queued);
8999 assert_eq!(
9000 after.attempts, 0,
9001 "losing the race is not an attempt at the task"
9002 );
9003 assert_eq!(after.last_error, None);
9004 }
9005
9006 #[test]
9007 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9008 let dir = tempfile::tempdir().unwrap();
9009 let path = dir.path().join("daemon.json");
9010
9011 let mut status = Status::new();
9012 status.idle = false;
9013 status.completed = 7;
9014 status.current = vec![Current {
9015 task: "20260902-000000-t111".to_owned(),
9016 run: "20260902-000001-r111".to_owned(),
9017 }];
9018 write_status_to(&path, &status).unwrap();
9019 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9020 assert_eq!(first.schema, SCHEMA);
9021 assert_eq!(first.pid, std::process::id());
9022 assert!(!first.idle);
9023 assert_eq!(first.completed, 7);
9024 assert_eq!(first.current, status.current);
9025 assert!(
9026 !path.with_extension("json.tmp").exists(),
9027 "the temp file is renamed, not left behind"
9028 );
9029
9030 std::thread::sleep(Duration::from_millis(5));
9031 status.updated_at = Timestamp::now();
9032 status.polls = 3;
9033 write_status_to(&path, &status).unwrap();
9034 let second: Status =
9035 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9036 assert!(
9037 second.updated_at > first.updated_at,
9038 "a reader can only detect staleness if the heartbeat moves"
9039 );
9040 assert_eq!(
9041 second.started_at, first.started_at,
9042 "the start time is not a heartbeat"
9043 );
9044 assert_eq!(second.polls, 3);
9045 }
9046
9047 #[test]
9048 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9049 let dir = tempfile::tempdir().unwrap();
9050
9051 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9052
9053 let mut status = Status::new();
9054 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9055 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9056 let stale = read_status(dir.path()).unwrap();
9057 assert!(
9058 !stale.running(Timestamp::now()),
9059 "a minute without a heartbeat is a dead daemon, not a busy one"
9060 );
9061 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9062
9063 status.updated_at = Timestamp::now();
9064 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9065 let fresh = read_status(dir.path()).unwrap();
9066 assert!(fresh.running(Timestamp::now()));
9067 }
9068
9069 #[test]
9070 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9071 let dir = tempfile::tempdir().unwrap();
9072 let now = Timestamp::now();
9073 let mine = "20260903-080619-01c2";
9074
9075 assert!(
9076 !is_working_on(dir.path(), mine, now),
9077 "no status file means nobody is working on anything"
9078 );
9079
9080 let mut status = Status::new();
9081 status.current = vec![Current {
9082 task: "20260903-080340-0167".to_owned(),
9083 run: mine.to_owned(),
9084 }];
9085 status.updated_at = now;
9086 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9087 assert!(is_working_on(dir.path(), mine, now));
9088 assert!(
9089 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9090 "a daemon busy with one run is not working on another"
9091 );
9092
9093 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9096 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9097 assert!(
9098 !is_working_on(dir.path(), mine, now),
9099 "a stale heartbeat is a dead daemon, so its run is a leftover"
9100 );
9101 }
9102
9103 #[test]
9104 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9105 let dir = tempfile::tempdir().unwrap();
9106 let now = Timestamp::now();
9107
9108 assert!(
9109 !is_working_on_short(dir.path(), "01c2", now),
9110 "no status file means nobody is working on anything"
9111 );
9112
9113 let mut status = Status::new();
9114 status.current = vec![Current {
9115 task: "20260903-080340-0167".to_owned(),
9116 run: "20260903-080619-01c2".to_owned(),
9117 }];
9118 status.updated_at = now;
9119 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9120 assert!(
9121 is_working_on_short(dir.path(), "01c2", now),
9122 "the run's short id is the last block of its full id"
9123 );
9124 assert!(
9125 !is_working_on_short(dir.path(), "3cbf", now),
9126 "a daemon busy with one worktree bay is not working on another"
9127 );
9128 }
9129
9130 #[test]
9131 fn a_newer_status_file_still_yields_a_reading() {
9132 let dir = tempfile::tempdir().unwrap();
9133 std::fs::write(
9136 dir.path().join("daemon.json"),
9137 serde_json::json!({
9138 "schema": 2,
9139 "updated_at": Timestamp::now().to_string(),
9140 "idle": true,
9141 "surprise": { "nested": [1, 2, 3] },
9142 })
9143 .to_string(),
9144 )
9145 .unwrap();
9146
9147 let reading = read_status(dir.path()).expect("a forward-compatible read");
9148 assert!(reading.running(Timestamp::now()));
9149 assert!(reading.idle);
9150 assert!(reading.current.is_empty());
9151 }
9152
9153 #[test]
9154 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9155 let dir = tempfile::tempdir().unwrap();
9161 std::fs::write(
9162 dir.path().join("daemon.json"),
9163 serde_json::json!({
9164 "schema": 1,
9165 "pid": 4242,
9166 "updated_at": Timestamp::now().to_string(),
9167 "idle": false,
9168 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9169 "completed": 3,
9170 "polls": 9,
9171 })
9172 .to_string(),
9173 )
9174 .unwrap();
9175
9176 let reading = read_status(dir.path()).expect("an older shape must still parse");
9177 assert!(reading.running(Timestamp::now()));
9178 assert_eq!(
9179 reading.current,
9180 vec![Current {
9181 task: "20260902-140501-aaaa".to_owned(),
9182 run: "20260902-140502-bbbb".to_owned(),
9183 }]
9184 );
9185 }
9186
9187 #[test]
9188 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9189 let dir = tempfile::tempdir().unwrap();
9190 std::fs::write(
9191 dir.path().join("daemon.json"),
9192 serde_json::json!({
9193 "schema": 1,
9194 "updated_at": Timestamp::now().to_string(),
9195 "idle": true,
9196 "current": null,
9197 })
9198 .to_string(),
9199 )
9200 .unwrap();
9201 let with_null = read_status(dir.path()).expect("null must still parse");
9202 assert!(with_null.current.is_empty());
9203
9204 std::fs::write(
9205 dir.path().join("daemon.json"),
9206 serde_json::json!({
9207 "schema": 1,
9208 "updated_at": Timestamp::now().to_string(),
9209 "idle": true,
9210 })
9211 .to_string(),
9212 )
9213 .unwrap();
9214 let absent = read_status(dir.path()).expect("a missing field must still parse");
9215 assert!(absent.current.is_empty());
9216 }
9217
9218 #[test]
9219 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9220 let fallback = Path::new("/default");
9221 let mut blank = task();
9222 blank.repo = PathBuf::new();
9223 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9224 let mut dot = task();
9225 dot.repo = PathBuf::from(".");
9226 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9227 assert_eq!(
9228 repo_for(&task(), fallback),
9229 PathBuf::from("/repo"),
9230 "a task that names a repository keeps it"
9231 );
9232 }
9233
9234 #[test]
9235 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9236 let dir = tempfile::tempdir().unwrap();
9237 let path = dir.path().join("explicit.toml");
9238 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9239 let opts = Opts {
9240 config: Some(path),
9241 merge: Some("none".into()),
9242 ..Default::default()
9243 };
9244 let mut t = task();
9245 t.set_implementers(
9246 2,
9247 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9248 .unwrap(),
9249 )
9250 .unwrap();
9251 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9252 assert_eq!(cfg.graph.implementers, 2);
9253 assert_eq!(cfg.graph.judges, 1);
9254 assert_eq!(cfg.merge.mode, MergeMode::None);
9255 t.set_implementers(1, &cfg).unwrap();
9256 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9257 apply_task_implementers(&mut cfg, &t);
9258 assert_eq!(cfg.graph.implementers, 1);
9259 assert_eq!(cfg.graph.judges, 1);
9260 assert_eq!(cfg.merge.mode, MergeMode::None);
9261 }
9262
9263 #[test]
9264 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9265 let dir = tempfile::tempdir().unwrap();
9266 let repo = dir.path().join("repo");
9267 std::fs::create_dir(&repo).unwrap();
9268 std::fs::write(
9269 repo.join("magi.toml"),
9270 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9271 )
9272 .unwrap();
9273 let daemon_config = dir.path().join("daemon.toml");
9274 std::fs::write(
9275 &daemon_config,
9276 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9277 )
9278 .unwrap();
9279 let opts = Opts {
9280 config: Some(daemon_config),
9281 merge: Some("pr".into()),
9282 ..Default::default()
9283 };
9284 let q = Queue::at(dir.path().join("queue"));
9285 let validation: Config =
9286 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9287 .unwrap();
9288 for record in ["{}", r#"{"candidates":2}"#] {
9290 let overrides = serde_json::from_str(record).unwrap();
9291 let filing = crate::direct::Filing {
9292 instruction: "work".into(),
9293 title: "direct".into(),
9294 repo: repo.clone(),
9295 source: crate::queue::Source::Human,
9296 solo: false,
9297 overrides,
9298 review_of: None,
9299 sources: Default::default(),
9300 };
9301 let filed =
9302 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9303 let crate::direct::Filed::Standalone {
9304 mut task,
9305 claim: _claim,
9306 } = filed
9307 else {
9308 panic!("local filing")
9309 };
9310 let before = prepare_for(&repo, &opts, &task).unwrap();
9311 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9312 assert_eq!(
9313 before.merge.mode,
9314 MergeMode::None,
9315 "daemon --merge must not leak"
9316 );
9317 for n in [2, 1, 2] {
9318 task.set_implementers(n, &validation).unwrap();
9319 q.put(&mut task).unwrap();
9320 task = q.get(&task.id).unwrap();
9321 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9322 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9323 apply_task_implementers(&mut cfg, &task);
9324 assert_eq!(cfg.graph.implementers, usize::from(n));
9325 assert_eq!(cfg.graph.judges, 2);
9326 assert_eq!(cfg.merge.mode, MergeMode::None);
9327 }
9328 }
9329 }
9330
9331 #[test]
9332 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9333 let cfg: Config =
9334 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9335 .unwrap();
9336 let mut t = task();
9337 t.review_of = Some("review-target".into());
9338 t.hold_manual(Some("retry".into()));
9339 let mut previous = RunState::new(
9340 PathBuf::from("/repo"),
9341 "main".into(),
9342 "abc1234".into(),
9343 "review".into(),
9344 cfg.clone(),
9345 );
9346 previous.config.graph.implementers = 2;
9347 for count in [1, 3] {
9348 t.set_implementers(count, &cfg).unwrap();
9349 t.release();
9350 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9351 for proposed in [
9352 Starter::Resume(previous.id.clone()),
9353 Starter::Review("recovery-target".into()),
9354 ] {
9355 assert_eq!(
9356 starter_with_implementers(&t, Some(&previous), proposed),
9357 Starter::Review("review-target".into())
9358 );
9359 }
9360 let mut configured = cfg.clone();
9361 apply_task_implementers(&mut configured, &t);
9362 assert_eq!(configured.graph.implementers, usize::from(count));
9363 assert_eq!(previous.config.graph.implementers, 2);
9364 }
9365 }
9366
9367 #[test]
9368 fn task_implementers_dispatch_and_resume_agree() {
9369 let mut t = task();
9370 let cfg: crate::config::Config =
9371 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9372 .unwrap();
9373 let mut run = RunState::new(
9374 PathBuf::from("/repo"),
9375 "main".into(),
9376 "abc1234".into(),
9377 "work".into(),
9378 cfg.clone(),
9379 );
9380 run.config.graph.implementers = 2;
9381 for n in [2, 1, 3, 2] {
9382 t.set_implementers(n, &cfg).unwrap();
9383 let mut dispatched = cfg.clone();
9384 apply_task_implementers(&mut dispatched, &t);
9385 assert_eq!(dispatched.graph.implementers, usize::from(n));
9386 assert_eq!(implementers_match(&t, &run), n == 2);
9387 for proposed in [
9388 Starter::Resume(run.id.clone()),
9389 Starter::Review("old-branch".into()),
9390 ] {
9391 let expected = if n == 2 {
9392 proposed.clone()
9393 } else {
9394 Starter::Start
9395 };
9396 assert_eq!(
9397 starter_with_implementers(&t, Some(&run), proposed),
9398 expected
9399 );
9400 }
9401 assert_eq!(
9402 run.config.graph.implementers, 2,
9403 "history must not be rewritten"
9404 );
9405 }
9406 t.solo = false;
9407 t.overrides = None;
9408 assert!(implementers_match(&t, &run));
9409 }
9410
9411 #[test]
9412 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9413 let mut solo_cfg = Config::default();
9419 solo_cfg.graph.implementers = 3;
9420 let mut solo_task = task();
9421 solo_task.solo = true;
9422 apply_task_implementers(&mut solo_cfg, &solo_task);
9423 assert_eq!(solo_cfg.graph.implementers, 1);
9424
9425 let mut plain_cfg = Config::default();
9426 plain_cfg.graph.implementers = 3;
9427 let plain_task = task();
9428 assert!(!plain_task.solo);
9429 apply_task_implementers(&mut plain_cfg, &plain_task);
9430 assert_eq!(
9431 plain_cfg.graph.implementers, 3,
9432 "a task that did not ask to run alone keeps the config's candidates"
9433 );
9434 }
9435
9436 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9437 QuotaLoss {
9438 seat: seat.into(),
9439 node: "judge".into(),
9440 at: at.parse().unwrap(),
9441 reset: reset.map(str::to_string),
9442 }
9443 }
9444
9445 #[test]
9446 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
9447 let old: Vec<QuotaLoss> = (1..=4)
9448 .map(|i| {
9449 loss(
9450 &format!("judge-{i}"),
9451 "2026-09-23T05:23:00Z",
9452 Some("2:40pm (Asia/Tokyo)"),
9453 )
9454 })
9455 .collect();
9456 let fresh = losses_this_attempt(&old, &old);
9457 assert!(fresh.is_empty());
9458 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
9459 }
9461
9462 #[test]
9463 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
9464 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
9465 let now = Timestamp::now();
9466 let mut after = old.clone();
9467 after.push(loss("judge-2", &now.to_string(), None));
9468 let fresh = losses_this_attempt(&old, &after);
9469 assert_eq!(fresh, vec![after[1].clone()]);
9470 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
9471 assert_eq!(
9472 until,
9473 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
9474 );
9475 }
9476
9477 #[test]
9478 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
9479 let before = vec![
9482 loss("judge-1", "2026-09-23T05:23:00Z", None),
9483 loss("judge-2", "2026-09-23T05:24:00Z", None),
9484 ];
9485 let after = vec![
9486 loss("judge-2", "2026-09-23T05:24:00Z", None),
9487 loss("judge-1", "2026-09-24T01:00:00Z", None),
9488 ];
9489 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
9490 }
9491
9492 #[test]
9493 fn merge_overrides_are_parsed_or_refused() {
9494 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
9495 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
9496 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
9497 assert!(merge_mode("squash").is_err());
9498 }
9499
9500 #[test]
9501 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
9502 let now = Timestamp::now();
9503 let fallback = Duration::from_secs(300);
9504 let cap = Duration::from_secs(1800);
9505
9506 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
9508
9509 let soon = now + jiff::SignedDuration::from_secs(600);
9511 assert_eq!(
9512 quota_wait(Some(soon), now, fallback, cap),
9513 Duration::from_secs(600)
9514 );
9515
9516 let past = now - jiff::SignedDuration::from_secs(60);
9519 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
9520
9521 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
9524 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
9525 }
9526
9527 #[test]
9528 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9529 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9530
9531 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9532 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9533
9534 let already_past =
9538 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9539 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9540
9541 assert!(
9542 parse_reset_hint("session limit reached", now, now).is_none(),
9543 "free text with no recognised shape is not guessed at"
9544 );
9545 assert!(
9546 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9547 "an unresolvable zone name is not guessed at either"
9548 );
9549 }
9550
9551 #[test]
9552 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9553 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9554
9555 let at = parse_reset_hint(
9556 "You've hit your usage limit. Visit \
9557 https://chatgpt.com/codex/settings/usage to purchase more \
9558 credits or try again at Sep 19th, 2026 5:10 PM.",
9559 now,
9560 now,
9561 )
9562 .expect("the codex reset wording is a recognised shape");
9563 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9564
9565 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9570 .expect("an explicit year needs no rollover");
9571 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9572
9573 assert!(
9574 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9575 "a two-digit year is not the documented shape and is not guessed at"
9576 );
9577 assert!(
9578 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9579 "a four-letter month name is not the documented three-letter abbreviation"
9580 );
9581 assert!(
9582 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9583 "an explicit zone on the dated shape is a format nobody has \
9584 documented, and is refused rather than guessed at as UTC"
9585 );
9586 }
9587
9588 #[test]
9589 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9590 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9591 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9592
9593 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9594 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9595
9596 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9597 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9598
9599 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9600 assert!(
9601 parse_reset_hint(bad, now, recorded).is_none(),
9602 "{bad:?} must not be guessed at"
9603 );
9604 }
9605 }
9606
9607 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9611 let config = dir.join("magi.toml");
9612 std::fs::write(
9613 &config,
9614 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9615 )
9616 .unwrap();
9617 let opts = Opts {
9618 poll: Duration::from_secs(30),
9619 config: Some(config),
9620 repo: dir.join("repo"),
9624 ..Opts::default()
9625 };
9626 let home = dir.join("home");
9635 let stamps = home.join("service-ticks");
9638 std::fs::create_dir_all(&stamps).unwrap();
9639 for service in ["waiter", "deputies", "release-watch"] {
9640 std::fs::write(
9641 stamps.join(service),
9642 Timestamp::now().as_second().to_string(),
9643 )
9644 .unwrap();
9645 }
9646 for operation in [
9647 "conductor",
9648 "maintenance",
9649 "housekeeping",
9650 "disk-pressure",
9651 "cache",
9652 "fetch",
9653 ] {
9654 std::fs::write(
9655 stamps.join(format!("daemon-{operation}")),
9656 serde_json::to_vec(&Timestamp::now()).unwrap(),
9657 )
9658 .unwrap();
9659 }
9660 let worktrees = dir.join("wt");
9661 (
9662 opts,
9663 Queue::at(dir.join("queue")),
9664 home.join("daemon.json"),
9665 home,
9666 worktrees,
9667 )
9668 }
9669
9670 #[test]
9671 fn a_stop_is_idempotent_and_once_set_stays_set() {
9672 let stop = Stop::new();
9673 assert!(!stop.stopped());
9674
9675 stop.stop();
9676 assert!(stop.stopped());
9677 stop.stop();
9678 assert!(stop.stopped(), "a second stop is not a toggle");
9679
9680 let shared = stop.clone();
9681 assert!(
9682 shared.stopped(),
9683 "a clone is the same stop; that is how the loop and its caller share one"
9684 );
9685 }
9686
9687 #[test]
9688 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9689 let stop = Stop::new();
9690 stop.busy.store(1, Ordering::SeqCst);
9691 assert!(
9692 !stop.finishing(),
9693 "a busy loop nobody has asked to stop is just running"
9694 );
9695
9696 stop.stop();
9697 assert!(
9698 stop.finishing(),
9699 "a stop asked for mid-run has not landed until the run is settled"
9700 );
9701
9702 stop.busy.store(0, Ordering::SeqCst);
9703 assert!(
9704 !stop.finishing(),
9705 "once the run is settled the stop has landed and there is nothing to finish"
9706 );
9707 }
9708
9709 #[test]
9710 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9711 let stop = Stop::new();
9712 stop.busy.store(2, Ordering::SeqCst);
9713 stop.stop();
9714 assert!(stop.finishing(), "two runs still in flight");
9715
9716 stop.busy.store(1, Ordering::SeqCst);
9717 assert!(
9718 stop.finishing(),
9719 "one run finished, but a sibling is still working"
9720 );
9721
9722 stop.busy.store(0, Ordering::SeqCst);
9723 assert!(
9724 !stop.finishing(),
9725 "the last run out is what actually lands the stop"
9726 );
9727 }
9728
9729 #[tokio::test]
9730 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9731 let dir = tempfile::tempdir().unwrap();
9732 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9733 let stop = Stop::new();
9734 stop.stop();
9735
9736 let began = std::time::Instant::now();
9737 tokio::time::timeout(
9738 Duration::from_secs(2),
9739 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9740 )
9741 .await
9742 .expect("a stopped loop must return, not sit out its poll interval")
9743 .expect("the loop's own setup and teardown must not fail");
9744 assert!(
9745 began.elapsed() < opts.poll,
9746 "returned only after {:?}, which is a poll interval, not a stop",
9747 began.elapsed()
9748 );
9749 }
9750
9751 #[tokio::test]
9752 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9753 let dir = tempfile::tempdir().unwrap();
9754 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9755 let stop = Stop::new();
9756
9757 let asker = {
9760 let stop = stop.clone();
9761 tokio::spawn(async move {
9762 tokio::time::sleep(Duration::from_millis(20)).await;
9763 stop.stop();
9764 })
9765 };
9766
9767 let began = std::time::Instant::now();
9768 tokio::time::timeout(
9769 Duration::from_secs(2),
9770 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9771 )
9772 .await
9773 .expect("a stop asked for while idle must wake the wait")
9774 .expect("the loop's own setup and teardown must not fail");
9775 asker.await.unwrap();
9776 assert!(
9777 began.elapsed() < opts.poll,
9778 "returned only after {:?}, so the stop waited on the sleep",
9779 began.elapsed()
9780 );
9781 }
9782
9783 #[tokio::test]
9784 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9785 let dir = tempfile::tempdir().unwrap();
9786 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9787 let stop = Stop::new();
9788 stop.stop();
9789
9790 tokio::time::timeout(
9791 Duration::from_secs(2),
9792 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9793 )
9794 .await
9795 .expect("a stopped loop must return")
9796 .expect("the loop's own setup and teardown must not fail");
9797
9798 assert!(
9799 home.is_dir(),
9800 "the loop did publish a status file, so its removal is the teardown and not an absence"
9801 );
9802 assert!(
9803 !status_file.exists(),
9804 "a stopped loop clears its status file"
9805 );
9806 assert!(
9807 read_status(&home).is_none(),
9808 "a reader must see no daemon at all, not a heartbeat that merely stopped"
9809 );
9810 }
9811
9812 #[tokio::test]
9813 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
9814 let dir = tempfile::tempdir().unwrap();
9815 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
9816
9817 let mut settled = RunState::new(
9818 dir.path().join("repo"),
9819 "main".to_owned(),
9820 "abc1234".to_owned(),
9821 "fixture".to_owned(),
9822 Config::default(),
9823 );
9824 settled.status = RunStatus::Ready;
9825 let run_dir = home.join("runs").join(&settled.id);
9826 std::fs::create_dir_all(&run_dir).unwrap();
9827 std::fs::write(
9828 run_dir.join("run.json"),
9829 serde_json::to_string_pretty(&settled).unwrap(),
9830 )
9831 .unwrap();
9832 let questions = Questions::at(home.join("questions"));
9833 let mut question = ask::Question::new(
9834 settled.id.clone(),
9835 "review".to_owned(),
9836 "reviewer-1".to_owned(),
9837 "Continue?".to_owned(),
9838 String::new(),
9839 Vec::new(),
9840 );
9841 questions.put(&mut question).unwrap();
9842
9843 run_service_worker(
9844 &home,
9845 serde_json::to_value(ServiceWorker {
9846 opts,
9847 queue: queue.root().to_path_buf(),
9848 worktrees_root: worktrees,
9849 operation: "housekeeping".to_owned(),
9850 })
9851 .unwrap(),
9852 )
9853 .await
9854 .unwrap();
9855
9856 assert_eq!(
9857 questions.get(&question.id).unwrap().status,
9858 ask::QuestionStatus::Abandoned,
9859 "the independent housekeeping worker settles ended-run questions"
9860 );
9861 }
9862
9863 #[test]
9864 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
9865 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9866
9867 assert!(
9868 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
9869 "never checked before: due at once"
9870 );
9871
9872 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
9873 assert!(
9874 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
9875 "well inside the interval: not due yet"
9876 );
9877
9878 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
9879 assert!(
9880 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
9881 "exactly at the edge: not yet due, same convention as `clean::due`"
9882 );
9883
9884 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9885 assert!(
9886 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
9887 "past the interval: due again"
9888 );
9889 }
9890
9891 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
9896 let config = dir.join("magi.toml");
9897 std::fs::write(
9903 &config,
9904 format!(
9905 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
9906 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
9907 cache_dir.display()
9908 ),
9909 )
9910 .unwrap();
9911 Opts {
9912 config: Some(config),
9913 repo: dir.join("repo"),
9914 ..Opts::default()
9915 }
9916 }
9917
9918 #[tokio::test]
9919 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
9920 let dir = tempfile::tempdir().unwrap();
9921 let home = dir.path().join("home");
9922 let cache_dir = dir.path().join("cache");
9923 std::fs::create_dir_all(&cache_dir).unwrap();
9924 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9925 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9926
9927 let running = Stop::new();
9930 let mut last_checked = None;
9931 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9932 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
9933 .await;
9934 assert_eq!(
9935 crate::disk::dir_size(&cache_dir),
9936 0,
9937 "over the cap on the first check ever: pruned at once, no idle queue required"
9938 );
9939 assert_eq!(last_checked, Some(t0));
9940
9941 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
9943 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
9944 maybe_prune_cache_between_runs(
9945 &opts.repo,
9946 &opts,
9947 &home,
9948 &running,
9949 &mut last_checked,
9950 too_soon,
9951 )
9952 .await;
9953 assert_eq!(
9954 crate::disk::dir_size(&cache_dir),
9955 10,
9956 "too soon since the last check: left alone rather than rescanned every call"
9957 );
9958 assert_eq!(
9959 last_checked,
9960 Some(t0),
9961 "an idle check does not reset the clock"
9962 );
9963
9964 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9966 maybe_prune_cache_between_runs(
9967 &opts.repo,
9968 &opts,
9969 &home,
9970 &running,
9971 &mut last_checked,
9972 due_again,
9973 )
9974 .await;
9975 assert_eq!(
9976 crate::disk::dir_size(&cache_dir),
9977 0,
9978 "due again: pruned back under the cap"
9979 );
9980 }
9981
9982 #[tokio::test]
9990 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
9991 let dir = tempfile::tempdir().unwrap();
9992 let home = dir.path().join("home");
9993 let cache_dir = dir.path().join("cache");
9994 std::fs::create_dir_all(&cache_dir).unwrap();
9995 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9996 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9997
9998 let stop = Stop::new();
9999 stop.stop();
10000 assert!(
10001 !stop.finishing(),
10002 "no run is in flight at a between-runs boundary, so nothing else \
10003 would tell the operator this stop had not taken effect yet"
10004 );
10005
10006 let mut last_checked = None;
10007 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10008 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10009 .await;
10010 assert_eq!(
10011 crate::disk::dir_size(&cache_dir),
10012 10,
10013 "over its cap, and due for the first check ever, but a stop outranks \
10014 it: the cap is a standing policy the next start measures again"
10015 );
10016 assert_eq!(
10017 last_checked, None,
10018 "a check that never happened must not claim the interval"
10019 );
10020 }
10021
10022 #[tokio::test]
10025 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10026 let dir = tempfile::tempdir().unwrap();
10027 let cache_dir = dir.path().join("cache");
10028 std::fs::create_dir_all(&cache_dir).unwrap();
10029 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10030
10031 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10032 opts.poll = Duration::from_millis(20);
10033 opts.max_attempts = 1_000;
10034
10035 let queue = Queue::at(dir.path().join("queue"));
10036 for i in 0..128 {
10037 let mut task = Task::new(
10038 format!("queued work {i}"),
10039 format!("queued work {i}"),
10040 opts.repo.clone(),
10041 Source::Human,
10042 );
10043 queue.put(&mut task).unwrap();
10044 }
10045
10046 let home = dir.path().join("home");
10047 let worktrees = dir.path().join("wt");
10048 run_service_worker(
10049 &home,
10050 serde_json::to_value(ServiceWorker {
10051 opts,
10052 queue: queue.root().to_path_buf(),
10053 worktrees_root: worktrees,
10054 operation: "cache".to_owned(),
10055 })
10056 .unwrap(),
10057 )
10058 .await
10059 .unwrap();
10060 let after = queue.list();
10061 assert_eq!(after.len(), 128);
10062 assert!(
10063 after
10064 .iter()
10065 .all(|task| task.status.runnable() && task.attempts == 0)
10066 );
10067
10068 assert_eq!(
10069 crate::disk::dir_size(&cache_dir),
10070 0,
10071 "an oversized cache must not be left to grow unboundedly just because the \
10072 queue kept the loop busy the whole time"
10073 );
10074 }
10075
10076 #[test]
10077 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10078 let dir = tempfile::tempdir().unwrap();
10079 let queue = Queue::at(dir.path().join("queue"));
10080 let questions = Questions::at(dir.path().join("questions"));
10081 let mut task = task();
10082 queue.put(&mut task).unwrap();
10083
10084 let mut task_question = ask::Question::new(
10085 task.id.clone(),
10086 crate::conduct::NODE.to_owned(),
10087 "conduct".to_owned(),
10088 "Which backend?".to_owned(),
10089 String::new(),
10090 Vec::new(),
10091 );
10092 questions.put(&mut task_question).unwrap();
10093 task.block(vec![task_question.id.clone()], None);
10094 queue.put(&mut task).unwrap();
10095
10096 let mut run_question = ask::Question::new(
10097 "20260101-000000-run1".to_owned(),
10098 "review".to_owned(),
10099 "reviewer-1".to_owned(),
10100 "Run question".to_owned(),
10101 String::new(),
10102 Vec::new(),
10103 );
10104 questions.put(&mut run_question).unwrap();
10105
10106 let mut coincidental = ask::Question::new(
10111 task.id.clone(),
10112 "review".to_owned(),
10113 "reviewer-1".to_owned(),
10114 "Unrelated review question".to_owned(),
10115 String::new(),
10116 Vec::new(),
10117 );
10118 questions.put(&mut coincidental).unwrap();
10119
10120 reconcile_task_questions(&queue, &questions);
10121 assert!(questions.get(&task_question.id).unwrap().status.open());
10122 assert!(questions.get(&run_question.id).unwrap().status.open());
10123 assert!(questions.get(&coincidental.id).unwrap().status.open());
10124
10125 task.release();
10126 queue.put(&mut task).unwrap();
10127 reconcile_task_questions(&queue, &questions);
10128 assert_eq!(
10129 questions.get(&task_question.id).unwrap().status,
10130 ask::QuestionStatus::Abandoned
10131 );
10132 assert!(
10133 questions.get(&run_question.id).unwrap().status.open(),
10134 "run questions remain the run janitor's responsibility"
10135 );
10136 assert!(
10137 questions.get(&coincidental.id).unwrap().status.open(),
10138 "a non-conductor question must not be abandoned just because its \
10139 run id coincides with a task id"
10140 );
10141 }
10142
10143 #[test]
10144 fn a_freshly_started_running_task_is_never_stalled() {
10145 let dir = tempfile::tempdir().unwrap();
10146 let mut t = task();
10147 t.start("run-1".to_owned());
10148 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10151 }
10152
10153 #[test]
10154 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10155 let dir = tempfile::tempdir().unwrap();
10156 let mut t = task();
10157 t.start("run-1".to_owned());
10158 t.updated_at = Timestamp::now()
10159 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10160 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10161 assert_eq!(
10162 stalled_tasks(
10163 &Queue::at(dir.path().join("q")),
10164 dir.path(),
10165 Timestamp::now()
10166 )
10167 .len(),
10168 0,
10169 "the task was never written to this queue"
10170 );
10171 }
10172
10173 #[test]
10174 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10175 let dir = tempfile::tempdir().unwrap();
10176 let mut t = task();
10177 t.id = "20260903-080340-0167".to_owned();
10178 t.start("20260903-080619-01c2".to_owned());
10179 t.updated_at = Timestamp::now()
10180 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10181
10182 let mut status = Status::new();
10183 status.current = vec![Current {
10184 task: t.id.clone(),
10185 run: "20260903-080619-01c2".to_owned(),
10186 }];
10187 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10188
10189 assert!(
10190 !is_stalled(&t, dir.path(), Timestamp::now()),
10191 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10192 );
10193 }
10194
10195 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10199 let path = queue.path_of(id);
10200 let body = std::fs::read_to_string(&path).unwrap();
10201 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10202 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10203 v["updated_at"] = serde_json::Value::String(old.to_string());
10204 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10205 }
10206
10207 #[test]
10208 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10209 let dir = tempfile::tempdir().unwrap();
10222 let queue = Queue::at(dir.path().join("queue"));
10223 let home = dir.path().join("home");
10224
10225 let mut t = task();
10226 t.id = "20260101-000001-lock".to_owned();
10227 t.start("run-1".to_owned());
10228 queue.put(&mut t).unwrap();
10229 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10230 std::fs::write(
10231 dir.path().join("queue").join(format!("{}.lock", t.id)),
10232 "not a pid",
10233 )
10234 .unwrap();
10235
10236 let now = Timestamp::now();
10237 assert!(
10238 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10239 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10240 and reclaim must leave the task alone"
10241 );
10242 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10243
10244 let stalled = stalled_tasks(&queue, &home, now);
10245 assert_eq!(
10246 stalled.len(),
10247 1,
10248 "reclaim's inability to claim it yet must not hide it from the conductor"
10249 );
10250 assert_eq!(stalled[0].id, t.id);
10251 }
10252
10253 #[test]
10254 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10255 let dir = tempfile::tempdir().unwrap();
10256 crate::run::set_home(dir.path().join("run-home"));
10257 let queue = Queue::at(dir.path().join("queue"));
10258 let home = dir.path().join("home");
10259 let questions = Questions::at(dir.path().join("questions"));
10260
10261 let mut t = task();
10262 t.id = "20260101-000003-dead".to_owned();
10263 t.start("missing-run".to_owned());
10264 queue.put(&mut t).unwrap();
10265 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10266
10267 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10270 assert_eq!(
10271 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10272 [&t.id]
10273 );
10274 assert_eq!(
10275 reclaim_orphaned_running(&queue, 2, &Default::default()),
10276 [t.id.clone()]
10277 );
10278 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10279
10280 crate::conduct::apply(
10283 &queue,
10284 &questions,
10285 &crate::conduct::Verdict {
10286 decisions: vec![crate::conduct::Decision {
10287 id: t.id.clone(),
10288 recovery: Some(crate::conduct::Recovery::Requeue),
10289 ..crate::conduct::Decision::default()
10290 }],
10291 },
10292 )
10293 .unwrap();
10294 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10295 }
10296
10297 #[test]
10298 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10299 let dir = tempfile::tempdir().unwrap();
10300 let queue = Queue::at(dir.path().join("queue"));
10301 let home = dir.path().join("home");
10302
10303 let mut fresh = task();
10304 fresh.id = "20260101-000001-aaaa".to_owned();
10305 fresh.start("run-1".to_owned());
10306 queue.put(&mut fresh).unwrap();
10307
10308 let mut old = task();
10309 old.id = "20260101-000002-bbbb".to_owned();
10310 old.start("run-2".to_owned());
10311 queue.put(&mut old).unwrap();
10312 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10313
10314 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10315 assert_eq!(stalled.len(), 1);
10316 assert_eq!(stalled[0].id, old.id);
10317 }
10318
10319 #[test]
10320 fn queued_and_finished_task_views_partition_by_status() {
10321 let dir = tempfile::tempdir().unwrap();
10322 let queue = Queue::at(dir.path().join("queue"));
10323
10324 let mut queued = task();
10325 queued.id = "20260101-000001-aaaa".to_owned();
10326 queue.put(&mut queued).unwrap();
10327
10328 let mut failed = task();
10329 failed.id = "20260101-000002-bbbb".to_owned();
10330 failed.start("run-1".to_owned());
10331 failed.fail("gate red", 5);
10332 queue.put(&mut failed).unwrap();
10333
10334 let mut held = task();
10335 held.id = "20260101-000003-cccc".to_owned();
10336 held.hold_machine(None);
10337 queue.put(&mut held).unwrap();
10338
10339 let mut running = task();
10340 running.id = "20260101-000004-dddd".to_owned();
10341 running.start("run-2".to_owned());
10342 queue.put(&mut running).unwrap();
10343
10344 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
10345 assert_eq!(queued_ids, [queued.id.clone()]);
10346
10347 let mut finished_ids: Vec<String> =
10348 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
10349 finished_ids.sort_unstable();
10350 let mut want = vec![failed.id.clone(), held.id.clone()];
10351 want.sort_unstable();
10352 assert_eq!(finished_ids, want);
10353 }
10354
10355 #[test]
10356 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
10357 for short in [false, true] {
10358 let dir = tempfile::tempdir().unwrap();
10359 let queue = Queue::at(dir.path().join("queue"));
10360 let questions = ask::Questions::at(dir.path().join("questions"));
10361 let mut running = task();
10362 running.id = "20261013-192213-feb2".into();
10363 running.start("existing-run".into());
10364 queue.put(&mut running).unwrap();
10365 let mut middle = task();
10366 middle.id = "20261013-193611-ca5b".into();
10367 middle.block(
10368 vec![if short {
10369 "feb2".into()
10370 } else {
10371 running.id.clone()
10372 }],
10373 None,
10374 );
10375 queue.put(&mut middle).unwrap();
10376 let mut waiting = task();
10377 waiting.id = "20261013-202849-ee37".into();
10378 waiting.block(
10379 vec![if short {
10380 "ca5b".into()
10381 } else {
10382 middle.id.clone()
10383 }],
10384 Some("wait for implementation".into()),
10385 );
10386 queue.put(&mut waiting).unwrap();
10387 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
10388 let inv = crate::blockers::Inventory::load(&queue, &questions);
10389 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
10390 assert!(inv.stuck().is_empty());
10391 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10392 assert!(report.quarantined.is_empty());
10393 assert!(report.asked.is_empty());
10394 resolve_blockers(&queue, &questions);
10395 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10396 assert!(questions.list().is_empty());
10397 running.succeed();
10400 queue.put(&mut running).unwrap();
10401 resolve_blockers(&queue, &questions);
10402 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
10403 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10404 middle = queue.get(&middle.id).unwrap();
10405 middle.succeed();
10406 queue.put(&mut middle).unwrap();
10407 resolve_blockers(&queue, &questions);
10408 let after = queue.get(&waiting.id).unwrap();
10409 assert_eq!(after.status, TaskStatus::Queued);
10410 assert!(after.blocked_by.is_empty());
10411 assert_eq!(after.instruction, waiting.instruction);
10412 }
10413 }
10414
10415 #[test]
10416 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
10417 for future_schema in [false, true] {
10418 let name = "20261013-193611-ca5b";
10419 let mut future = task();
10420 future.id = name.into();
10421 future.schema = crate::queue::SCHEMA + 1;
10422 let contents = if future_schema {
10423 serde_json::to_string(&future).unwrap()
10424 } else {
10425 "corrupt".into()
10426 };
10427 let dir = tempfile::tempdir().unwrap();
10428 let queue = Queue::at(dir.path().join("queue"));
10429 let questions = ask::Questions::at(dir.path().join("questions"));
10430 let mut waiting = task();
10431 waiting.id = "20261013-202849-ee37".into();
10432 waiting.block(vec!["ca5b".into()], None);
10433 queue.put(&mut waiting).unwrap();
10434 std::fs::write(queue.path_of(name), contents).unwrap();
10435 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
10436 assert!(
10437 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
10438 );
10439 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10440 assert!(report.asked.is_empty());
10441 resolve_blockers(&queue, &questions);
10442 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10443 assert!(questions.list().is_empty());
10444 }
10445 let dir = tempfile::tempdir().unwrap();
10446 let queue = Queue::at(dir.path().join("queue"));
10447 let questions = ask::Questions::at(dir.path().join("questions"));
10448 let mut waiting = task();
10449 waiting.id = "20261013-202849-ee37".into();
10450 waiting.block(vec!["ca5b".into()], None);
10451 queue.put(&mut waiting).unwrap();
10452 let mut done = task();
10453 done.id = "20261013-193611-ca5b".into();
10454 done.succeed();
10455 queue.put(&mut done).unwrap();
10456 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
10459 assert!(queue.get("ca5b").is_err());
10460 assert_eq!(
10461 crate::queue::dependency(&queue, &questions, "ca5b"),
10462 crate::queue::Dependency::Ambiguous
10463 );
10464 resolve_blockers(&queue, &questions);
10465 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10466 assert!(report.asked.is_empty());
10467 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10468 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
10469 }
10470
10471 #[test]
10472 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
10473 for held in [false, true] {
10474 for question in [false, true] {
10475 for reference in ["ca5b", "20261013-193611-ca5b"] {
10476 let dir = tempfile::tempdir().unwrap();
10477 let queue = Queue::at(dir.path().join("queue"));
10478 let questions = ask::Questions::at(dir.path().join("questions"));
10479 let mut prerequisite = task();
10480 prerequisite.id = "20261013-193611-ca5b".into();
10481 prerequisite.start("existing-run".into());
10482 queue.put(&mut prerequisite).unwrap();
10483 let mut waiting = task();
10484 waiting.id = "20261013-202849-ee37".into();
10485 if held {
10486 waiting.hold_machine(Some("waiting for separate consent".into()));
10487 }
10488 queue.put(&mut waiting).unwrap();
10489 let reason = "Deleting saved data requires separate owner consent";
10490 crate::conduct::apply(
10491 &queue,
10492 &questions,
10493 &crate::conduct::Verdict {
10494 decisions: vec![crate::conduct::Decision {
10495 id: waiting.id.clone(),
10496 blocked_by: vec![reference.into()],
10497 question: question.then(|| "May the saved data be deleted?".into()),
10498 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
10499 reason: Some(reason.into()),
10500 ..crate::conduct::Decision::default()
10501 }],
10502 },
10503 )
10504 .unwrap();
10505 let before = queue.get(&waiting.id).unwrap();
10506 if question {
10507 assert_eq!(before.status, TaskStatus::Blocked);
10508 let q = questions.get(&before.blocked_by[0]).unwrap();
10509 assert_eq!(q.summary, "May the saved data be deleted?");
10510 assert!(q.status.open());
10511 } else {
10512 assert_eq!(before.status, TaskStatus::Held);
10513 let recorded_reason = before.hold_reason.as_deref().unwrap();
10514 assert!(recorded_reason.starts_with(reason));
10515 if held {
10516 assert!(recorded_reason.contains("waiting for separate consent"));
10517 }
10518 }
10519 prerequisite.succeed();
10520 queue.put(&mut prerequisite).unwrap();
10521 resolve_blockers(&queue, &questions);
10522 let after = queue.get(&waiting.id).unwrap();
10523 assert_eq!(after.status, before.status);
10524 assert_eq!(after.blocked_by, before.blocked_by);
10525 assert_eq!(after.hold_reason, before.hold_reason);
10526 assert_eq!(after.instruction, waiting.instruction);
10527 if question {
10528 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10529 }
10530 assert!(
10531 queue.next_runnable().is_none(),
10532 "dependency completion is not consent"
10533 );
10534 }
10535 }
10536 }
10537 }
10538
10539 #[test]
10540 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10541 let dir = tempfile::tempdir().unwrap();
10542 let queue = Queue::at(dir.path().join("queue"));
10543 let questions = ask::Questions::at(dir.path().join("questions"));
10544
10545 let mut dep = task();
10546 dep.id = "20260101-000001-dep0".to_owned();
10547 dep.succeed();
10548 queue.put(&mut dep).unwrap();
10549
10550 let mut still_going = task();
10551 still_going.id = "20260101-000002-dep1".to_owned();
10552 queue.put(&mut still_going).unwrap();
10553
10554 let mut blocked = task();
10555 blocked.id = "20260101-000003-main".to_owned();
10556 blocked.block(
10557 vec![dep.id.clone(), still_going.id.clone()],
10558 Some("waits on both".to_owned()),
10559 );
10560 queue.put(&mut blocked).unwrap();
10561
10562 resolve_blockers(&queue, &questions);
10563
10564 let after = queue.get(&blocked.id).unwrap();
10565 assert_eq!(
10566 after.status,
10567 TaskStatus::Blocked,
10568 "one dependency is still outstanding"
10569 );
10570 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10571 }
10572
10573 #[test]
10574 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10575 let dir = tempfile::tempdir().unwrap();
10576 let queue = Queue::at(dir.path().join("queue"));
10577 let questions = ask::Questions::at(dir.path().join("questions"));
10578
10579 let mut q = crate::ask::Question::new(
10580 "20260101-000001-main".to_owned(),
10581 crate::conduct::NODE.to_owned(),
10582 "conduct".to_owned(),
10583 "Which backend?".to_owned(),
10584 String::new(),
10585 Vec::new(),
10586 );
10587 questions.put(&mut q).unwrap();
10588 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10589 .unwrap();
10590 questions.put(&mut q).unwrap();
10591
10592 let mut blocked = task();
10593 blocked.id = "20260101-000001-main".to_owned();
10594 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10595 queue.put(&mut blocked).unwrap();
10596
10597 resolve_blockers(&queue, &questions);
10598
10599 let after = queue.get(&blocked.id).unwrap();
10600 assert_eq!(
10601 after.status,
10602 TaskStatus::Queued,
10603 "the only blocker resolved"
10604 );
10605 assert_eq!(after.answers.len(), 1);
10606 assert_eq!(after.answers[0].question, "Which backend?");
10607 assert_eq!(after.answers[0].answer, "SQLite");
10608
10609 let instruction = instruction_for(&after);
10611 assert!(instruction.contains("Which backend?"));
10612 assert!(instruction.contains("SQLite"));
10613 }
10614
10615 #[test]
10616 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10617 let dir = tempfile::tempdir().unwrap();
10618 let queue = Queue::at(dir.path().join("queue"));
10619 let questions = ask::Questions::at(dir.path().join("questions"));
10620
10621 let mut q = crate::ask::Question::new(
10622 "20260101-000001-main".to_owned(),
10623 crate::conduct::NODE.to_owned(),
10624 "conduct".to_owned(),
10625 "Is the setup done?".to_owned(),
10626 String::new(),
10627 Vec::new(),
10628 );
10629 q.abandon("no answer within 60s of asking");
10630 questions.put(&mut q).unwrap();
10631
10632 let mut blocked = task();
10633 blocked.id = "20260101-000001-main".to_owned();
10634 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10635 queue.put(&mut blocked).unwrap();
10636
10637 resolve_blockers(&queue, &questions);
10638
10639 let after = queue.get(&blocked.id).unwrap();
10640 assert_eq!(
10641 after.status,
10642 TaskStatus::Held,
10643 "never left blocked on nothing"
10644 );
10645 assert!(!after.operator_held(), "a machine hold, for triage");
10646 assert!(
10647 after
10648 .hold_reason
10649 .as_deref()
10650 .unwrap_or_default()
10651 .contains("went unanswered")
10652 );
10653 }
10654
10655 #[test]
10656 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10657 let dir = tempfile::tempdir().unwrap();
10663 let queue = Queue::at(dir.path().join("queue"));
10664 let questions = ask::Questions::at(dir.path().join("questions"));
10665
10666 let mut q = crate::ask::Question::new(
10667 "20260101-000001-main".to_owned(),
10668 crate::conduct::NODE.to_owned(),
10669 "conduct".to_owned(),
10670 "How should this be handled?".to_owned(),
10671 String::new(),
10672 Vec::new(),
10673 );
10674 questions.put(&mut q).unwrap();
10675 q.answer(crate::ask::Answer::Text(
10676 "leave it held, a human will look at it later".to_owned(),
10677 ))
10678 .unwrap();
10679 questions.put(&mut q).unwrap();
10680
10681 let mut held = task();
10682 held.id = "20260101-000001-main".to_owned();
10683 held.hold_machine(Some("out of attempts".to_owned()));
10684 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10685 queue.put(&mut held).unwrap();
10686
10687 resolve_blockers(&queue, &questions);
10688
10689 let after = queue.get(&held.id).unwrap();
10690 assert_eq!(after.status, TaskStatus::Held);
10691 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10692 assert_eq!(
10693 after.answers[0].answer,
10694 "leave it held, a human will look at it later"
10695 );
10696 }
10697
10698 #[test]
10699 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10700 let dir = tempfile::tempdir().unwrap();
10701 let queue = Queue::at(dir.path().join("queue"));
10702 let questions = ask::Questions::at(dir.path().join("questions"));
10703 let mut dep = task();
10704 dep.id = "20260101-000001-gone".to_owned();
10705 queue.put(&mut dep).unwrap();
10706 let mut blocked = task();
10707 blocked.id = "20260101-000003-main".to_owned();
10708 blocked.block(vec![dep.id.clone()], None);
10709 queue.put(&mut blocked).unwrap();
10710
10711 let claim = queue.claim(&blocked.id).unwrap();
10713 queue.remove(&dep.id, false, &questions).unwrap();
10714 drop(claim);
10715 resolve_blockers(&queue, &questions);
10716
10717 let after = queue.get(&blocked.id).unwrap();
10718 assert_eq!(after.status, TaskStatus::Queued);
10719 assert!(after.blocked_by.is_empty());
10720 }
10721
10722 #[test]
10723 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10724 let dir = tempfile::tempdir().unwrap();
10730 let queue = Queue::at(dir.path().join("queue"));
10731 let questions = ask::Questions::at(dir.path().join("questions"));
10732
10733 let mut still_going = task();
10734 still_going.id = "20260101-000002-dep1".to_owned();
10735 queue.put(&mut still_going).unwrap();
10736
10737 let mut blocked = task();
10738 blocked.id = "20260101-000003-main".to_owned();
10739 blocked.block(
10740 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10741 Some("waits on both".to_owned()),
10742 );
10743 queue.put(&mut blocked).unwrap();
10744
10745 resolve_blockers(&queue, &questions);
10746
10747 let after = queue.get(&blocked.id).unwrap();
10748 assert_eq!(
10749 after.status,
10750 TaskStatus::Held,
10751 "a missing dependency must not leave the task blocked forever"
10752 );
10753 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10754 assert!(after.blocked_by.is_empty());
10755 let reason = after.hold_reason.as_deref().unwrap_or_default();
10756 assert!(
10757 reason.contains("20260101-000001-gone"),
10758 "the missing id must be named so an operator can tell what happened: {reason}"
10759 );
10760 assert!(
10761 reason.contains(&still_going.id),
10762 "the still-valid dependency must not silently vanish from the record: {reason}"
10763 );
10764 }
10765
10766 #[test]
10767 fn instruction_for_is_unchanged_without_any_answers() {
10768 let t = task();
10769 assert_eq!(instruction_for(&t), t.instruction);
10770 }
10771
10772 #[test]
10773 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10774 let dir = tempfile::tempdir().unwrap();
10775 let q = Queue::at(dir.path().join("queue"));
10776 let src = dir.path().join("shot.png");
10777 std::fs::write(&src, "x").unwrap();
10778 let mut t = task();
10779 q.attach(&mut t, &[src]).unwrap();
10780 let paths = task_attachments(&q, &t).unwrap();
10781 assert_eq!(paths.len(), 1);
10782 assert!(paths[0].is_absolute() && paths[0].is_file());
10783 std::fs::remove_file(&paths[0]).unwrap();
10784 let err = task_attachments(&q, &t).unwrap_err().to_string();
10785 assert!(err.contains("shot.png"), "{err}");
10786 }
10787
10788 #[tokio::test]
10789 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
10790 let dir = tempfile::tempdir().unwrap();
10791 let queue_path = dir.path().join("queue");
10792 let queue = Queue::at(queue_path.clone());
10793 let mut t = task();
10794 t.source = Source::Agent {
10795 run: "originating-chat".into(),
10796 node: crate::queue::CHAT_NODE.into(),
10797 };
10798 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
10799 stage: "config-fetch",
10800 target: "origin/main".into(),
10801 });
10802 record_config_failure(&mut t, &fetch);
10803 t.hold_machine(Some(format!("{fetch:#}")));
10804 let fetch_key = t.chat_notice_due().unwrap();
10805 t.chat_notices.push(fetch_key.clone());
10806 queue.put(&mut t).unwrap();
10807
10808 let queue = Queue::at(queue_path);
10809 let mut t = queue.get(&t.id).unwrap();
10810 let a = dir.path().join("a.toml");
10811 let b = dir.path().join("b.toml");
10812 for (path, text, new_cause) in [
10813 (&a, "[graph", true),
10814 (&a, "[verify", false),
10815 (&a, "[graph]\nimplementers = 'invalid-type'", true),
10816 (&b, "[graph]\nimplementers = 'invalid-type'", true),
10817 ] {
10818 t.release_operator("test", None);
10819 std::fs::write(path, text).unwrap();
10820 let error = Config::discover_fetched(dir.path(), Some(path))
10821 .await
10822 .unwrap_err();
10823 record_config_failure(&mut t, &error);
10824 t.hold_machine(Some(format!("{error:#}")));
10825 let due = t.chat_notice_due();
10826 assert_eq!(due.is_some(), new_cause, "{error:#}");
10827 if let Some(key) = due {
10828 assert_ne!(key, fetch_key);
10829 t.chat_notices.push(key);
10830 }
10831 }
10832 t.succeed();
10833 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
10834 }
10835
10836 #[test]
10837 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
10838 let dir = tempfile::tempdir().unwrap();
10839 let queue_path = dir.path().join("queue");
10840 let queue = Queue::at(queue_path.clone());
10841 let mut t = task();
10842 t.source = crate::queue::Source::Agent {
10843 run: "originating-chat".into(),
10844 node: crate::queue::CHAT_NODE.into(),
10845 };
10846 let a = dir.path().join("a.png");
10847 let b = dir.path().join("b.png");
10848 std::fs::write(&a, "a").unwrap();
10849 std::fs::write(&b, "b").unwrap();
10850 queue.attach(&mut t, &[a, b]).unwrap();
10851 let paths = task_attachments(&queue, &t).unwrap();
10852 for path in &paths {
10853 std::fs::remove_file(path).unwrap();
10854 }
10855 let missing = task_attachments(&queue, &t).unwrap_err();
10856 t.attempts += 1;
10857 let reason = missing.to_string();
10858 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10859 let first = t.chat_notice_due().expect("first attachment notice");
10860 t.chat_notices.push(first.clone());
10861 queue.put(&mut t).unwrap();
10862
10863 let queue = Queue::at(queue_path.clone());
10864 let mut t = queue.get(&t.id).unwrap();
10865 assert!(t.chat_notice_due().is_none(), "restart does not replay");
10866 t.release_operator("test", None);
10867 t.attempts += 1;
10868 let missing = task_attachments(&queue, &t).unwrap_err();
10869 t.fail_startup_target(
10870 "startup-attachments",
10871 missing.name,
10872 "same missing attachment with regenerated wording",
10873 1,
10874 );
10875 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
10876
10877 std::fs::write(&paths[0], "restored").unwrap();
10878 t.release_operator("test", None);
10879 t.attempts += 1;
10880 let missing = task_attachments(&queue, &t).unwrap_err();
10881 let reason = missing.to_string();
10882 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10883 let second = t
10884 .chat_notice_due()
10885 .expect("different attachment must notify");
10886 assert_ne!(first, second);
10887 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
10888 }
10889
10890 #[test]
10891 fn resumed_instruction_is_unchanged_without_any_answers() {
10892 let t = task();
10893 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
10894 }
10895
10896 #[test]
10897 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
10898 let mut t = task();
10899 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10900 let old = t.instruction.clone();
10904
10905 let refreshed = resumed_instruction(&old, &t);
10906 assert!(refreshed.starts_with(&old), "the original text is kept");
10907 assert!(refreshed.contains("Which backend?"));
10908 assert!(refreshed.contains("SQLite"));
10909 }
10910
10911 #[test]
10912 fn resumed_instruction_keeps_an_original_answers_heading() {
10913 let mut t = task();
10914 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
10915 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10916
10917 let refreshed = resumed_instruction(&t.instruction, &t);
10918
10919 assert!(
10920 refreshed.starts_with(&t.instruction),
10921 "an answers heading in the original instruction is not the appended block"
10922 );
10923 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
10924 assert!(refreshed.contains("Which backend?"));
10925 assert!(refreshed.contains("SQLite"));
10926
10927 let repeated = resumed_instruction(&refreshed, &t);
10928 assert_eq!(
10929 repeated, refreshed,
10930 "only the final appended block is refreshed"
10931 );
10932 }
10933
10934 #[test]
10935 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
10936 let mut t = task();
10937 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10938
10939 let once = resumed_instruction(&t.instruction, &t);
10943 let twice = resumed_instruction(&once, &t);
10944 assert_eq!(once, twice);
10945 assert_eq!(once.matches("Which backend?").count(), 1);
10946
10947 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
10949 let refreshed = resumed_instruction(&once, &t);
10950 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
10951 assert!(refreshed.contains("Which backend?"));
10952 assert!(refreshed.contains("Which cache?"));
10953 }
10954
10955 #[test]
10956 fn prepare_instruction_covers_all_three_starters() {
10957 let mut t = task();
10958 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10959
10960 assert_eq!(
10963 prepare_instruction(&Starter::Start, None, &t),
10964 Some(instruction_for(&t))
10965 );
10966
10967 let old = t.instruction.clone();
10970 assert_eq!(
10971 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
10972 Some(resumed_instruction(&old, &t))
10973 );
10974
10975 assert_eq!(
10979 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
10980 None
10981 );
10982 }
10983
10984 #[test]
10985 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
10986 assert_eq!(
10987 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
10988 Starter::Review("magi/eba2/A".to_owned())
10989 );
10990 }
10991
10992 #[test]
10993 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
10994 assert_eq!(
10995 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
10996 Starter::Start,
10997 "a vanished review branch must not fall back to resuming the old run either"
10998 );
10999 }
11000
11001 #[test]
11002 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11003 let mut t = task();
11004 t.start("old-run".to_owned());
11005 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11006 t.release();
11007 let branch = t.review_branch.take();
11008 assert_eq!(
11009 choose_starter(branch.as_deref(), true, Some("old-run")),
11010 Starter::Review("magi/eba2/A".to_owned()),
11011 "a review wins over resuming the old run"
11012 );
11013 }
11014
11015 #[test]
11016 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11017 assert_eq!(
11018 choose_starter(None, false, Some("some-run")),
11019 Starter::Resume("some-run".to_owned())
11020 );
11021 assert_eq!(choose_starter(None, false, None), Starter::Start);
11022 }
11023
11024 #[test]
11025 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11026 let mut released = task();
11027 released.start("stalled-run".to_owned());
11028 released.requeue();
11029 let unfinished = (!released.fresh_start)
11030 .then(|| Some("stalled-run".to_owned()))
11031 .flatten();
11032 assert_eq!(
11033 choose_starter(None, false, unfinished.as_deref()),
11034 Starter::Start,
11035 "release keeps run history but must not resume it"
11036 );
11037 assert_eq!(released.runs, ["stalled-run"]);
11038 }
11039
11040 fn stopped_public_run(status: RunStatus) -> RunState {
11041 let mut state = run_state(status);
11042 state.driver_exited = true;
11043 state.config.graph.land = true;
11044 state.config.graph.review_rounds = 1;
11045 state.reviews = vec![review_round(1)];
11046 state.merge = Some(crate::run::MergeOutcome {
11047 mode: MergeMode::Pr,
11048 ok: false,
11049 detail: "landing stopped".into(),
11050 empty: false,
11051 });
11052 state.pr = Some(crate::run::PrRecord {
11053 url: "https://example.test/pull/1".into(),
11054 number: 1,
11055 state: "open".into(),
11056 checks: "red".into(),
11057 round: 1,
11058 rounds: 1,
11059 red_at_merge: Vec::new(),
11060 });
11061 state
11062 }
11063
11064 #[test]
11065 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11066 for status in [RunStatus::Blocked, RunStatus::Gating] {
11067 let home = tempfile::tempdir().unwrap();
11068 let mut state = stopped_public_run(status);
11069 state.save_under(home.path()).unwrap();
11070 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11071 assert!(!exhausted_review_budget(&state));
11072 let mut task = held_task_with(&state.id);
11073 task.release();
11074 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11075 assert_eq!(
11076 choose_starter(None, false, unfinished.as_deref()),
11077 Starter::Resume(state.id.clone())
11078 );
11079 let question = action_question(&state.id, resume_action(&state.id));
11080 assert_eq!(
11081 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11082 ActionDecision::Resume(state.id.clone())
11083 );
11084 assert_eq!(
11086 crate::resume::validate(home.path(), &state.id).unwrap().id,
11087 state.id
11088 );
11089 let queue = Queue::at(home.path().join("queue"));
11090 let mut controlled = Task::new(
11091 "resume land".into(),
11092 state.instruction.clone(),
11093 state.repo.clone(),
11094 Source::Agent {
11095 run: "talk".into(),
11096 node: crate::queue::CHAT_NODE.into(),
11097 },
11098 );
11099 controlled.runs.push(state.id.clone());
11100 controlled.hold_machine(Some("land stopped".into()));
11101 queue.put(&mut controlled).unwrap();
11102 let resumed =
11103 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11104 assert_eq!(resumed.status, TaskStatus::Queued);
11105 assert_eq!(
11106 resumed.resume_override.unwrap().pinned_run,
11107 Some(state.id.clone())
11108 );
11109 assert_eq!(resumed.attempts, 0);
11110 }
11111 }
11112
11113 #[test]
11114 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11115 for review_rounds in [0, 1] {
11116 let home = tempfile::tempdir().unwrap();
11117 let mut state = run_state(RunStatus::Blocked);
11118 state.config.graph.review_rounds = review_rounds;
11119 if review_rounds > 0 {
11120 let mut review = review_round(1);
11121 review.clean = true;
11122 state.reviews = vec![review];
11123 }
11124 state.gate_ran = true;
11125 state.gate = vec![crate::run::CommandOutcome {
11126 command: "verify".into(),
11127 code: Some(1),
11128 output_tail: "gate failed after repair budget exhausted".into(),
11129 duration_ms: 0,
11130 resource_blocked: false,
11131 }];
11132 state.save_under(home.path()).unwrap();
11133 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11134 assert!(exhausted_review_budget(&state));
11135 let mut task = held_task_with(&state.id);
11136 task.release();
11137 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11138 let question = action_question(&state.id, resume_action(&state.id));
11139 assert_ne!(
11140 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11141 ActionDecision::Resume(state.id.clone())
11142 );
11143 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11144
11145 state.gate_ran = false;
11147 state.gate.clear();
11148 assert!(!exhausted_review_budget(&state));
11149 assert_eq!(
11150 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11151 Some(state.id)
11152 );
11153 }
11154 }
11155
11156 #[test]
11157 fn resource_blocked_verification_resumes_the_same_run() {
11158 for (kind, stage) in [
11159 ("cache", "clean"),
11160 ("coordination", "clean"),
11161 ("cache", "review-e2e"),
11162 ("coordination", "no-reviews"),
11163 ("coordination", "handoff"),
11164 ] {
11165 let home = tempfile::tempdir().unwrap();
11166 let mut state = run_state(RunStatus::Blocked);
11167 state.config.graph.review_rounds = 1;
11168 let mut review = review_round(1);
11169 review.clean = true;
11170 state.reviews = vec![review];
11171 if stage == "review-e2e" {
11172 let review = &mut state.reviews[0];
11173 review.clean = false;
11174 review.blocking = 1;
11175 review.e2e = vec![crate::run::CommandOutcome {
11176 command: "verify".into(),
11177 code: None,
11178 output_tail: "resource wait expired".into(),
11179 duration_ms: 0,
11180 resource_blocked: true,
11181 }];
11182 } else if stage == "no-reviews" {
11183 state.config.graph.review_rounds = 0;
11184 state.reviews.clear();
11185 } else if stage == "handoff" {
11186 let review = &mut state.reviews[0];
11189 review.clean = false;
11190 review.blocking = 1;
11191 review.e2e = vec![crate::run::CommandOutcome {
11192 command: "verify".into(),
11193 code: Some(0),
11194 output_tail: String::new(),
11195 duration_ms: 0,
11196 resource_blocked: false,
11197 }];
11198 }
11199 state.gate_ran = false;
11200 state.resource_block = Some(crate::run::ResourceBlock {
11201 node: if stage == "review-e2e" {
11202 "review"
11203 } else {
11204 "gate"
11205 }
11206 .into(),
11207 kind: kind.into(),
11208 detail: "resource wait expired".into(),
11209 at: jiff::Timestamp::now(),
11210 });
11211 state.save_under(home.path()).unwrap();
11212 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11213 assert!(!exhausted_review_budget(&state));
11214 let mut task = held_task_with(&state.id);
11215 task.release();
11216 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11217 assert_eq!(
11218 choose_starter(None, false, unfinished.as_deref()),
11219 Starter::Resume(state.id.clone())
11220 );
11221 let question = action_question(&state.id, resume_action(&state.id));
11222 assert_eq!(
11223 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11224 ActionDecision::Resume(state.id.clone())
11225 );
11226 assert_eq!(
11227 crate::resume::validate(home.path(), &state.id).unwrap().id,
11228 state.id
11229 );
11230 let queue = Queue::at(home.path().join("queue"));
11231 let mut controlled = Task::new(
11232 "resume verification".into(),
11233 state.instruction.clone(),
11234 state.repo.clone(),
11235 Source::Agent {
11236 run: "talk".into(),
11237 node: crate::queue::CHAT_NODE.into(),
11238 },
11239 );
11240 controlled.runs.push(state.id.clone());
11241 controlled.hold_machine(Some("verification resource unavailable".into()));
11242 queue.put(&mut controlled).unwrap();
11243 let resumed =
11244 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11245 assert_eq!(resumed.status, TaskStatus::Queued);
11246 assert_eq!(
11247 resumed.resume_override.unwrap().pinned_run,
11248 Some(state.id.clone())
11249 );
11250 assert_eq!(resumed.attempts, 0);
11251
11252 state.config.graph.review_rounds = 1;
11254 let mut unresolved = review_round(1);
11255 unresolved.blocking = 1;
11256 state.reviews = vec![unresolved];
11257 state.resource_block = None;
11258 state.save_under(home.path()).unwrap();
11259 assert!(exhausted_review_budget(&state));
11260 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11261 }
11262 }
11263
11264 #[tokio::test]
11265 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11266 let home = tempfile::tempdir().unwrap();
11267 let mut owner = stopped_public_run(RunStatus::Blocked);
11268 owner.save_under(home.path()).unwrap();
11269 let history = vec![owner.id.clone()];
11270 let repo = owner.repo.clone();
11271 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11272 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11273 let initial =
11274 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11275 assert!(
11276 initial.is_pending(),
11277 "a busy record must not hold or resume the task"
11278 );
11279 owner.retired_to = Some("20260101-000001-bbbb".into());
11280 owner.save_under(home.path()).unwrap();
11281 drop(lock);
11282 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11283 .await
11284 .expect("ownership lock released")
11285 .expect("read settled owner");
11286 assert!(
11287 result.is_none(),
11288 "the retired owner must not be resumed from a stale read"
11289 );
11290 }
11291
11292 #[tokio::test]
11293 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11294 let home = tempfile::tempdir().unwrap();
11295 let mut owner = stopped_public_run(RunStatus::Blocked);
11296 let repo = tempfile::tempdir().unwrap();
11297 crate::git::git(repo.path(), &["init"]).await.unwrap();
11298 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11299 owner.id = "20260101-000000-aaaa".into();
11300 owner.save_under(home.path()).unwrap();
11301 let mut successor = run_state(RunStatus::Blocked);
11302 successor.id = "20260101-000001-bbbb".into();
11303 successor.save_under(home.path()).unwrap();
11304 let runs = vec![owner.id.clone(), successor.id.clone()];
11305 assert_eq!(
11306 public_resume_owner(home.path(), &runs, &owner.repo)
11307 .await
11308 .unwrap()
11309 .unwrap()
11310 .id,
11311 owner.id
11312 );
11313 let subdir = owner.repo.join("nested");
11314 std::fs::create_dir(&subdir).unwrap();
11315 assert_eq!(
11316 public_resume_owner(home.path(), &runs, &subdir)
11317 .await
11318 .unwrap()
11319 .unwrap()
11320 .id,
11321 owner.id
11322 );
11323 #[cfg(unix)]
11324 {
11325 let alias = home.path().join("repo-alias");
11326 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
11327 assert_eq!(
11328 public_resume_owner(home.path(), &runs, &alias.join("nested"))
11329 .await
11330 .unwrap()
11331 .unwrap()
11332 .id,
11333 owner.id
11334 );
11335 }
11336 let different = tempfile::tempdir().unwrap();
11337 crate::git::git(different.path(), &["init"]).await.unwrap();
11338 assert!(
11339 public_resume_owner(home.path(), &runs, different.path())
11340 .await
11341 .is_err()
11342 );
11343 std::fs::remove_file(
11344 home.path()
11345 .join("runs")
11346 .join(&successor.id)
11347 .join("run.json"),
11348 )
11349 .unwrap();
11350 assert!(
11351 public_resume_owner(home.path(), &runs, &owner.repo)
11352 .await
11353 .is_err()
11354 );
11355 }
11356
11357 #[test]
11358 fn an_ordinary_release_keeps_a_resumable_run_available() {
11359 let mut released = task();
11360 released.start("stalled-run".to_owned());
11361 released.release();
11362 let unfinished = (!released.fresh_start)
11363 .then(|| Some("stalled-run".to_owned()))
11364 .flatten();
11365 assert_eq!(
11366 choose_starter(None, false, unfinished.as_deref()),
11367 Starter::Resume("stalled-run".to_owned()),
11368 "manual release must preserve the normal resume path"
11369 );
11370 }
11371
11372 #[test]
11373 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
11374 let mut state = run_state(RunStatus::Blocked);
11375 state.config.graph.review_rounds = 3;
11376 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
11377 assert!(exhausted_review_budget(&state));
11378
11379 state.reviews.pop();
11381 assert!(!exhausted_review_budget(&state));
11382
11383 let mut stalled = run_state(RunStatus::Stalled);
11386 stalled.config.graph.review_rounds = 1;
11387 stalled.reviews = vec![review_round(1)];
11388 assert!(!exhausted_review_budget(&stalled));
11389 }
11390
11391 fn review_round(round: usize) -> crate::run::ReviewRound {
11392 crate::run::ReviewRound {
11393 round,
11394 head: "deadbeef".to_owned(),
11395 verified_head: None,
11396 verified_at: None,
11397 reviews: Vec::new(),
11398 e2e: Vec::new(),
11399 verify_retried: false,
11400 e2e_deferred: false,
11401 e2e_defer_reason: None,
11402 fix: None,
11403 blocking: 0,
11404 answered: 1,
11405 expected: 1,
11406 clean: false,
11407 progressed: true,
11408 vote_split: false,
11409 reconsideration: Vec::new(),
11410 verdict: None,
11411 }
11412 }
11413
11414 #[test]
11415 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
11416 let mut run = RunState::new(
11417 PathBuf::from("/repo"),
11418 "main".to_owned(),
11419 "abc1234def".to_owned(),
11420 "add retries".to_owned(),
11421 Config::default(),
11422 );
11423 run.status = RunStatus::Judging;
11424 run.parked = true;
11425 let mut task = Task::new(
11426 "add retries".to_owned(),
11427 "add retries".to_owned(),
11428 PathBuf::from("/repo"),
11429 crate::queue::Source::Human,
11430 );
11431 task.status = TaskStatus::Failed;
11432 task.runs = vec![run.id.clone()];
11433 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
11434 assert!(with(&task, &run), "parked after judging is the case");
11435 let mut parked_task = task.clone();
11436 parked_task.status = TaskStatus::Parked;
11437 assert!(with(&parked_task, &run), "the Parked status is the case");
11438
11439 let mut not_parked = run.clone();
11440 not_parked.parked = false;
11441 not_parked.status = RunStatus::Stalled;
11442 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
11443
11444 let mut fresh = task.clone();
11445 fresh.fresh_start = true;
11446 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
11447
11448 let mut review = task.clone();
11449 review.review_branch = Some("magi/x/A".to_owned());
11450 assert!(!with(&review, &run), "review is ranked before resume");
11451
11452 let mut held = task.clone();
11453 held.status = TaskStatus::Held;
11454 assert!(!with(&held, &run), "a hold stays visible to the conductor");
11455
11456 let mut released = run.clone();
11457 released.released_to = Some("20260901-000000-new1".to_owned());
11458 assert!(!with(&task, &released), "nothing left to resume into");
11459
11460 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
11461 "unreadable"
11462 )));
11463 }
11464
11465 #[test]
11466 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
11467 let mut released = RunState::new(
11468 PathBuf::from("/repo"),
11469 "main".to_owned(),
11470 "abc1234def".to_owned(),
11471 "add retries".to_owned(),
11472 Config::default(),
11473 );
11474 released.status = RunStatus::Blocked;
11475 assert_eq!(
11476 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11477 Some(released.id.clone())
11478 );
11479 released.released_to = Some("20260901-000000-new1".to_owned());
11480 assert_eq!(
11481 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11482 None,
11483 "there is nothing left to resume it into"
11484 );
11485 }
11486
11487 #[test]
11488 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
11489 let mut exhausted = RunState::new(
11498 PathBuf::from("/repo"),
11499 "main".to_owned(),
11500 "abc1234def".to_owned(),
11501 "add retries".to_owned(),
11502 Config::default(),
11503 );
11504 exhausted.status = RunStatus::Blocked;
11505 exhausted.config.graph.review_rounds = 1;
11506 exhausted.reviews = vec![review_round(1)];
11507
11508 assert_eq!(
11509 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
11510 None,
11511 "an exhausted `Blocked` run must not be offered as resumable"
11512 );
11513
11514 let mut has_budget_left = RunState::new(
11517 PathBuf::from("/repo"),
11518 "main".to_owned(),
11519 "abc1234def".to_owned(),
11520 "add retries".to_owned(),
11521 Config::default(),
11522 );
11523 has_budget_left.status = RunStatus::Blocked;
11524 has_budget_left.config.graph.review_rounds = 3;
11525 has_budget_left.reviews = vec![review_round(1)];
11526
11527 assert_eq!(
11528 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
11529 Ok(has_budget_left.clone())
11530 }),
11531 Some(has_budget_left.id.clone())
11532 );
11533 }
11534
11535 #[test]
11536 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
11537 let mut run = RunState::new(
11538 PathBuf::from("/repo"),
11539 "main".to_owned(),
11540 "abc1234def".to_owned(),
11541 "add retries".to_owned(),
11542 Config::default(),
11543 );
11544 run.status = RunStatus::Blocked;
11545 run.config.graph.review_rounds = 1;
11546 run.reviews = vec![review_round(1)];
11547 run.pr_publication_pending = true;
11548 assert_eq!(
11549 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11550 Some(run.id.clone()),
11551 "a spent review budget must not turn a failed publication into a re-competition"
11552 );
11553 }
11554
11555 #[test]
11556 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11557 let mut task = Task::new(
11558 "recover work".into(),
11559 "recover work".into(),
11560 PathBuf::from("/repo"),
11561 crate::queue::Source::Human,
11562 );
11563 let mut history = std::collections::BTreeMap::new();
11564 for index in 0..8 {
11565 let mut run = run_state(RunStatus::Blocked);
11566 run.id = format!("20260101-000000-{index:04}");
11567 run.base_commit = format!("upstream-{index}");
11568 if index > 0 {
11570 run.candidates.clear();
11571 }
11572 task.runs.extend([run.id.clone(), run.id.clone()]);
11573 history.insert(run.id.clone(), run);
11574 task.requeue();
11575 }
11576 let check = |task: &Task, starter: &Starter, limit| {
11577 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11578 };
11579 assert!(
11580 check(&task, &Starter::Start, 8)
11581 .unwrap()
11582 .contains("8 unretired")
11583 );
11584 assert!(check(&task, &Starter::Start, 9).is_none());
11585 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11586 task.release_operator("owner", None);
11587 assert_eq!(task.attempts, 0);
11588 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11589 assert!(check(&restored, &Starter::Start, 8).is_some());
11590 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11591 assert!(check(&restored, &Starter::Start, 0).is_none());
11592 }
11593
11594 #[test]
11595 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11596 let mut task = Task::new(
11597 "recover work".into(),
11598 "recover work".into(),
11599 PathBuf::from("/repo"),
11600 crate::queue::Source::Human,
11601 );
11602 let mut run = run_state(RunStatus::Superseded);
11603 run.candidates.clear();
11604 task.runs.push(run.id.clone());
11605 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11606 run.retired_to = Some("authority".into());
11607 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11608 run.retired_to = None;
11609 run.released_to = Some("authority".into());
11610 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11611 assert!(
11612 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11613 .unwrap()
11614 .contains("Cannot verify")
11615 );
11616 assert!(
11618 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11619 panic!("resume must not load historical attempts")
11620 })
11621 .is_none()
11622 );
11623 }
11624
11625 #[test]
11626 fn retired_resume_requires_a_valid_same_repository_authority() {
11627 let mut source = run_state(RunStatus::Blocked);
11628 let mut empty = run_state(RunStatus::Blocked);
11629 empty.retired_to = Some(source.id.clone());
11630 let runs = vec![source.id.clone(), empty.id.clone()];
11631 let resolve = |source: &RunState, empty: &RunState| {
11632 unfinished_run_with(&runs, "fixture", |id| {
11633 if id == source.id {
11634 Ok(source.clone())
11635 } else if id == empty.id {
11636 Ok(empty.clone())
11637 } else {
11638 bail!("unreadable authority")
11639 }
11640 })
11641 };
11642 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11643 assert_eq!(
11644 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11645 Starter::Resume(source.id.clone())
11646 );
11647 source.retired_to = Some(empty.id.clone());
11648 assert!(
11649 resolve(&source, &empty).is_none(),
11650 "cycles do not revive obsolete runs"
11651 );
11652 source.retired_to = None;
11653 source.repo = PathBuf::from("/another-repository");
11654 assert!(resolve(&source, &empty).is_none());
11655 source.repo = empty.repo.clone();
11656 source.released_to = Some("later".into());
11657 assert!(resolve(&source, &empty).is_none());
11658 empty.retired_to = Some("unlisted-run".into());
11659 assert!(resolve(&source, &empty).is_none());
11660 }
11661
11662 #[test]
11663 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11664 let mut older_stalled = RunState::new(
11672 PathBuf::from("/repo"),
11673 "main".to_owned(),
11674 "abc1234def".to_owned(),
11675 "add retries".to_owned(),
11676 Config::default(),
11677 );
11678 older_stalled.status = RunStatus::Stalled;
11679
11680 let mut newest_exhausted = RunState::new(
11681 PathBuf::from("/repo"),
11682 "main".to_owned(),
11683 "abc1234def".to_owned(),
11684 "add retries".to_owned(),
11685 Config::default(),
11686 );
11687 newest_exhausted.status = RunStatus::Blocked;
11688 newest_exhausted.config.graph.review_rounds = 1;
11689 newest_exhausted.reviews = vec![review_round(1)];
11690
11691 assert_eq!(
11692 unfinished_run_with(
11693 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11694 "t",
11695 |_| Ok(newest_exhausted.clone())
11696 ),
11697 None,
11698 "the newest run is exhausted, so nothing here is worth resuming - \
11699 least of all the older, already-superseded run"
11700 );
11701 }
11702
11703 #[test]
11704 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11705 assert_eq!(
11706 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11707 Err(anyhow::anyhow!("fixture is absent"))
11708 }),
11709 None
11710 );
11711 }
11712
11713 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11714 let mut q = ask::Question::new(
11715 run.to_owned(),
11716 "implement".to_owned(),
11717 "impl-A".to_owned(),
11718 "continue?".to_owned(),
11719 String::new(),
11720 vec!["resume で続行する".to_owned(), "other".to_owned()],
11721 );
11722 q.actions.insert("resume で続行する".to_owned(), action);
11723 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11724 .unwrap();
11725 q
11726 }
11727
11728 fn held_task_with(run: &str) -> Task {
11729 let mut t = task();
11730 t.runs = vec![run.to_owned()];
11731 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11732 t
11733 }
11734
11735 fn resume_action(run: &str) -> ask::ChoiceAction {
11736 ask::ChoiceAction::Resume { run: run.into() }
11737 }
11738
11739 #[test]
11740 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11741 for parked in [false, true] {
11742 let mut t = task();
11743 t.start("A".into());
11744 t.start("B".into());
11745 assert_eq!(t.runs, vec!["A", "B"]);
11746 t.start("A".into());
11747 let mut resumed = run_state(RunStatus::Blocked);
11748 resumed.id = "A".into();
11749 resumed.parked = parked;
11750 resumed.driver_exited = true;
11751 t.hold_machine(Some("resumed run stopped".into()));
11752 let q = action_question("A", resume_action("A"));
11753 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11754 assert_eq!(
11755 decide_action(&t, &q, &PHRASES_EN, |id| {
11756 assert_eq!(id, "A");
11757 Ok(resumed.clone())
11758 }),
11759 ActionDecision::Resume("A".into())
11760 );
11761 assert_eq!(
11762 action_standing(&t, &action_question("B", resume_action("B"))),
11763 ActionStanding::Stale
11764 );
11765 t.release_operator("cli", None);
11766 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11767 assert_eq!(id, "A");
11768 Ok(resumed.clone())
11769 });
11770 assert_eq!(
11771 choose_starter(None, false, unfinished.as_deref()),
11772 Starter::Resume("A".into())
11773 );
11774 assert_eq!(t.runs, vec!["B", "A"]);
11775 }
11776 }
11777
11778 #[test]
11779 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
11780 let mut state = crate::land::tests::repair_state(true, true);
11781 state.status = RunStatus::Landing;
11782 state.parked = true;
11783 state.driver_exited = true;
11784 let mut q = ask::Question::new(
11785 state.id.clone(),
11786 land::APPROVAL_NODE.into(),
11787 "land".into(),
11788 "Merge?".into(),
11789 String::new(),
11790 vec![land::APPROVE.into(), land::HOLD.into()],
11791 );
11792 q.id = state.id.clone();
11793 state.land_approval = Some(crate::run::LandApproval {
11794 question: q.id.clone(),
11795 head: "current-head".into(),
11796 });
11797 assert!(land::upgrade_repair_choice(&mut q, &state));
11798 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11799 let mut task = held_task_with(&state.id);
11800 task.status = TaskStatus::Parked;
11801 assert_eq!(
11802 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11803 ActionDecision::Resume(state.id.clone())
11804 );
11805 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11806 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
11807 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
11808 assert_eq!(
11809 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11810 ActionDecision::Resume(state.id.clone()),
11811 "a repair answer newer than the manual hold resumes the existing owner"
11812 );
11813 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11814 for answered_at in [q.answered_at, None] {
11815 let mut older = q.clone();
11816 older.answered_at = answered_at;
11817 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
11818 panic!("a later manual hold must prevent dispatch")
11819 });
11820 let ActionDecision::Refuse(reason) = refused else {
11821 panic!("expected an actionable refusal, got {refused:?}");
11822 };
11823 assert!(reason.contains("later operator decision"));
11824 assert!(reason.contains(&format!("magi task release {}", task.id)));
11825 }
11826 task.operator_decided_at = None;
11827 let mut superseded = state.clone();
11828 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
11829 assert_eq!(
11830 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
11831 ActionDecision::Stale
11832 );
11833 for case in 1..4 {
11834 let mut changed = state.clone();
11835 match case {
11836 1 => changed.status = RunStatus::Blocked,
11837 2 => changed.released_to = Some("new-owner".into()),
11838 _ => changed.land_approval.as_mut().unwrap().head.clear(),
11839 }
11840 assert!(matches!(
11841 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
11842 ActionDecision::Refuse(_)
11843 ));
11844 }
11845 assert!(matches!(
11846 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
11847 ActionDecision::Refuse(_)
11848 ));
11849 task.runs.push("replacement".into());
11850 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11851 assert_eq!(
11852 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
11853 "old owner must not load"
11854 )),
11855 ActionDecision::Stale
11856 );
11857 }
11858
11859 #[test]
11860 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
11861 let mut state = crate::land::tests::repair_state(true, true);
11862 state.status = RunStatus::Landing;
11863 state.parked = true;
11864 state.driver_exited = true;
11865 let mut q = ask::Question::new(
11866 state.id.clone(),
11867 land::APPROVAL_NODE.into(),
11868 "land".into(),
11869 "Merge?".into(),
11870 String::new(),
11871 vec![land::APPROVE.into(), land::HOLD.into()],
11872 );
11873 q.id = state.id.clone();
11874 state.land_approval = Some(crate::run::LandApproval {
11875 question: q.id.clone(),
11876 head: "current-head".into(),
11877 });
11878 state.save().unwrap();
11879 land::upgrade_repair_choice(&mut q, &state);
11880 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11881 Questions::open().put(&mut q).unwrap();
11882 let tmp = tempfile::tempdir().unwrap();
11883 let queue = Queue::at(tmp.path().join("queue"));
11884 let mut task = held_task_with(&state.id);
11885 task.status = TaskStatus::Parked;
11886 queue.put(&mut task).unwrap();
11887 let empty = Arc::new(tokio::sync::Semaphore::new(0));
11888 let free = Arc::new(tokio::sync::Semaphore::new(1));
11889 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
11890 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
11891 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
11892 assert!(claim.permit.is_some());
11893 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
11894 drop(claim);
11895 Questions::open()
11896 .update(&q.id, |q| {
11897 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
11898 Ok(())
11899 })
11900 .unwrap();
11901 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
11902 assert!(claim.permit.is_none());
11903 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
11904 }
11905
11906 #[test]
11907 fn decide_action_resumes_only_the_latest_resumable_run() {
11908 let t = held_task_with("r1");
11909 let q = action_question("r1", resume_action("r1"));
11910 let load = |s: RunState| move |_: &str| Ok(s);
11911 assert_eq!(
11912 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
11913 ActionDecision::Resume("r1".into())
11914 );
11915 let q_other = action_question("r1", resume_action("r0"));
11917 assert!(matches!(
11918 decide_action(
11919 &t,
11920 &q_other,
11921 &PHRASES_EN,
11922 load(run_state(RunStatus::Blocked))
11923 ),
11924 ActionDecision::Refuse(_)
11925 ));
11926 assert!(matches!(
11928 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
11929 ActionDecision::Refuse(_)
11930 ));
11931 let mut released = run_state(RunStatus::Blocked);
11933 released.released_to = Some("elsewhere".into());
11934 assert!(matches!(
11935 decide_action(&t, &q, &PHRASES_EN, load(released)),
11936 ActionDecision::Refuse(_)
11937 ));
11938 assert!(matches!(
11940 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
11941 ActionDecision::Refuse(_)
11942 ));
11943 }
11944
11945 #[test]
11946 fn decide_action_ignores_a_question_about_an_earlier_run() {
11947 let mut t = held_task_with("r1");
11948 t.runs.push("r2".to_owned());
11949 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11950 assert_eq!(
11951 decide_action(
11952 &t,
11953 &action_question("r1", ask::ChoiceAction::Done),
11954 &PHRASES_EN,
11955 never
11956 ),
11957 ActionDecision::Stale
11958 );
11959 }
11960
11961 #[test]
11962 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
11963 let mut t = held_task_with("r1");
11964 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11965 assert_eq!(
11966 decide_action(
11967 &t,
11968 &action_question("r1", ask::ChoiceAction::Requeue),
11969 &PHRASES_EN,
11970 never
11971 ),
11972 ActionDecision::Requeue
11973 );
11974 let done_q = action_question("r1", ask::ChoiceAction::Done);
11975 assert_eq!(
11976 decide_action(&t, &done_q, &PHRASES_EN, never),
11977 ActionDecision::Done
11978 );
11979 t.mark_action_applied(&done_q.id);
11980 assert_eq!(
11981 decide_action(&t, &done_q, &PHRASES_EN, never),
11982 ActionDecision::Skip
11983 );
11984
11985 let mut plain = action_question("r1", ask::ChoiceAction::Done);
11987 plain.actions.clear();
11988 assert_eq!(
11989 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
11990 ActionDecision::Skip
11991 );
11992 let mut running = held_task_with("r1");
11994 running.status = TaskStatus::Running;
11995 assert_eq!(
11996 decide_action(
11997 &running,
11998 &action_question("r1", ask::ChoiceAction::Done),
11999 &PHRASES_EN,
12000 never
12001 ),
12002 ActionDecision::Skip
12003 );
12004 }
12005
12006 #[test]
12007 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12008 crate::run::pin_test_home();
12009 for legacy in [false, true] {
12010 let dir = tempfile::tempdir().unwrap();
12011 let queue = Queue::at(dir.path().join("queue"));
12012 let questions = Questions::at(dir.path().join("questions"));
12013 let run = "20260101-000000-hold";
12014 let mut task = held_task_with(run);
12015 task.status = TaskStatus::Parked;
12016 let mut q = ask::Question::new(
12017 run.into(),
12018 land::APPROVAL_NODE.into(),
12019 "land".into(),
12020 "Repair?".into(),
12021 String::new(),
12022 vec![land::REPAIR.into()],
12023 );
12024 q.actions.insert(land::REPAIR.into(), resume_action(run));
12025 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12026 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12028 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12029 questions.put(&mut q).unwrap();
12030 task.hold_manual(Some("Keep this PR paused".into()));
12031 if legacy {
12032 task.hold_source = None;
12033 }
12034 queue.put(&mut task).unwrap();
12035
12036 apply_task_choice_actions(&queue, &questions, dir.path());
12037 let after = queue.get(&task.id).unwrap();
12038 assert!(after.operator_held());
12039 assert_eq!(after.hold_source, task.hold_source);
12040 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12041 assert_eq!(after.runs, task.runs);
12042 assert_eq!(after.instruction, task.instruction);
12043 assert!(after.resume_override.is_none());
12044 assert!(after.action_applied(&q.id));
12045 let reason = after.hold_reason.as_deref().unwrap();
12046 assert!(reason.starts_with("Keep this PR paused"));
12047 assert!(reason.contains("later operator decision"));
12048 assert!(reason.contains(&format!("magi task release {}", task.id)));
12049
12050 for recovery in [
12051 crate::conduct::Recovery::Review,
12052 crate::conduct::Recovery::Requeue,
12053 ] {
12054 crate::conduct::apply(
12055 &queue,
12056 &questions,
12057 &crate::conduct::Verdict {
12058 decisions: vec![crate::conduct::Decision {
12059 id: task.id.clone(),
12060 recovery: Some(recovery),
12061 ..Default::default()
12062 }],
12063 },
12064 )
12065 .unwrap();
12066 }
12067 apply_task_choice_actions(&queue, &questions, dir.path());
12070 let still_held = queue.get(&task.id).unwrap();
12071 assert!(still_held.operator_held());
12072 assert_eq!(still_held.hold_reason, after.hold_reason);
12073 assert_eq!(still_held.runs, task.runs);
12074 assert!(still_held.review_branch.is_none());
12075 assert!(!still_held.fresh_start);
12076 assert!(queue.next_runnable().is_none());
12077 }
12078 }
12079
12080 #[test]
12081 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
12082 for held in [false, true] {
12083 for retry in [false, true] {
12084 let dir = tempfile::tempdir().unwrap();
12085 let queue = Queue::at(dir.path().join("queue"));
12086 let questions = Questions::at(dir.path().join("questions"));
12087 let home = dir.path().join("home");
12088 let mut t = task();
12089 if held {
12090 t.hold_machine(Some("driver state unknown".into()));
12091 }
12092 queue.put(&mut t).unwrap();
12093 let verdict = serde_json::from_value(serde_json::json!({
12094 "decisions": [{
12095 "id": t.id,
12096 "question": "Driver state unknown; retry the checks or hold?",
12097 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
12098 "actions": {
12099 "daemon: recheck and retry": {"do": "requeue"},
12100 "daemon: hold the task": {"do": "hold"}
12101 }
12102 }]
12103 }))
12104 .unwrap();
12105 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
12106 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12107 let mut q = questions.list().pop().unwrap();
12108 let choice = if retry {
12109 "daemon: recheck and retry"
12110 } else {
12111 "daemon: hold the task"
12112 };
12113 assert_eq!(q.actions.len(), 2);
12114 q.answer(ask::Answer::Choice(choice.into())).unwrap();
12115 questions.put(&mut q).unwrap();
12116
12117 questions.beat(&q.id, ask::WaiterKind::Deputy);
12119 for _ in 0..2 {
12120 resolve_blockers(&queue, &questions);
12121 apply_task_choice_actions(&queue, &questions, &home);
12122 let waiting = queue.get(&t.id).unwrap();
12123 assert_eq!(waiting.status, TaskStatus::Blocked);
12124 assert_eq!(waiting.blocked_by, [q.id.clone()]);
12125 assert!(waiting.answers.is_empty());
12126 assert!(!waiting.action_applied(&q.id));
12127 assert!(runnable(&queue).is_empty());
12128 let free = Arc::new(tokio::sync::Semaphore::new(1));
12129 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
12130 }
12131 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12132
12133 resolve_blockers(&queue, &questions);
12135 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12136 apply_task_choice_actions(&queue, &questions, &home);
12137 let after = queue.get(&t.id).unwrap();
12138 assert_eq!(
12139 after.status,
12140 if retry {
12141 TaskStatus::Queued
12142 } else {
12143 TaskStatus::Held
12144 }
12145 );
12146 assert!(after.action_applied(&q.id));
12147 assert_eq!(after.operator_held(), !retry);
12148 assert_eq!(after.answers.len(), 1);
12149
12150 let mut changed = after;
12152 if retry {
12153 changed.hold_manual(Some("later hold".into()));
12154 } else {
12155 changed.release_operator("test", None);
12156 }
12157 let expected = changed.status;
12158 queue.put(&mut changed).unwrap();
12159 apply_task_choice_actions(&queue, &questions, &home);
12160 assert_eq!(queue.get(&t.id).unwrap().status, expected);
12161 }
12162 }
12163 }
12164
12165 #[test]
12166 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
12167 let dir = tempfile::tempdir().unwrap();
12168 let queue = Queue::at(dir.path().join("queue"));
12169 let questions = Questions::at(dir.path().join("questions"));
12170 let home = dir.path().join("home");
12171 let mut state = run_state(RunStatus::Blocked);
12172 state.id = "20260101-000000-act1".to_owned();
12173 state.save_under(&home).unwrap();
12174
12175 let mut t = held_task_with(&state.id);
12176 queue.put(&mut t).unwrap();
12177 let mut q = action_question(&state.id, resume_action(&state.id));
12178 questions.put(&mut q).unwrap();
12179
12180 apply_task_choice_actions(&queue, &questions, &home);
12181 let after = queue.get(&t.id).unwrap();
12182 assert_eq!(after.status, TaskStatus::Queued);
12183 assert!(!after.fresh_start);
12184 assert!(after.action_applied(&q.id));
12185 let pin = after.resume_override.clone().unwrap();
12186 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
12187 assert!(pin.forced);
12188
12189 let mut again = queue.get(&t.id).unwrap();
12191 again.hold_machine(Some("later".into()));
12192 queue.put(&mut again).unwrap();
12193 apply_task_choice_actions(&queue, &questions, &home);
12194 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12195 }
12196
12197 #[test]
12198 fn a_delivered_answer_is_still_acted_on_exactly_once() {
12199 let dir = tempfile::tempdir().unwrap();
12200 let queue = Queue::at(dir.path().join("queue"));
12201 let questions = Questions::at(dir.path().join("questions"));
12202 let home = dir.path().join("home");
12203 let mut state = run_state(RunStatus::Blocked);
12204 state.id = "20260101-000000-act2".to_owned();
12205 state.save_under(&home).unwrap();
12206
12207 let mut t = held_task_with(&state.id);
12208 queue.put(&mut t).unwrap();
12209 let mut q = action_question(&state.id, resume_action(&state.id));
12210 q.answer_delivered = true;
12211 questions.put(&mut q).unwrap();
12212
12213 apply_task_choice_actions(&queue, &questions, &home);
12216 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12217 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
12218
12219 let mut again = queue.get(&t.id).unwrap();
12221 again.hold_machine(Some("later".into()));
12222 queue.put(&mut again).unwrap();
12223 apply_task_choice_actions(&queue, &questions, &home);
12224 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12225 }
12226
12227 #[test]
12228 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
12229 let dir = tempfile::tempdir().unwrap();
12230 let queue = Queue::at(dir.path().join("queue"));
12231 let questions = Questions::at(dir.path().join("questions"));
12232 let home = dir.path().join("home");
12233 let mut state = run_state(RunStatus::Blocked);
12234 state.id = "20260101-000000-act3".to_owned();
12235 state.save_under(&home).unwrap();
12236 let mut t = held_task_with(&state.id);
12237 queue.put(&mut t).unwrap();
12238 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
12239 questions.put(&mut q).unwrap();
12240
12241 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
12242 apply_task_choice_actions(&queue, &questions, &home);
12243 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12244
12245 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12246 apply_task_choice_actions(&queue, &questions, &home);
12247 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12248 }
12249
12250 #[test]
12251 fn action_standing_tells_the_reasons_apart() {
12252 let mut t = held_task_with("r1");
12253 let q = action_question("r1", ask::ChoiceAction::Requeue);
12254 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12255 let mut plain = q.clone();
12256 plain.actions.clear();
12257 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
12258
12259 t.mark_action_applied(&q.id);
12261 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
12262
12263 let mut t = held_task_with("r1");
12265 t.start("r2".to_owned());
12266 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
12267 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
12269 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12270 t.status = TaskStatus::Blocked;
12271 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12272
12273 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
12275 c.node = crate::conduct::NODE.to_owned();
12276 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
12277 assert!(!ActionStanding::Busy.daemon_owns());
12278 }
12279
12280 #[test]
12281 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
12282 let dir = tempfile::tempdir().unwrap();
12283 let queue = Queue::at(dir.path().join("queue"));
12284 let questions = Questions::at(dir.path().join("questions"));
12285 let home = dir.path().join("home");
12286 let mut state = run_state(RunStatus::Blocked);
12287 state.id = "20260101-000000-act4".to_owned();
12288 state.save_under(&home).unwrap();
12289 let mut t = held_task_with(&state.id);
12290 t.status = TaskStatus::Running;
12291 queue.put(&mut t).unwrap();
12292 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
12293 q.cwd = Some(dir.path().display().to_string());
12294 questions.put(&mut q).unwrap();
12295
12296 let running = queue.get(&t.id).unwrap();
12299 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
12300 assert!(matches!(
12301 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
12302 crate::waiter::Action::Deliver(_)
12303 ));
12304 apply_task_choice_actions(&queue, &questions, &home);
12306 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
12307 let mut back = queue.get(&t.id).unwrap();
12308 back.hold_machine(Some("later".into()));
12309 queue.put(&mut back).unwrap();
12310 apply_task_choice_actions(&queue, &questions, &home);
12311 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
12312 let mut again = queue.get(&t.id).unwrap();
12313 again.hold_machine(Some("again".into()));
12314 queue.put(&mut again).unwrap();
12315 apply_task_choice_actions(&queue, &questions, &home);
12316 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12317 }
12318}