1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 pub task: String,
101 pub run: String,
103}
104
105#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct Status {
113 pub schema: u32,
115 pub pid: u32,
117 pub started_at: Timestamp,
119 pub updated_at: Timestamp,
121 pub idle: bool,
123 pub current: Vec<Current>,
129 pub completed: usize,
131 pub polls: u64,
133 #[serde(default)]
135 pub question_recovery: Option<crate::question_delivery::Recovery>,
136 #[serde(default)]
138 pub worker_protocol: Option<u32>,
139}
140
141impl Status {
142 #[must_use]
144 pub fn new() -> Self {
145 let now = Timestamp::now();
146 Self {
147 schema: SCHEMA,
148 pid: std::process::id(),
149 started_at: now,
150 updated_at: now,
151 idle: true,
152 current: Vec::new(),
153 completed: 0,
154 polls: 0,
155 question_recovery: None,
156 worker_protocol: Some(1),
157 }
158 }
159}
160
161impl Default for Status {
162 fn default() -> Self {
163 Self::new()
164 }
165}
166
167#[derive(Debug, Clone, Serialize, Deserialize)]
169#[serde(default)]
170pub struct Opts {
171 pub repo: PathBuf,
173 pub config: Option<PathBuf>,
175 pub poll: Duration,
177 pub max_attempts: usize,
179 pub once: bool,
181 pub merge: Option<String>,
183 pub worktrees_root: Option<PathBuf>,
192}
193
194impl Default for Opts {
195 fn default() -> Self {
196 Self {
197 repo: PathBuf::from("."),
198 config: None,
199 poll: POLL,
200 max_attempts: 2,
201 once: false,
202 merge: None,
203 worktrees_root: None,
204 }
205 }
206}
207
208impl Opts {
209 fn absolute(&self) -> Result<Self> {
210 let mut opts = self.clone();
211 opts.repo = std::path::absolute(&opts.repo)?;
212 opts.config = opts
213 .config
214 .as_deref()
215 .map(std::path::absolute)
216 .transpose()?;
217 opts.worktrees_root = opts
218 .worktrees_root
219 .as_deref()
220 .map(std::path::absolute)
221 .transpose()?;
222 Ok(opts)
223 }
224}
225
226fn max_concurrent(n: usize) -> usize {
231 n.max(1)
232}
233
234#[must_use]
236pub fn status_path() -> PathBuf {
237 crate::run::home().join("daemon.json")
238}
239
240pub fn write_status(status: &Status) -> Result<()> {
242 write_status_to(&status_path(), status)
243}
244
245pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
250 if let Some(parent) = path.parent() {
251 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
252 }
253 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
254 let tmp = path.with_extension("json.tmp");
255 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
256 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
257 Ok(())
258}
259
260pub fn clear_status() {
263 clear_status_at(&status_path());
264}
265
266fn clear_status_at(path: &Path) {
270 let _ = std::fs::remove_file(path);
271}
272
273#[derive(Debug, Clone, Default)]
285pub struct Stop {
286 stopped: Arc<AtomicBool>,
290 busy: Arc<std::sync::atomic::AtomicUsize>,
292 wake: Arc<Notify>,
296 pause: crate::graph::Pause,
299}
300
301impl Stop {
302 #[must_use]
304 pub fn new() -> Self {
305 Self::default()
306 }
307
308 pub fn stop(&self) {
311 self.stopped.store(true, Ordering::SeqCst);
312 self.wake.notify_one();
316 }
317
318 #[must_use]
320 pub fn stopped(&self) -> bool {
321 self.stopped.load(Ordering::SeqCst)
322 }
323
324 #[must_use]
327 pub fn finishing(&self) -> bool {
328 self.stopped() && self.busy_now()
329 }
330
331 pub fn park(&self, cause: crate::run::ParkCause) {
337 self.pause.park_for(cause);
338 self.stop();
339 }
340
341 #[must_use]
343 pub fn parking(&self) -> bool {
344 self.pause.parked()
345 }
346
347 #[must_use]
349 pub fn pause(&self) -> crate::graph::Pause {
350 self.pause.clone()
351 }
352
353 #[must_use]
355 pub fn busy_now(&self) -> bool {
356 self.busy.load(Ordering::SeqCst) > 0
357 }
358
359 async fn idle(&self, poll: Duration) {
361 tokio::select! {
362 () = tokio::time::sleep(poll) => {}
363 () = self.wake.notified() => {}
364 }
365 }
366}
367
368#[derive(Debug, Clone, Default, Deserialize)]
375#[serde(default)]
376pub struct Reading {
377 pub schema: u32,
379 pub pid: Option<u32>,
381 pub started_at: Option<Timestamp>,
383 pub updated_at: Option<Timestamp>,
385 pub idle: bool,
387 #[serde(deserialize_with = "de_current")]
399 pub current: Vec<Current>,
400 pub completed: u64,
402 pub polls: u64,
404 #[serde(default)]
406 pub question_recovery: Option<crate::question_delivery::Recovery>,
407 pub worker_protocol: Option<u32>,
409}
410
411fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
414where
415 D: serde::Deserializer<'de>,
416{
417 #[derive(Deserialize)]
418 #[serde(untagged)]
419 enum Shape {
420 Many(Vec<Current>),
421 One(Current),
422 }
423 Ok(
424 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
425 Shape::Many(v) => v,
426 Shape::One(c) => vec![c],
427 }),
428 )
429}
430
431impl Reading {
432 #[must_use]
435 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
436 self.updated_at
437 .map(|at| (now.as_second() - at.as_second()).max(0))
438 }
439
440 #[must_use]
444 pub fn running(&self, now: Timestamp) -> bool {
445 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
446 }
447}
448
449#[must_use]
456pub fn read_status(home: &Path) -> Option<Reading> {
457 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
458 serde_json::from_str(&body).ok()
459}
460
461#[must_use]
470pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
471 let mut current = read_status(home)
472 .filter(|reading| reading.running(now))
473 .map(|reading| reading.current)
474 .unwrap_or_default();
475 let workers = crate::worker::observe_active(home).and_then(|jobs| {
476 jobs.into_iter()
477 .filter(|job| job.kind == "task")
478 .map(|job| {
479 let request: TaskWorker = serde_json::from_value(job.payload)?;
480 let run = Queue::at(request.queue)
481 .get(&request.task)
482 .ok()
483 .and_then(|task| task.runs.last().cloned())
484 .unwrap_or_default();
485 Ok(Current {
486 task: request.task,
487 run,
488 })
489 })
490 .collect::<Result<Vec<_>>>()
491 });
492 match workers {
493 Ok(workers) => {
494 for worker in workers {
495 if let Some(existing) = current.iter_mut().find(|c| c.task == worker.task) {
496 *existing = worker;
497 } else {
498 current.push(worker);
499 }
500 }
501 }
502 Err(error) => {
503 tracing::error!("cannot determine active workers: {error:#}");
506 current.extend(
507 Queue::at(home.join("queue"))
508 .list()
509 .into_iter()
510 .map(|task| Current {
511 task: task.id,
512 run: task.runs.last().cloned().unwrap_or_default(),
513 }),
514 );
515 current.extend(
516 crate::run::list_ids_in(&home.join("runs"))
517 .into_iter()
518 .map(|run| Current {
519 task: String::new(),
520 run,
521 }),
522 );
523 }
524 }
525 current
526}
527
528#[must_use]
530pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
531 current_work(home, now).iter().any(|c| c.run == run)
532}
533
534#[must_use]
544pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
545 current_work(home, now)
546 .iter()
547 .any(|c| crate::run::short_of(&c.run) == short)
548}
549
550#[must_use]
552pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
553 current_work(home, now).iter().any(|c| c.task == task)
554}
555
556pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
596 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
597}
598
599fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
603where
604 F: Fn(u32) -> bool,
605{
606 let this_process = std::process::id();
607 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
608 .into_iter()
609 .flatten()
610 .flatten()
611 .map(|e| e.path())
612 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
613 .filter(|p| {
614 match std::fs::read_to_string(p)
615 .ok()
616 .and_then(|body| body.trim().parse::<u32>().ok())
617 {
618 Some(pid) if pid == this_process => false,
622 Some(pid) => !pid_alive(pid),
623 None => p
624 .metadata()
625 .and_then(|m| m.modified())
626 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
627 .is_ok_and(|age| age >= older_than),
628 }
629 })
630 .filter(|p| std::fs::remove_file(p).is_ok())
631 .filter_map(|p| {
632 p.file_stem()
633 .and_then(|s| s.to_str())
634 .map(std::borrow::ToOwned::to_owned)
635 })
636 .collect();
637 swept.sort_unstable();
638 swept
639}
640
641fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
646 task.status == TaskStatus::Running
647 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
648 && !is_working_on_task(home, &task.id, now)
649}
650
651fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
654 queue
655 .list()
656 .into_iter()
657 .filter(|t| is_stalled(t, home, now))
658 .collect()
659}
660
661fn queued_tasks(queue: &Queue) -> Vec<Task> {
667 queue
668 .list()
669 .into_iter()
670 .filter(|t| t.status == TaskStatus::Queued)
671 .collect()
672}
673
674fn finished_tasks(queue: &Queue) -> Vec<Task> {
677 queue
678 .list()
679 .into_iter()
680 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
681 .collect()
682}
683
684fn resolve_blockers(queue: &Queue, questions: &Questions) {
701 for listed in queue.list() {
702 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
703 continue;
704 }
705 let Ok(_claim) = queue.claim(&listed.id) else {
706 continue;
707 };
708 let Ok(mut task) = queue.get(&listed.id) else {
709 continue;
710 };
711 if task.status != TaskStatus::Blocked {
712 continue;
713 }
714 let deleted = queue.apply_deleted_blockers(&mut task, questions);
718 if !deleted.is_empty() {
719 record(queue, &mut task);
720 for id in &deleted {
721 queue.note_dependency_deleted(&task, id);
722 }
723 if task.status != TaskStatus::Blocked {
724 continue;
725 }
726 }
727 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
728 if !missing.is_empty() {
729 let language = language_of(&task, Path::new("."));
730 let mut cause = missing.clone();
731 cause.sort();
732 cause.dedup();
733 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
734 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
735 &task.blocked_by,
736 &missing,
737 &language,
738 )));
739 record(queue, &mut task);
740 continue;
741 }
742 if let Some(q) = task.blocked_by.iter().find_map(|id| {
745 let crate::queue::Dependency::Question(full) =
746 crate::queue::dependency(queue, questions, id)
747 else {
748 return None;
749 };
750 questions.get(&full).ok().filter(|q| {
751 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
752 })
753 }) {
754 let language = language_of(&task, Path::new("."));
755 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
756 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
757 record(queue, &mut task);
758 continue;
759 }
760 let mut changed = false;
761 for id in task.blocked_by.clone() {
762 let resolved = crate::queue::dependency(queue, questions, &id);
763 if let crate::queue::Dependency::Task(full) = &resolved {
764 if queue
765 .get(full)
766 .is_ok_and(|dep| dep.status == TaskStatus::Done)
767 {
768 task.unblock(&id);
769 changed = true;
770 }
771 continue;
772 }
773 if let crate::queue::Dependency::Question(full) = resolved
774 && let Ok(q) = questions.get(&full)
775 && q.status == ask::QuestionStatus::Answered
776 {
777 if q.node == crate::conduct::NODE
781 && q.chosen_action().is_some()
782 && !task.action_applied(&q.id)
783 {
784 continue;
785 }
786 let answer = match &q.answer {
787 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
788 None => String::new(),
789 };
790 task.record_answer(q.summary.clone(), answer);
791 task.unblock(&id);
792 changed = true;
793 }
794 }
795 if changed {
796 record(queue, &mut task);
797 }
798 }
799}
800
801fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
803 if crate::lang::is_japanese(language) {
804 format!(
805 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
806 q.short(),
807 q.summary
808 )
809 } else {
810 format!(
811 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
812 q.short(),
813 q.summary
814 )
815 }
816}
817
818#[derive(Debug, Clone, Copy, PartialEq, Eq)]
821pub(crate) enum ActionStanding {
822 NoAction,
824 Pending,
826 Applied,
829 Stale,
831 Busy,
833}
834
835impl ActionStanding {
836 pub(crate) fn daemon_owns(self) -> bool {
838 matches!(self, Self::Pending | Self::Applied | Self::Stale)
839 }
840}
841
842pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
846 if q.chosen_action().is_none() {
847 ActionStanding::NoAction
848 } else if task.action_applied(&q.id) {
849 ActionStanding::Applied
850 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
851 ActionStanding::Stale
852 } else if matches!(
853 task.status,
854 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
855 ) {
856 ActionStanding::Busy
857 } else {
858 ActionStanding::Pending
859 }
860}
861
862#[derive(Debug, Clone, PartialEq, Eq)]
864enum ActionDecision {
865 Skip,
867 Resume(String),
869 Requeue,
871 Hold,
873 Done,
875 Stale,
878 Refuse(String),
881}
882
883fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
891where
892 F: FnOnce(&str) -> Result<RunState>,
893{
894 let Some(action) = q.chosen_action() else {
895 return ActionDecision::Skip;
896 };
897 if task.action_applied(&q.id)
898 || matches!(
899 task.status,
900 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
901 )
902 {
903 return ActionDecision::Skip;
904 }
905 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
908 return ActionDecision::Stale;
909 }
910 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
911 let stopping = q.node == land::APPROVAL_NODE
912 && q.resolution()
913 .as_deref()
914 .is_some_and(crate::merge_decision::stops);
915 let decision_at = if repair || stopping {
919 q.answered_at.unwrap_or(q.asked_at)
920 } else {
921 q.asked_at
922 };
923 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
924 return if repair || stopping {
925 ActionDecision::Refuse(format!(
926 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
927 if repair { "repair" } else { "stop" },
928 q.id,
929 q.run,
930 task.id
931 ))
932 } else {
933 ActionDecision::Stale
934 };
935 }
936 match action {
937 a if a.maintenance() => ActionDecision::Skip,
938 ask::ChoiceAction::ResumeRun { .. }
939 | ask::ChoiceAction::ReleaseTask { .. }
940 | ask::ChoiceAction::RerunChecks { .. }
941 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
942 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
943 ask::ChoiceAction::Hold => ActionDecision::Hold,
944 ask::ChoiceAction::Done => ActionDecision::Done,
945 ask::ChoiceAction::Resume { run } => {
946 if task.runs.last() != Some(run) {
947 return ActionDecision::Refuse((p.resume_not_latest)(
948 q.short(),
949 ask::short_id(run),
950 ));
951 }
952 match load(run) {
953 Ok(s) if stopping => {
954 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
955 || !s
956 .land_approval
957 .as_ref()
958 .is_some_and(|binding| binding.question == q.id)
959 {
960 ActionDecision::Stale
961 } else if s.liveness(false) == Liveness::Live {
962 ActionDecision::Skip
963 } else if s.status == RunStatus::Landing
964 && !s.released()
965 && s.retired_to.is_none()
966 {
967 ActionDecision::Resume(run.clone())
968 } else {
969 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
970 }
971 }
972 Ok(s)
973 if q.node == land::APPROVAL_NODE
974 && q.resolution().as_deref() == Some(land::REPAIR) =>
975 {
976 if s.merge_repairs.iter().any(|r| r.question == q.id)
977 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
978 {
979 ActionDecision::Stale
982 } else if s.liveness(false) == Liveness::Live {
983 ActionDecision::Skip
984 } else if s.status != RunStatus::Landing
985 || s.released()
986 || s.reviews.is_empty()
987 || !implementers_match(task, &s)
988 || !s
989 .land_approval
990 .as_ref()
991 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
992 {
993 ActionDecision::Refuse(format!(
994 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
995 q.id, run, run
996 ))
997 } else {
998 ActionDecision::Resume(run.clone())
999 }
1000 }
1001 Ok(s) if s.liveness(false) == Liveness::Live => {
1002 ActionDecision::Refuse("run already has a live driver".into())
1003 }
1004 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
1005 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
1006 q.short(),
1007 ask::short_id(run),
1008 )),
1009 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1010 q.short(),
1011 ask::short_id(run),
1012 &format!("{e:#}"),
1013 )),
1014 }
1015 }
1016 }
1017}
1018
1019struct Phrases {
1033 graph_stopped: fn(&str, &str) -> String,
1035 quorum_lost: &'static str,
1036 quota_took_out: &'static str,
1038 run_ended: &'static str,
1040 waiting_for_answer: &'static str,
1042 recovered_running: &'static str,
1043 no_run_to_recover: &'static str,
1044 could_not_start: &'static str,
1045 resume_not_latest: fn(&str, &str) -> String,
1046 resume_cannot_progress: fn(&str, &str) -> String,
1047 resume_unreadable: fn(&str, &str, &str) -> String,
1048 handover_refused: fn(&crate::handover::Refused) -> String,
1051 handover_hint: &'static str,
1053 reviewers_never_answered: fn(usize, usize) -> String,
1056}
1057
1058const PHRASES_EN: Phrases = Phrases {
1059 graph_stopped: |status, detail| {
1060 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1061 },
1062 quorum_lost: "the judging panel lost its quorum",
1063 quota_took_out: "; quota took out ",
1064 run_ended: "run ended ",
1065 waiting_for_answer: " - waiting for operator answer to question ",
1066 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1067 no_run_to_recover: "task was `running` with no live daemon and no readable \
1068 run to recover; held for a human to check what happened",
1069 could_not_start: "could not start the run: ",
1070 resume_not_latest: |q, run| {
1071 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1072 },
1073 resume_cannot_progress: |q, run| {
1074 format!("question {q} asked to resume run {run}, which cannot make progress")
1075 },
1076 resume_unreadable: |q, run, e| {
1077 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1078 },
1079 handover_refused: |r| r.to_string(),
1080 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1081 reviewers_never_answered: |missing, rounds| {
1082 format!(
1083 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1084 refusing to call it clean"
1085 )
1086 },
1087};
1088
1089const PHRASES_JA: Phrases = Phrases {
1090 graph_stopped: |status, detail| {
1091 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1092 },
1093 quorum_lost: "審査パネルが定足数を失いました",
1094 quota_took_out: "。クォータで脱落: ",
1095 run_ended: "run 終了: ",
1096 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1097 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1098 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1099 何が起きたか人が確認するため保留にしました",
1100 could_not_start: "run を開始できませんでした: ",
1101 resume_not_latest: |q, run| {
1102 format!(
1103 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1104 )
1105 },
1106 resume_cannot_progress: |q, run| {
1107 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1108 },
1109 resume_unreadable: |q, run, e| {
1110 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1111 },
1112 handover_refused: |r| {
1113 use crate::handover::Refused;
1114 match r {
1115 Refused::Foreign { branch, path, why } => format!(
1116 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1117 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1118 から、やり直してください(詳細: {why})"
1119 ),
1120 Refused::Unsafe { branch, path, why } => format!(
1121 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1122 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1123 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1124 ),
1125 Refused::ReleaseFailed { branch, path, run } => format!(
1126 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1127 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1128 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1129 ),
1130 }
1131 },
1132 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1133 reviewers_never_answered: |missing, rounds| {
1134 format!(
1135 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1136 クリーンとは認めません"
1137 )
1138 },
1139};
1140
1141fn phrases(language: &str) -> &'static Phrases {
1142 if crate::lang::is_japanese(language) {
1143 &PHRASES_JA
1144 } else {
1145 &PHRASES_EN
1146 }
1147}
1148
1149fn language_of(task: &Task, fallback: &Path) -> String {
1153 crate::lang::of_repo(&repo_for(task, fallback))
1154}
1155
1156pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1160 if q.node == crate::conduct::NODE {
1161 return tasks.iter().find(|t| t.id == q.run);
1162 }
1163 tasks.iter().find(|t| t.runs.contains(&q.run))
1164}
1165
1166fn repair_choice_offers(questions: &Questions) {
1169 for q in questions.list().into_iter().filter(|q| {
1170 q.status.open()
1171 && q.node == crate::conduct::NODE
1172 && q.choices
1173 .iter()
1174 .any(|c| crate::choice_effect::describe(q, c).is_none())
1175 }) {
1176 let _ = questions.update(&q.id, |current| {
1177 if current.status.open() {
1178 crate::choice_effect::executable_conductor(current);
1179 if let Some(deputy) = &mut current.deputy {
1180 deputy.brief = crate::deputy::brief(
1181 ¤t.run,
1182 ¤t.detail,
1183 ¤t.choices,
1184 ¤t.actions,
1185 );
1186 }
1187 }
1188 Ok(())
1189 });
1190 }
1191}
1192
1193pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1196 apply_maintenance_choices(queue, questions, home).await;
1197 apply_task_choice_actions(queue, questions, home);
1198}
1199
1200async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1203 for q in questions.list() {
1204 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1205 continue;
1206 };
1207 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1210 continue;
1211 }
1212 let dir = home.join("choice-actions");
1213 if std::fs::create_dir_all(&dir).is_err() {
1214 continue;
1215 }
1216 let Ok(lock) = std::fs::OpenOptions::new()
1217 .create(true)
1218 .truncate(false)
1219 .write(true)
1220 .open(dir.join(format!("{}.lock", q.id)))
1221 else {
1222 continue;
1223 };
1224 if crate::pr_head::try_lock_file(&lock).is_err() {
1225 continue;
1226 }
1227 let fresh = !home
1228 .join("choice-actions")
1229 .join(format!("{}.json", q.id))
1230 .exists();
1231 let result = async {
1232 let receipt = dir.join(format!("{}.json", q.id));
1233 if receipt.exists() {
1234 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1235 anyhow::ensure!(
1236 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1237 "previous maintenance attempt did not confirm completion; inspect its receipt"
1238 );
1239 return Ok::<_, anyhow::Error>(());
1240 }
1241 let record = |status: &str| -> anyhow::Result<()> {
1244 let tmp = receipt.with_extension("tmp");
1245 std::fs::write(
1246 &tmp,
1247 serde_json::to_vec(&serde_json::json!({
1248 "action": action, "status": status,
1249 }))?,
1250 )?;
1251 std::fs::rename(tmp, &receipt)?;
1252 Ok(())
1253 };
1254 let applied = async {
1255 match action {
1256 ask::ChoiceAction::ResumeRun { run } => {
1257 crate::resume::validate(home, run)?;
1258 record("started")?;
1259 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1260 crate::resume::validate_unqueued(home, run)?;
1261 crate::resume::start(home, run.clone()).await?;
1262 }
1263 }
1264 ask::ChoiceAction::ReleaseTask {
1265 task,
1266 held_at,
1267 reason,
1268 } => {
1269 anyhow::ensure!(
1270 queue.resolve_id(task)? == *task,
1271 "task action requires an exact task ID"
1272 );
1273 let _claim = queue.claim(task)?;
1274 let mut task = queue.get(task)?;
1275 if task.action_applied(&q.id) {
1276 return Ok(());
1277 }
1278 if q.node == crate::conduct::NODE
1283 && q.run == task.id
1284 && task.status == TaskStatus::Blocked
1285 && task.blocked_from == Some(TaskStatus::Held)
1286 && task.blocked_by.as_slice() == [q.id.as_str()]
1287 {
1288 task.unblock(&q.id);
1289 }
1290 anyhow::ensure!(
1291 task.status == TaskStatus::Held
1292 && task.held_at == Some(*held_at)
1293 && task.hold_reason.as_deref() == reason.as_deref()
1294 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1295 "task no longer has the hold this question names"
1296 );
1297 task.release_operator("choice", Some(&q.id));
1298 task.mark_action_applied(&q.id);
1299 queue.put(&mut task)?;
1300 }
1301 ask::ChoiceAction::AbandonRun { run } => {
1302 record("started")?;
1303 crate::driver::abandon_proven_dead(home, run)?;
1304 }
1305 ask::ChoiceAction::RerunChecks {
1306 repo,
1307 pr,
1308 head,
1309 runs,
1310 } => {
1311 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1312 .await?;
1313 }
1314 _ => unreachable!(),
1315 }
1316 Ok::<_, anyhow::Error>(())
1317 }
1318 .await;
1319 match applied {
1320 Ok(()) => record("applied"),
1321 Err(e) => {
1322 record(&format!("refused or failed: {e:#}"))?;
1323 Err(e)
1324 }
1325 }
1326 }
1327 .await;
1328 if home
1333 .join("choice-actions")
1334 .join(format!("{}.json", q.id))
1335 .exists()
1336 && let Some(listed) = task_of_question(&queue.list(), &q)
1337 && !listed.action_applied(&q.id)
1338 && let Ok(_claim) = queue.claim(&listed.id)
1339 && let Ok(mut origin) = queue.get(&listed.id)
1340 {
1341 if !origin.operator_held()
1342 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1343 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1344 {
1345 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1346 origin.hold_machine(Some(format!(
1347 "Maintenance choice {} settled; inspect its target before releasing this task",
1348 q.id
1349 )));
1350 } else {
1351 origin.unblock(&q.id);
1352 }
1353 }
1354 origin.mark_action_applied(&q.id);
1355 let _ = queue.put(&mut origin);
1356 }
1357 if let Err(e) = result
1358 && fresh
1359 {
1360 notices::raise_in(
1361 home,
1362 Notice::warn(
1363 &format!("action:{}", q.id),
1364 format!("Choice action {} was not completed: {e:#}", q.id),
1365 ),
1366 );
1367 }
1368 }
1369}
1370
1371pub(crate) async fn rerun_checks_once(
1378 home: &Path,
1379 request: &str,
1380 repo: &Path,
1381 pr: &str,
1382 head: &str,
1383 runs: &[String],
1384 started: impl FnOnce() -> anyhow::Result<()>,
1385) -> anyhow::Result<()> {
1386 use crate::release_watch::{GhForge, ReleaseForge};
1387 anyhow::ensure!(
1388 !runs.is_empty()
1389 && runs.len() <= 20
1390 && runs
1391 .iter()
1392 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1393 && !pr.starts_with('-'),
1394 "invalid rerun targets"
1395 );
1396 let snap = GhForge.snapshot(repo, pr).await?;
1397 anyhow::ensure!(
1398 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1399 "pull request head or lifecycle changed"
1400 );
1401 anyhow::ensure!(
1402 runs.iter().all(|r| snap
1403 .checks
1404 .iter()
1405 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1406 "named workflow is no longer failed on this pull request"
1407 );
1408 started()?;
1409 let dir = home.join("choice-actions");
1410 std::fs::create_dir_all(&dir)?;
1411 let rerun_lock = std::fs::OpenOptions::new()
1412 .create(true)
1413 .truncate(false)
1414 .write(true)
1415 .open(dir.join("reruns.lock"))?;
1416 crate::pr_head::try_lock_file(&rerun_lock)?;
1417 let ledger = dir.join("reruns.json");
1418 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1419 serde_json::from_slice(&std::fs::read(&ledger)?)?
1420 } else {
1421 Default::default()
1422 };
1423 for run in runs {
1424 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1425 if !recorded.insert(key) {
1426 continue;
1427 }
1428 let tmp = ledger.with_extension("tmp");
1429 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1430 std::fs::rename(tmp, &ledger)?;
1431 GhForge.rerun(repo, run).await?;
1432 }
1433 Ok(())
1434}
1435
1436pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1445 let tasks = queue.list();
1446 for q in questions.list() {
1447 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1449 continue;
1450 }
1451 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1452 continue;
1453 }
1454 let Some(listed) = task_of_question(&tasks, &q) else {
1455 if q.node == land::APPROVAL_NODE
1456 && q.resolution().as_deref().is_some_and(|choice| {
1457 choice == land::REPAIR || crate::merge_decision::stops(choice)
1458 })
1459 {
1460 notices::raise_in(
1461 home,
1462 Notice::warn(
1463 &format!("repair-owner:{}", q.id),
1464 format!(
1465 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1466 q.run, q.run
1467 ),
1468 ),
1469 );
1470 }
1471 continue;
1472 };
1473 if listed.action_applied(&q.id) {
1474 continue;
1475 }
1476 let Ok(_claim) = queue.claim(&listed.id) else {
1477 continue;
1478 };
1479 let Ok(mut task) = queue.get(&listed.id) else {
1480 continue;
1481 };
1482 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1485 let blockers: Vec<_> = task
1486 .blocked_by
1487 .iter()
1488 .filter(|id| {
1489 crate::queue::dependency(queue, questions, id)
1490 == crate::queue::Dependency::Question(q.id.clone())
1491 })
1492 .cloned()
1493 .collect();
1494 if !blockers.is_empty() {
1495 for id in blockers {
1496 task.unblock(&id);
1497 }
1498 let answer = match &q.answer {
1499 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1500 None => String::new(),
1501 };
1502 task.record_answer(q.summary.clone(), answer);
1503 }
1504 }
1505 let language = language_of(&task, Path::new("."));
1506 let decision = decide_action(&task, &q, phrases(&language), |id| {
1507 RunState::load_under(id, home)
1508 });
1509 let ran = matches!(
1510 decision,
1511 ActionDecision::Resume(_)
1512 | ActionDecision::Requeue
1513 | ActionDecision::Hold
1514 | ActionDecision::Done
1515 );
1516 if ran && asker_may_still_read(questions, &q, home) {
1517 continue;
1526 }
1527 let done = matches!(decision, ActionDecision::Done);
1528 match decision {
1529 ActionDecision::Skip => continue,
1530 ActionDecision::Resume(run) => {
1531 task.release_operator("choice", Some(&q.id));
1532 task.resume_override = Some(crate::queue::OperatorResume {
1533 question_id: q.id.clone(),
1534 at: Timestamp::now(),
1535 conductor_rehold: None,
1536 forced: true,
1537 pinned_run: Some(run),
1538 });
1539 }
1540 ActionDecision::Stale => {}
1541 ActionDecision::Requeue => {
1542 task.release_operator("choice-requeue", Some(&q.id));
1543 task.requeue();
1544 }
1545 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1546 ActionDecision::Done => task.succeed(),
1547 ActionDecision::Refuse(why) => {
1548 if task.operator_held() {
1549 task.hold_reason = Some(match task.hold_reason.take() {
1552 Some(reason) => format!("{reason}\n\n{why}"),
1553 None => why,
1554 });
1555 } else {
1556 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1557 task.hold_machine(Some(why));
1558 }
1559 notices::raise(
1560 Notice::warn(
1561 &format!("action:{}", q.id),
1562 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1563 )
1564 .link(Link::Task {
1565 id: task.id.clone(),
1566 }),
1567 );
1568 }
1569 }
1570 task.mark_action_applied(&q.id);
1571 record(queue, &mut task);
1574 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1575 supersede_prior_runs(&task, home);
1576 }
1577 let _ = questions.update(&q.id, |r| {
1580 r.answer_delivered = true;
1581 Ok(())
1582 });
1583 }
1584}
1585
1586fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1590 let now = Timestamp::now();
1591 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1592 return true;
1593 }
1594 q.node != crate::conduct::NODE
1595 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1596 s.seats_active()
1597 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1598 })
1599}
1600
1601fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1612 let tasks = queue.list();
1613 let by_id: std::collections::BTreeMap<&str, &Task> =
1614 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1615 let referenced: Vec<&str> = tasks
1616 .iter()
1617 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1618 .collect();
1619
1620 for mut question in questions.list() {
1621 if !question.status.open() || question.node != crate::conduct::NODE {
1622 continue;
1623 }
1624 if referenced
1629 .iter()
1630 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1631 {
1632 continue;
1633 }
1634 let Some(task) = by_id.get(question.run.as_str()) else {
1635 continue;
1636 };
1637 question.abandon(format!(
1638 "task {} no longer waits for this answer",
1639 task.short()
1640 ));
1641 if let Err(e) = questions.put(&mut question) {
1642 tracing::warn!(
1643 "could not retire question {} for task {}: {e:#}",
1644 question.short(),
1645 task.short()
1646 );
1647 }
1648 }
1649}
1650
1651#[derive(Debug, Clone, Copy)]
1658pub struct Verdict {
1659 pub status: RunStatus,
1661 pub left_pr: bool,
1663 pub quota_hit: bool,
1665 pub parked: bool,
1667 pub no_viable_candidates: bool,
1674}
1675
1676pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1733 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1734}
1735
1736fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1738 if verdict.parked {
1744 task.park(detail);
1745 return;
1746 }
1747 match verdict.status {
1748 RunStatus::Merged | RunStatus::Ready
1749 if task.pr_recovery.as_ref().is_some_and(|s| {
1750 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1751 }) =>
1752 {
1753 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1754 }
1755 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1756 RunStatus::AlreadyInBase => task.already_landed(detail),
1757 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1758 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1759 task.stall(detail)
1760 }
1761 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1762 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1763 RunStatus::Blocked => task.fail(detail, max_attempts),
1764 RunStatus::VerifiedNoop => task.handed_off(detail),
1765 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1766 }
1767}
1768
1769pub fn supersede_prior_runs(task: &Task, home: &Path) {
1818 let now = Timestamp::now();
1819 let last_run_succeeded = task
1826 .runs
1827 .last()
1828 .and_then(|id| RunState::load_under(id, home).ok())
1829 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1830 for id in task.superseded_attempts(last_run_succeeded) {
1831 let mut state = match RunState::load_under(id, home) {
1832 Ok(s) => s,
1833 Err(e) => {
1834 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1835 continue;
1836 }
1837 };
1838 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1839 continue;
1840 }
1841 let daemon_claims = is_working_on(home, id, now);
1842 if state.liveness(daemon_claims) == Liveness::Live {
1843 continue;
1844 }
1845 state.status = RunStatus::Superseded;
1846 if let Err(e) = state.save_under(home) {
1847 tracing::warn!("could not mark run {id} superseded: {e:#}");
1848 }
1849 }
1850}
1851
1852fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1874 for task in queue.list() {
1875 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1876 continue;
1877 }
1878 supersede_prior_runs(&task, home);
1879 }
1880}
1881
1882fn settle_and_diagnose(
1889 task: &mut Task,
1890 verdict: Verdict,
1891 detail: &str,
1892 max_attempts: usize,
1893 state: &RunState,
1894) {
1895 let p = phrases(&state.config.graph.language);
1896 if let Some(d) = &state.merge_decision {
1899 use crate::merge_decision as md;
1900 if task.runs.last().is_some_and(|id| id != &state.id) {
1901 return;
1902 }
1903 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1904 task.succeed();
1905 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1906 if state.liveness(false) != Liveness::Dead {
1907 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1908 return;
1909 }
1910 let replacement = md::Replacement {
1911 question: d.question.clone(),
1912 run: state.id.clone(),
1913 pr: d.target.pr.clone(),
1914 agent: agent.to_owned(),
1915 context: format!(
1916 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1917 d.target.pr,
1918 state.id,
1919 serde_json::to_string(&state.reviews).unwrap_or_default()
1920 ),
1921 };
1922 task.requeue();
1923 task.pr_recovery = None;
1924 task.review_of = None;
1925 task.replacement = Some(replacement);
1926 } else {
1927 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1928 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1929 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1930 }
1931 task.mark_action_applied(&d.question);
1932 if let Some(pass) = state.passes.last() {
1933 task.record_pass_at(
1934 &state.id,
1935 pass.index,
1936 Some(pass.started_at),
1937 task.attempts,
1938 None,
1939 );
1940 }
1941 return;
1942 }
1943 if let Some(replacement) = &task.replacement
1944 && !state.parked
1945 && state.resource_block.is_none()
1946 && (state.viable().is_empty()
1947 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1948 {
1949 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1950 return;
1951 }
1952 let attempts_before = task.attempts;
1953 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
1959 task.notice_cause(
1960 &format!("resource:{}:{}", block.kind, block.node),
1961 None,
1962 None,
1963 );
1964 task.last_error = Some(detail.to_owned());
1965 task.diagnostic = None;
1966 task.attempts = task.attempts.saturating_sub(1);
1967 if block.kind == "disk pressure" {
1968 task.hold_machine(Some(format!(
1969 "{}{}",
1970 crate::disk::RUNNING_PRESSURE,
1971 block.detail
1972 )));
1973 } else {
1974 task.hold_machine(Some(format!(
1975 "{}{} stayed busy past the resource-wait limit before `{}` could start; \
1976 no agent was asked and no attempt was spent",
1977 crate::cache::RESOURCE_HOLD,
1978 block.kind,
1979 block.node
1980 )));
1981 }
1982 if let Some(pass) = state.passes.last() {
1983 task.record_pass_at(
1984 &state.id,
1985 pass.index,
1986 Some(pass.started_at),
1987 attempts_before,
1988 None,
1989 );
1990 }
1991 return;
1992 }
1993 let unresolved = task.hold_notice_cause.clone();
1994 if !verdict.parked
1995 && task
1996 .pr_recovery
1997 .as_ref()
1998 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
1999 && matches!(
2000 verdict.status,
2001 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2002 )
2003 {
2004 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2005 task.succeed();
2006 } else {
2007 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2008 }
2009 } else if !verdict.parked
2010 && task.pr_recovery.is_some()
2011 && matches!(
2012 verdict.status,
2013 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2014 )
2015 && !verdict.quota_hit
2016 {
2017 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2018 } else {
2019 settle_in(task, verdict, detail, max_attempts, p);
2020 }
2021 if !verdict.parked
2022 && !verdict.quota_hit
2023 && matches!(
2024 verdict.status,
2025 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2026 )
2027 {
2028 task.recovery_failed(state);
2029 }
2030 if state.prep_failure.is_some() {
2031 task.hold_notice_cause = unresolved;
2034 task.startup_notice_cause("startup-candidates");
2035 }
2036 if let Some(pass) = state.passes.last() {
2039 let parked = match &pass.end {
2040 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2041 _ => None,
2042 };
2043 task.record_pass_at(
2044 &state.id,
2045 pass.index,
2046 Some(pass.started_at),
2047 attempts_before,
2048 parked,
2049 );
2050 }
2051 if task.status == TaskStatus::Held {
2052 if state.prep_failure.is_none() {
2053 task.notice_cause(
2054 label(verdict.status),
2055 Some(&state.id),
2056 state.passes.last().map(|p| p.index),
2057 );
2058 }
2059 task.diagnostic = diagnostic(state);
2060 note_open_question(task, &state.id, p);
2061 }
2062}
2063
2064fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2079 let Some(home) = crate::run::try_home() else {
2080 return;
2081 };
2082 let open = Questions::at(home.join("questions")).open_for(run);
2083 let Some(q) = open.first() else {
2084 return;
2085 };
2086 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2087 decisions.sort_unstable();
2088 task.notice_cause(
2089 &format!("questions:{}", decisions.join(",")),
2090 Some(run),
2091 None,
2092 );
2093 let base = task.hold_reason.clone().unwrap_or_default();
2094 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2095}
2096
2097fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2108 match last_run {
2109 Some(state) => {
2110 let verdict = Verdict {
2111 status: state.status,
2112 left_pr: state.pr.is_some(),
2113 quota_hit: !state.quota.is_empty(),
2114 parked: state.parked,
2115 no_viable_candidates: state.viable().is_empty(),
2116 };
2117 let detail = format!(
2118 "{}{}",
2119 phrases(&state.config.graph.language).recovered_running,
2120 describe(&state)
2121 );
2122 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2123 }
2124 None => {
2125 let why = phrases(language).no_run_to_recover;
2128 task.last_error = Some(why.to_owned());
2129 task.notice_cause("missing-run", None, None);
2132 task.hold_machine(Some(why.to_owned()));
2133 }
2134 }
2135}
2136
2137fn reclaim_orphaned_running(
2159 queue: &Queue,
2160 max_attempts: usize,
2161 live_workers: &std::collections::BTreeSet<String>,
2162) -> Vec<String> {
2163 let mut reclaimed = Vec::new();
2164 for listed in queue.list() {
2165 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2166 continue;
2167 }
2168 let Ok(_claim) = queue.claim(&listed.id) else {
2169 continue;
2170 };
2171 let Ok(mut task) = queue.get(&listed.id) else {
2175 continue;
2176 };
2177 if task.status != TaskStatus::Running {
2178 continue;
2179 }
2180 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2181 if let Some(state) = &last_run
2191 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2192 {
2193 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2194 }
2195 let language = if last_run.is_none() {
2196 language_of(&task, Path::new("."))
2197 } else {
2198 String::new()
2199 };
2200 reclaim(&mut task, last_run, max_attempts, &language);
2201 if task.status == TaskStatus::Done {
2202 supersede_prior_runs(&task, &crate::run::home());
2203 }
2204 record(queue, &mut task);
2205 reclaimed.push(task.id.clone());
2206 }
2207 reclaimed
2208}
2209
2210fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2242 reclaim_abandoned_runs_with(
2243 home,
2244 now,
2245 crate::proc::pid_status,
2246 crate::proc::process_started_at,
2247 )
2248}
2249
2250fn reclaim_abandoned_runs_with<F, G>(
2256 home: &Path,
2257 now: Timestamp,
2258 query: F,
2259 identity: G,
2260) -> Vec<String>
2261where
2262 F: Fn(u32) -> Option<bool>,
2263 G: Fn(u32) -> Option<String>,
2264{
2265 let mut abandoned = Vec::new();
2266 for entry in std::fs::read_dir(home.join("runs"))
2267 .into_iter()
2268 .flatten()
2269 .flatten()
2270 {
2271 let id = entry.file_name().to_string_lossy().into_owned();
2272 if !crate::run::is_run_id(&id) {
2273 continue;
2274 }
2275 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2278 continue;
2279 };
2280 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2288 continue;
2289 };
2290 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2291 continue;
2292 };
2293 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2294 continue;
2295 }
2296 let daemon_claims = is_working_on(home, &id, now);
2308 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2309 != crate::run::Liveness::Dead
2310 {
2311 continue;
2312 }
2313 state.abandon("daemon");
2314 if let Err(e) = state.save_under(home) {
2315 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2316 continue;
2317 }
2318 if let Some(notice) = notices::run_ended(&state) {
2321 notices::raise_in(home, notice);
2322 }
2323 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2331 tracing::warn!("abandon questions for {id}: {e:#}");
2332 }
2333 abandoned.push(id);
2334 }
2335 abandoned
2336}
2337
2338#[derive(Debug, Clone, Serialize, Deserialize)]
2339struct TaskWorker {
2340 opts: Opts,
2341 queue: PathBuf,
2342 task: String,
2343 #[serde(default)]
2344 lane: PermitKind,
2345 #[serde(default)]
2346 standalone: bool,
2347}
2348
2349fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2350 Ok(crate::worker::active(home)?
2351 .into_iter()
2352 .filter(|job| job.kind == "task")
2353 .collect())
2354}
2355
2356fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2357 task_jobs(home)?
2358 .into_iter()
2359 .filter_map(
2360 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2361 Ok(request) if !request.standalone => Some(Ok(job)),
2362 Ok(_) => None,
2363 Err(error) => Some(Err(error.into())),
2364 },
2365 )
2366 .collect()
2367}
2368
2369#[must_use]
2371pub fn managed(home: &Path) -> bool {
2372 match crate::worker::active(home) {
2373 Err(error) => {
2374 tracing::error!("cannot determine scheduler ownership: {error:#}");
2375 true
2376 }
2377 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2378 Ok(_) => {
2379 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2380 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2381 }
2382 }
2383}
2384
2385pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2387 anyhow::ensure!(
2388 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2389 "the scheduler is stopping; wait for retirement before starting it again"
2390 );
2391 crate::worker::spawn(
2392 home,
2393 "scheduler",
2394 "scheduler",
2395 serde_json::to_value(opts.absolute()?)?,
2396 )
2397 .await
2398}
2399
2400fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2401 home.join("worker-parks").join(format!("{}.json", job.id))
2402}
2403
2404fn request_task_park(
2405 home: &Path,
2406 job: &crate::worker::Job,
2407 cause: crate::run::ParkCause,
2408) -> Result<()> {
2409 let path = park_request_path(home, job);
2410 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2411 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2412 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2413}
2414
2415pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2417 if control == crate::worker::Control::Park {
2418 for job in owned_task_jobs(home)? {
2419 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2420 }
2421 }
2422 crate::worker::request(home, "scheduler", control)
2423}
2424
2425pub async fn restart_detached(home: &Path) -> Result<()> {
2427 let active = crate::worker::active(home)?;
2428 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2429 if prior.is_none() && !managed(home) {
2430 return Ok(());
2431 }
2432 let opts: Opts = match prior {
2433 Some(job) => serde_json::from_value(job.payload)?,
2434 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2435 };
2436 request_control(home, crate::worker::Control::Stop)?;
2437 loop {
2438 let reserved = crate::worker::active(home)?
2439 .iter()
2440 .any(|job| job.kind == "scheduler");
2441 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2442 break;
2443 }
2444 tokio::time::sleep(Duration::from_millis(100)).await;
2445 }
2446 spawn_detached(home, &opts).await?;
2447 Ok(())
2448}
2449
2450pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2452 let opts: Opts = serde_json::from_value(payload)?;
2453 let _ = home;
2454 serve_until(opts, Stop::new()).await
2455}
2456
2457pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2459 let request: TaskWorker = serde_json::from_value(payload)?;
2460 let queue = Queue::at(request.queue);
2461 let key = format!("task:{}", request.task);
2462 let job = task_jobs(home)?
2463 .into_iter()
2464 .find(|job| job.key == key)
2465 .context("task worker reservation is missing")?;
2466 let park_path = park_request_path(home, &job);
2467 let _claim = loop {
2470 match queue.claim(&request.task) {
2471 Ok(claim) => break claim,
2472 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2475 return Err(e.context("claim the task for its worker"));
2476 }
2477 Err(_) => {
2478 sweep_stale_claims(&queue, STALE_CLAIM);
2479 tokio::time::sleep(Duration::from_millis(50)).await;
2480 }
2481 }
2482 };
2483 let mut task = queue.get(&request.task)?;
2484 if !task.status.runnable()
2485 || (!request.standalone
2486 && request.lane
2487 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2488 {
2489 return Ok(());
2490 }
2491 let stop = Stop::new();
2492 let watcher = {
2493 let home = home.to_path_buf();
2494 let stop = stop.clone();
2495 let park_path = park_path.clone();
2496 tokio::spawn(async move {
2497 loop {
2498 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2499 let cause = std::fs::read(&park_path)
2500 .ok()
2501 .and_then(|body| serde_json::from_slice(&body).ok())
2502 .unwrap_or(crate::run::ParkCause::Operator);
2503 stop.park(cause);
2504 return;
2505 }
2506 tokio::time::sleep(Duration::from_millis(200)).await;
2507 }
2508 })
2509 };
2510 let quota = if request.standalone {
2511 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2512 } else {
2513 let status = Arc::new(Mutex::new(Status::new()));
2514 Box::pin(attempt(
2515 &request.opts,
2516 &queue,
2517 &status,
2518 &stop,
2519 crate::graph::Pause::new(),
2520 &mut task,
2521 ))
2522 .await
2523 };
2524 watcher.abort();
2525 let _ = std::fs::remove_file(park_path);
2526 persist_cooldown(home, "a).await?;
2527 Ok(())
2528}
2529
2530pub async fn spawn_claimed(
2533 home: &Path,
2534 opts: &Opts,
2535 queue: &Queue,
2536 task: &Task,
2537 claim: crate::queue::Claim,
2538) -> Result<crate::worker::Job> {
2539 let request = TaskWorker {
2540 opts: opts.absolute()?,
2541 queue: std::path::absolute(queue.root())?,
2542 task: task.id.clone(),
2543 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2544 standalone: true,
2545 };
2546 let job = crate::worker::spawn(
2547 home,
2548 "task",
2549 &format!("task:{}", task.id),
2550 serde_json::to_value(request)?,
2551 )
2552 .await?;
2553 drop(claim);
2554 Ok(job)
2555}
2556
2557pub async fn run_detached_claimed(
2560 opts: &Opts,
2561 queue: &Queue,
2562 task: &mut Task,
2563 claim: crate::queue::Claim,
2564) -> Result<()> {
2565 let home = crate::run::home();
2566 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2567 loop {
2568 crate::worker::active(&home)?;
2571 let current = crate::worker::get(&home, &job.id)?;
2572 if matches!(
2573 current.status,
2574 crate::worker::Status::Finished | crate::worker::Status::Failed
2575 ) {
2576 *task = queue.get(&task.id)?;
2577 anyhow::ensure!(
2578 current.status != crate::worker::Status::Failed,
2579 "task worker failed: {}",
2580 current.result.as_deref().unwrap_or("unknown failure")
2581 );
2582 return Ok(());
2583 }
2584 tokio::time::sleep(Duration::from_millis(200)).await;
2585 }
2586}
2587
2588async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2589 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2590 let _lease = loop {
2591 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2592 break lease;
2593 }
2594 tokio::time::sleep(Duration::from_millis(10)).await;
2595 };
2596 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2597 let path = home.join("task-quota-cooldown.json");
2598 let temporary = path.with_extension("json.tmp");
2599 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2600 std::fs::rename(temporary, path)?;
2601 }
2602 }
2603 Ok(())
2604}
2605
2606fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2607 match std::fs::read(home.join("task-quota-cooldown.json")) {
2608 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2609 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2610 Err(error) => Err(error.into()),
2611 }
2612}
2613
2614pub async fn serve(opts: Opts) -> Result<()> {
2620 tracing::info!(
2621 "automatic update and restart requires magi web; standalone serve does not restart"
2622 );
2623 serve_until(opts, Stop::new()).await
2624}
2625
2626pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2637 let opts = opts.absolute()?;
2638 let home = crate::run::home();
2639 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2640 .context("another scheduler already owns this home")?;
2641 if let Some(reading) = read_status(&home)
2642 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2643 {
2644 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2645 }
2646 if !crate::worker::active(&home)?
2647 .iter()
2648 .any(|job| job.kind == "scheduler")
2649 {
2650 crate::worker::clear_control(&home, "scheduler")?;
2651 }
2652 std::fs::write(
2653 home.join("scheduler-options.json"),
2654 serde_json::to_vec(&opts)?,
2655 )?;
2656 let control = {
2657 let home = home.clone();
2658 let stop = stop.clone();
2659 tokio::spawn(async move {
2660 loop {
2661 match crate::worker::control(&home, "scheduler").ok().flatten() {
2662 Some(crate::worker::Control::Park) => {
2663 let _ = request_control(&home, crate::worker::Control::Park);
2664 stop.park(crate::run::ParkCause::Operator);
2665 break;
2666 }
2667 Some(crate::worker::Control::Stop) => {
2668 stop.stop();
2669 break;
2670 }
2671 None => {}
2672 }
2673 tokio::time::sleep(Duration::from_millis(100)).await;
2674 }
2675 })
2676 };
2677 let signal = {
2678 let stop = stop.clone();
2679 tokio::spawn(async move {
2680 if tokio::signal::ctrl_c().await.is_ok() {
2681 stop.stop();
2682 tracing::info!("scheduler shutdown requested; independent workers continue");
2683 }
2684 })
2685 };
2686
2687 let worktrees_root = opts
2688 .worktrees_root
2689 .clone()
2690 .unwrap_or_else(crate::run::default_worktree_root);
2691 let outcome = drive(
2692 &opts,
2693 &Queue::open(),
2694 &status_path(),
2695 &crate::run::home(),
2696 &worktrees_root,
2697 &stop,
2698 )
2699 .await;
2700
2701 signal.abort();
2702 control.abort();
2703 outcome
2704}
2705
2706async fn drive(
2719 opts: &Opts,
2720 queue: &Queue,
2721 status_file: &Path,
2722 home: &Path,
2723 worktrees_root: &Path,
2724 stop: &Stop,
2725) -> Result<()> {
2726 let recovery_cfg = prepare(&opts.repo, opts).ok();
2734 let daemon_cfg = recovery_cfg
2735 .as_ref()
2736 .map(|c| c.daemon.clone())
2737 .unwrap_or_default();
2738 let reconciled = crate::driver::reconcile(home);
2739 if !reconciled.is_empty() {
2740 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2741 }
2742 let mut initial_status = Status::new();
2743 let owned = owned_task_jobs(home)?;
2744 initial_status.current = current_work(home, Timestamp::now())
2745 .into_iter()
2746 .filter(|current| {
2747 owned
2748 .iter()
2749 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2750 })
2751 .collect();
2752 initial_status.idle = initial_status.current.is_empty();
2753 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2754 recovery_cfg.as_ref(),
2755 daemon_cfg.max_deputies,
2756 ));
2757 let status = Arc::new(Mutex::new(initial_status));
2758 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2759 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2760
2761 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2767
2768 tracing::info!(
2769 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2770 queue.root().display(),
2771 opts.poll.as_secs(),
2772 opts.max_attempts,
2773 concurrency,
2774 if daemon_cfg.pause_for_interrupts {
2775 ", interrupts enabled"
2776 } else {
2777 ""
2778 }
2779 );
2780
2781 let outcome = tokio::select! {
2782 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2783 max_concurrent: concurrency,
2784 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2785 recovery_cfg,
2786 }) => outcome,
2787 () = async {
2788 while !stop.stopped() {
2789 tokio::time::sleep(Duration::from_millis(25)).await;
2790 }
2791 } => {
2792 if stop.parking() {
2793 request_control(home, crate::worker::Control::Park)?;
2794 }
2795 Ok(())
2796 }
2797 };
2798
2799 stop.busy.store(0, Ordering::SeqCst);
2800 beat.abort();
2801 clear_status_at(status_file);
2802 outcome
2803}
2804
2805pub(crate) fn release_watch_settings_from_config(
2808 home: &Path,
2809 repo: &Path,
2810 cfg: &Config,
2811) -> (Vec<PathBuf>, u64) {
2812 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2813 .into_iter()
2814 .map(|r| r.path)
2815 .collect();
2816 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2819 .list()
2820 .into_iter()
2821 .map(|t| t.repo)
2822 .collect();
2823 let runs = home.join("runs");
2824 for id in crate::run::list_ids_in(&runs) {
2825 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2826 .ok()
2827 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2828 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2829 known.extend(repo);
2830 }
2831 for k in known {
2832 let k = k.canonicalize().unwrap_or(k);
2833 if k.join(".git").exists() && !paths.contains(&k) {
2834 paths.push(k);
2835 }
2836 }
2837 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2838 if !paths.contains(&own) {
2839 paths.push(own);
2840 }
2841 (paths, cfg.daemon.release_stall_minutes)
2842}
2843
2844const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2846
2847async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2853 loop {
2854 tokio::time::sleep(HEARTBEAT).await;
2855 let snapshot = {
2856 let mut guard = lock(&status);
2857 guard.updated_at = Timestamp::now();
2858 guard.clone()
2859 };
2860 if let Err(e) = write_status_to(&path, &snapshot) {
2861 tracing::warn!("could not refresh the daemon status file: {e:#}");
2864 }
2865 }
2866}
2867
2868#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2871enum LandResume {
2872 NotLanding,
2875 StillWaiting,
2880 Ready,
2884}
2885
2886fn land_resume_state(task: &Task) -> LandResume {
2890 let Some(run_id) = task.runs.last() else {
2891 return LandResume::NotLanding;
2892 };
2893 let Ok(state) = RunState::load(run_id) else {
2894 return LandResume::NotLanding;
2895 };
2896 if !state.parked {
2897 return LandResume::NotLanding;
2898 }
2899 if state.status == RunStatus::Landing
2900 && state.land_approval.as_ref().is_some_and(|binding| {
2901 ask::Questions::open()
2902 .get(&binding.question)
2903 .is_ok_and(|q| {
2904 q.run == state.id
2905 && q.resolution()
2906 .as_deref()
2907 .is_some_and(crate::merge_decision::stops)
2908 })
2909 })
2910 {
2911 return LandResume::Ready;
2912 }
2913 if !implementers_match(task, &state) {
2914 return LandResume::NotLanding;
2915 }
2916 let node = if state.status == RunStatus::Landing {
2920 land::APPROVAL_NODE
2921 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2922 crate::github_text::ASK_NODE
2923 } else {
2924 return LandResume::NotLanding;
2925 };
2926 let store = ask::Questions::open();
2927 let waiting = store
2928 .list()
2929 .into_iter()
2930 .filter(|q| &q.run == run_id && q.node == node)
2931 .max_by(|a, b| a.id.cmp(&b.id));
2932 let Some(mut q) = waiting else {
2933 return LandResume::Ready;
2934 };
2935 if !q.status.open() {
2936 if q.resolution().as_deref() == Some(land::REPAIR) {
2939 return if state.liveness(false) == Liveness::Live {
2940 LandResume::StillWaiting
2941 } else {
2942 LandResume::NotLanding
2943 };
2944 }
2945 return LandResume::Ready;
2946 }
2947 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2954 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2955 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2956 q.abandon(format!(
2957 "no answer within {}s of asking",
2958 timeout.as_secs().max(1)
2959 ));
2960 if store.put(&mut q).is_ok() {
2963 return LandResume::Ready;
2964 }
2965 }
2966 LandResume::StillWaiting
2967}
2968
2969const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
2977
2978const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
2990
2991#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3004enum Interrupt {
3005 Idle,
3007 Parking {
3019 parked: Vec<String>,
3020 interrupt_task: String,
3021 },
3022 Running {
3030 parked: Vec<String>,
3031 interrupt_task: String,
3032 },
3033 Resuming { parked: Vec<String> },
3040}
3041
3042fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3063 match state {
3064 Interrupt::Idle => {
3065 if in_flight.len() != 1 {
3077 return Interrupt::Idle;
3078 }
3079 match runnable.iter().find(|t| t.interrupt) {
3080 Some(t) => Interrupt::Parking {
3081 parked: in_flight.to_vec(),
3082 interrupt_task: t.id.clone(),
3083 },
3084 None => Interrupt::Idle,
3085 }
3086 }
3087 Interrupt::Parking {
3088 parked,
3089 interrupt_task,
3090 } => {
3091 if in_flight.iter().any(|id| parked.contains(id)) {
3092 Interrupt::Parking {
3094 parked,
3095 interrupt_task,
3096 }
3097 } else if in_flight.contains(&interrupt_task) {
3098 Interrupt::Running {
3099 parked,
3100 interrupt_task,
3101 }
3102 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3103 Interrupt::Parking {
3107 parked,
3108 interrupt_task,
3109 }
3110 } else {
3111 Interrupt::Resuming { parked }
3116 }
3117 }
3118 Interrupt::Running {
3119 parked,
3120 interrupt_task,
3121 } => {
3122 if in_flight.contains(&interrupt_task) {
3123 Interrupt::Running {
3124 parked,
3125 interrupt_task,
3126 }
3127 } else {
3128 Interrupt::Resuming { parked }
3134 }
3135 }
3136 Interrupt::Resuming { parked } => {
3137 if in_flight.iter().any(|id| parked.contains(id)) {
3138 Interrupt::Idle
3144 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3145 Interrupt::Resuming { parked }
3146 } else {
3147 Interrupt::Idle
3150 }
3151 }
3152 }
3153}
3154
3155fn advance_interrupt_tick(
3161 enabled: bool,
3162 state: Interrupt,
3163 in_flight: &[String],
3164 runnable: &[Task],
3165) -> Interrupt {
3166 if !enabled {
3167 return Interrupt::Idle;
3168 }
3169 advance_interrupt(state, in_flight, runnable)
3170}
3171
3172fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3177 match state {
3178 Interrupt::Idle => candidates,
3179 Interrupt::Parking {
3180 parked,
3181 interrupt_task,
3182 } => {
3183 if in_flight.iter().any(|id| parked.contains(id)) {
3184 Vec::new()
3185 } else {
3186 candidates
3187 .into_iter()
3188 .filter(|t| &t.id == interrupt_task)
3189 .collect()
3190 }
3191 }
3192 Interrupt::Running { .. } => Vec::new(),
3193 Interrupt::Resuming { parked } => candidates
3201 .into_iter()
3202 .find(|t| parked.contains(&t.id))
3203 .into_iter()
3204 .collect(),
3205 }
3206}
3207
3208struct DispatchLimits {
3212 max_concurrent: usize,
3215 pause_for_interrupts: bool,
3217 recovery_cfg: Option<Config>,
3218}
3219
3220#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3229enum PermitKind {
3230 None,
3235 Urgent,
3242 #[default]
3245 Ordinary,
3246}
3247
3248fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3251 if priority {
3252 PermitKind::None
3253 } else if urgent {
3254 PermitKind::Urgent
3255 } else {
3256 PermitKind::Ordinary
3257 }
3258}
3259
3260fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3261 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3262 (0usize, 0usize),
3263 |(ordinary, urgent), request| match request.lane {
3264 PermitKind::Ordinary => (ordinary + 1, urgent),
3265 PermitKind::Urgent => (ordinary, urgent + 1),
3266 PermitKind::None => (ordinary, urgent),
3267 },
3268 );
3269 (
3270 limit.saturating_sub(ordinary),
3271 1usize.saturating_sub(urgent),
3272 )
3273}
3274
3275struct DispatchClaim {
3276 task: Task,
3277 claim: crate::queue::Claim,
3278 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3279}
3280
3281fn claim_dispatch(
3284 queue: &Queue,
3285 candidate: &Task,
3286 cooling_down: bool,
3287 ordinary: &Arc<tokio::sync::Semaphore>,
3288 urgent: &Arc<tokio::sync::Semaphore>,
3289) -> Option<DispatchClaim> {
3290 let Ok(claim) = queue.claim(&candidate.id) else {
3291 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3292 return None;
3293 };
3294 let task = match queue.get(&candidate.id) {
3295 Ok(t) if t.status.runnable() => t,
3296 Ok(_) => return None,
3297 Err(e) => {
3298 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3299 return None;
3300 }
3301 };
3302 let resume = land_resume_state(&task);
3303 if resume == LandResume::StillWaiting {
3304 return None;
3305 }
3306 let priority = resume == LandResume::Ready;
3307 if !priority && cooling_down {
3308 return None;
3309 }
3310 let permit = match permit_kind(priority, task.urgent) {
3313 PermitKind::None => None,
3314 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3315 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3316 };
3317 let stamp = Timestamp::now();
3320 let mut task = task;
3321 task.dispatched_at = Some(stamp);
3322 if let Err(e) = queue.modify(&task.id, |t| {
3323 t.dispatched_at = Some(stamp);
3324 true
3325 }) {
3326 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3327 }
3328 Some(DispatchClaim {
3329 task,
3330 claim,
3331 permit,
3332 })
3333}
3334
3335#[derive(Serialize, Deserialize)]
3354struct ServiceWorker {
3355 opts: Opts,
3356 queue: PathBuf,
3357 worktrees_root: PathBuf,
3358 operation: String,
3359}
3360
3361fn load_conductor(home: &Path) -> Result<Conductor> {
3362 match std::fs::read(home.join("conductor-state.json")) {
3363 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3364 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3365 Err(error) => Err(error.into()),
3366 }
3367}
3368
3369fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3370 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3371 .into_iter()
3372 .map(|c| c.task)
3373 .collect();
3374 let stalled = stalled_tasks(queue, home, Timestamp::now());
3375 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3376 let finished = finished_tasks(queue)
3377 .into_iter()
3378 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3379 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3380 .collect();
3381 let queued = queued_tasks(queue)
3382 .into_iter()
3383 .filter(|t| !live.contains(&t.id))
3384 .collect();
3385 (queued, stalled, finished)
3386}
3387
3388async fn start_service(
3389 home: &Path,
3390 opts: &Opts,
3391 queue: &Queue,
3392 worktrees_root: &Path,
3393 operation: &str,
3394 cfg: Option<&Config>,
3395 busy: bool,
3396) -> Result<()> {
3397 let fetch_cfg = if operation == "fetch" {
3400 match prepare(&opts.repo, opts) {
3401 Ok(cfg) => Some(cfg),
3402 Err(error) => {
3403 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3404 return Ok(());
3405 }
3406 }
3407 } else {
3408 None
3409 };
3410 let cfg = fetch_cfg.as_ref().or(cfg);
3411 let ticks = home.join("service-ticks");
3412 std::fs::create_dir_all(&ticks)?;
3413 let receipt = ticks.join(format!("daemon-{operation}"));
3414 let last = std::fs::read(&receipt)
3415 .ok()
3416 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3417 let interval = match operation {
3418 "disk-pressure" => 30,
3419 "cache" => CACHE_CHECK_INTERVAL_SECS,
3420 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3421 _ => opts.poll.as_secs().max(1),
3422 };
3423 let now = Timestamp::now();
3424 if !cache_check_due(last, now, interval) {
3425 return Ok(());
3426 }
3427 let key = format!("service:daemon-{operation}");
3428 if crate::worker::active(home)?
3429 .iter()
3430 .any(|job| job.key == key)
3431 {
3432 return Ok(());
3433 }
3434 let eligible = match operation {
3435 "conductor" => {
3436 let (queued, stalled, finished) = conductor_work(queue, home);
3437 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3438 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3439 }
3440 "maintenance" => Questions::at(home.join("questions"))
3441 .list()
3442 .iter()
3443 .any(|q| {
3444 q.chosen_action().is_some_and(|action| action.maintenance())
3445 && !home
3446 .join("choice-actions")
3447 .join(format!("{}.json", q.id))
3448 .exists()
3449 }),
3450 "housekeeping" => !busy && runnable(queue).is_empty(),
3451 "cache" => {
3452 !busy
3453 && !runnable(queue).is_empty()
3454 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3455 }
3456 "disk-pressure" => busy,
3457 "fetch" => {
3458 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3459 }
3460 _ => false,
3461 };
3462 if !eligible {
3463 return Ok(());
3464 }
3465 let request = ServiceWorker {
3466 opts: opts.absolute()?,
3467 queue: std::path::absolute(queue.root())?,
3468 worktrees_root: std::path::absolute(worktrees_root)?,
3469 operation: operation.to_owned(),
3470 };
3471 if let Err(error) =
3472 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3473 {
3474 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3475 }
3476 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3477 Ok(())
3478}
3479
3480pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3483 let request: ServiceWorker = serde_json::from_value(payload)?;
3484 let opts = &request.opts;
3485 let queue = Queue::at(request.queue);
3486 let questions = Questions::at(home.join("questions"));
3487 match request.operation.as_str() {
3488 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3489 "conductor" => {
3490 let path = home.join("conductor-state.json");
3491 let mut conductor = load_conductor(home)?;
3492 let (queued, stalled, finished) = conductor_work(&queue, home);
3493 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3494 && conductor.worth_a_look(&queue, &stalled, &finished)
3495 {
3496 let cfg = prepare(&opts.repo, opts)?;
3497 conductor
3498 .maybe_run(
3499 &cfg,
3500 &opts.repo,
3501 &queue,
3502 &questions,
3503 home,
3504 &queued,
3505 &stalled,
3506 &finished,
3507 opts.max_attempts,
3508 )
3509 .await;
3510 let temporary = path.with_extension("json.tmp");
3511 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3512 std::fs::rename(temporary, path)?;
3513 }
3514 }
3515 "fetch" => {
3516 let cfg = prepare(&opts.repo, opts)?;
3517 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3518 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3519 tracing::debug!("fetch origins: {outcome:?}");
3520 }
3521 }
3522 "cache" => {
3523 if task_jobs(home)?.is_empty() {
3524 let mut checked = None;
3525 maybe_prune_cache_between_runs(
3526 &opts.repo,
3527 opts,
3528 home,
3529 &Stop::new(),
3530 &mut checked,
3531 Timestamp::now(),
3532 )
3533 .await;
3534 }
3535 }
3536 "disk-pressure" => {
3537 if let Ok(cfg) = prepare(&opts.repo, opts) {
3538 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3539 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3540 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3541 }
3542 for current in current_work(home, Timestamp::now()) {
3543 if let Ok(state) = RunState::load_under(¤t.run, home)
3544 && let Some(reason) = disk_gate(&state.repo, &state.config)
3545 {
3546 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3547 job.key.strip_prefix("task:") == Some(current.task.as_str())
3548 }) {
3549 request_task_park(
3550 home,
3551 &job,
3552 crate::run::ParkCause::Interrupt {
3553 reason: reason.clone(),
3554 },
3555 )?;
3556 }
3557 notices::raise_in(
3558 home,
3559 Notice::warn(
3560 "disk:pressure",
3561 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3562 ),
3563 );
3564 }
3565 }
3566 }
3567 }
3568 "housekeeping" => {
3569 if task_jobs(home)?.is_empty() {
3570 if let Ok(cfg) = prepare(&opts.repo, opts) {
3571 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3572 }
3573 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3574 resweep_superseded_attempts(&queue, home);
3575 triage_held(&queue, home, opts).await;
3576 }
3577 }
3578 operation => bail!("unknown daemon service {operation}"),
3579 }
3580 Ok(())
3581}
3582
3583fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3587 let key = format!("task:{}", request.task);
3588 let payload = serde_json::to_value(request)?;
3589 if crate::worker::active(home)?
3590 .iter()
3591 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3592 {
3593 tracing::warn!(
3594 "task {} acquired a competing worker reservation: {error:#}",
3595 crate::run::short_of(&request.task)
3596 );
3597 return Ok(());
3598 }
3599 tracing::error!("task worker startup stopped dispatch: {error:#}");
3600 notices::raise_in(
3601 home,
3602 Notice::error(
3603 "loop:worker-startup",
3604 "A task worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Startup details are kept in the local execution log.",
3605 ),
3606 );
3607 Err(error).with_context(|| {
3608 format!(
3609 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3610 crate::run::short_of(&request.task)
3611 )
3612 })
3613}
3614
3615async fn poll(
3616 opts: &Opts,
3617 queue: &Queue,
3618 status: &Arc<Mutex<Status>>,
3619 home: &Path,
3620 worktrees_root: &Path,
3621 stop: &Stop,
3622 limits: DispatchLimits,
3623) -> Result<()> {
3624 let mut attempted = Vec::new();
3625 let interrupt_path = home.join("scheduler-interrupt.json");
3626 let mut interrupt = match std::fs::read(&interrupt_path) {
3627 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3628 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3629 Err(error) => return Err(error.into()),
3630 };
3631 let mut previous = std::collections::BTreeSet::<String>::new();
3632 while !stop.stopped() {
3633 let jobs: Vec<_> = crate::worker::active(home)?
3634 .into_iter()
3635 .filter(|job| job.kind == "task")
3636 .collect();
3637 let requests: Vec<_> = jobs
3638 .iter()
3639 .map(|job| {
3640 serde_json::from_value::<TaskWorker>(job.payload.clone())
3641 .with_context(|| format!("read live task worker {}", job.id))
3642 })
3643 .collect::<Result<_>>()?;
3644 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3645 let owned: std::collections::BTreeSet<_> = requests
3646 .iter()
3647 .filter(|r| !r.standalone)
3648 .map(|r| r.task.clone())
3649 .collect();
3650 let current: Vec<_> = requests
3651 .iter()
3652 .filter(|request| !request.standalone)
3653 .map(|request| Current {
3654 task: request.task.clone(),
3655 run: queue
3656 .get(&request.task)
3657 .ok()
3658 .and_then(|task| task.runs.last().cloned())
3659 .unwrap_or_default(),
3660 })
3661 .collect();
3662 {
3663 let mut snapshot = lock(status);
3664 snapshot.polls += 1;
3665 snapshot.completed += previous.difference(&owned).count();
3666 snapshot.current = current;
3667 snapshot.idle = owned.is_empty();
3668 }
3669 previous = owned.clone();
3670 stop.busy.store(owned.len(), Ordering::SeqCst);
3671
3672 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3675 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3676 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3677 let mut permits = Vec::new();
3678 sweep_stale_claims(queue, STALE_CLAIM);
3679 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3680 reclaim_abandoned_runs(home, Timestamp::now());
3681 let questions = Questions::at(home.join("questions"));
3682
3683 crate::task_chat::sweep(
3686 queue,
3687 &crate::talk::Talks::at(home.join("talks")),
3688 &questions,
3689 home,
3690 );
3691 repair_choice_offers(&questions);
3692 resolve_blockers(queue, &questions);
3693 apply_task_choice_actions(queue, &questions, home);
3694 triage::apply_answers(queue, &questions);
3695 reconcile_task_questions(queue, &questions);
3696 let talks = crate::talk::Talks::at(home.join("talks"));
3697 crate::chat_notify::sweep(queue, &talks, home);
3698 crate::chat_watch::sweep(home, &talks);
3699 let mut turns = tokio::task::JoinSet::new();
3700 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3701 while turns.join_next().await.is_some() {}
3703
3704 crate::consult::start_services(
3705 home,
3706 &opts.repo,
3707 limits.recovery_cfg.as_ref(),
3708 opts.config.as_deref(),
3709 )
3710 .await?;
3711 for operation in [
3712 "conductor",
3713 "maintenance",
3714 "housekeeping",
3715 "disk-pressure",
3716 "cache",
3717 "fetch",
3718 ] {
3719 start_service(
3720 home,
3721 opts,
3722 queue,
3723 worktrees_root,
3724 operation,
3725 limits.recovery_cfg.as_ref(),
3726 !live.is_empty(),
3727 )
3728 .await?;
3729 }
3730 let latest = crate::worker::active(home)?;
3731 if latest.iter().any(|job| job.key == "service:daemon-cache")
3732 || latest
3733 .iter()
3734 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3735 {
3736 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3737 continue;
3738 }
3739 let candidates: Vec<Task> = runnable(queue)
3740 .into_iter()
3741 .filter(|task| !live.contains(&task.id))
3742 .filter(|task| !opts.once || !attempted.contains(&task.id))
3743 .collect();
3744 let in_flight: Vec<_> = owned.iter().cloned().collect();
3745 let prior_interrupt = std::mem::discriminant(&interrupt);
3746 interrupt = advance_interrupt_tick(
3747 limits.pause_for_interrupts,
3748 interrupt,
3749 &in_flight,
3750 &candidates,
3751 );
3752 if prior_interrupt != std::mem::discriminant(&interrupt) {
3755 let temporary = interrupt_path.with_extension("json.tmp");
3756 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3757 std::fs::rename(temporary, &interrupt_path)?;
3758 }
3759 if let Interrupt::Parking {
3760 parked,
3761 interrupt_task,
3762 } = &interrupt
3763 {
3764 for job in jobs.iter().filter(|job| {
3765 parked
3766 .iter()
3767 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3768 }) {
3769 if crate::worker::control(home, &job.key)?.is_none() {
3770 request_task_park(
3771 home,
3772 job,
3773 crate::run::ParkCause::Interrupt {
3774 reason: format!(
3775 "task {} asked to run first",
3776 crate::run::short_of(interrupt_task)
3777 ),
3778 },
3779 )?;
3780 }
3781 }
3782 }
3783 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3784 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3785 let mut started = false;
3786 let mut resync = false;
3787 for candidate in candidates {
3788 if stop.stopped() {
3789 break;
3790 }
3791 let Some(DispatchClaim {
3792 task,
3793 claim,
3794 permit,
3795 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3796 else {
3797 continue;
3798 };
3799 if crate::worker::active(home)?
3800 .iter()
3801 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3802 {
3803 drop(claim);
3804 resync = true;
3805 break;
3806 }
3807 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3808 let request = TaskWorker {
3809 opts: opts.absolute()?,
3810 queue: std::path::absolute(queue.root())?,
3811 task: task.id.clone(),
3812 lane,
3813 standalone: false,
3814 };
3815 if let Err(error) = crate::worker::spawn(
3818 home,
3819 "task",
3820 &format!("task:{}", task.id),
3821 serde_json::to_value(&request)?,
3822 )
3823 .await
3824 {
3825 task_worker_spawn_error(home, &request, error)?;
3826 drop(claim);
3827 resync = true;
3828 break;
3829 }
3830 drop(claim);
3831 permits.extend(permit);
3832 attempted.push(task.id);
3833 started = true;
3834 }
3835 if started || resync {
3836 continue;
3837 }
3838 if owned.is_empty() && opts.once {
3839 while crate::worker::active(home)?
3840 .iter()
3841 .any(|job| job.key == "service:daemon-housekeeping")
3842 {
3843 stop.idle(RECHECK_WHILE_BUSY).await;
3844 if stop.stopped() {
3845 return Ok(());
3846 }
3847 }
3848 break;
3849 }
3850 stop.idle(if live.is_empty() {
3851 opts.poll
3852 } else {
3853 RECHECK_WHILE_BUSY.min(opts.poll)
3854 })
3855 .await;
3856 }
3857 stop.busy.store(0, Ordering::SeqCst);
3860 Ok(())
3861}
3862
3863async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
3867 let busy = |error: &anyhow::Error| {
3868 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
3869 error.kind() == std::io::ErrorKind::WouldBlock
3870 || cfg!(windows) && error.raw_os_error() == Some(33)
3871 })
3872 };
3873 match crate::pr_head::owner_lock(home, id) {
3874 Ok(lock) => return Ok(lock),
3875 Err(error) if busy(&error) => {}
3876 Err(error) => return Err(error),
3877 }
3878 let wait = RunState::load_under(id, home)?
3879 .config
3880 .graph
3881 .timeout_resource_wait;
3882 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
3883}
3884
3885async fn public_resume_owner(
3889 home: &Path,
3890 runs: &[String],
3891 repo: &Path,
3892) -> Result<Option<RunState>> {
3893 let current_task = crate::worker::current_job(home)?
3897 .filter(|job| job.kind == "task" && job.pid == Some(std::process::id()))
3898 .map(|job| serde_json::from_value::<TaskWorker>(job.payload).map(|request| request.task))
3899 .transpose()?;
3900 let mut owner = None;
3901 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3902 let _lock = resume_owner_lock(home, id).await?;
3903 let state = RunState::load_under(id, home)?;
3904 if state.released()
3905 || state.retired_to.is_some()
3906 || state.pr_taken_over_by.is_some()
3907 || !crate::handover::landing_checkpoint(&state)
3908 {
3909 continue;
3910 }
3911 anyhow::ensure!(
3912 crate::git::toplevel(&state.repo).await?.canonicalize()?
3913 == crate::git::toplevel(repo).await?.canonicalize()?,
3914 "public owner repository differs; inspect run lineage"
3915 );
3916 anyhow::ensure!(
3917 owner.is_none(),
3918 "multiple retained public owners; inspect run lineage"
3919 );
3920 anyhow::ensure!(
3921 crate::handover::can_resume(&state),
3922 "public owner cannot resume"
3923 );
3924 let claimed = current_work(home, Timestamp::now())
3925 .iter()
3926 .any(|claim| claim.run == *id && current_task.as_deref() != Some(claim.task.as_str()));
3927 anyhow::ensure!(
3928 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3929 "public owner's driver is live or unknown; wait before retrying"
3930 );
3931 owner = Some(state);
3932 }
3933 Ok(owner)
3934}
3935
3936async fn attempt(
3942 opts: &Opts,
3943 queue: &Queue,
3944 status: &Arc<Mutex<Status>>,
3945 stop: &Stop,
3946 interrupt_pause: crate::graph::Pause,
3947 task: &mut Task,
3948) -> Vec<QuotaLoss> {
3949 let repo = repo_for(task, &opts.repo);
3950 tracing::info!(
3951 "task {} — {} (repo {})",
3952 task.short(),
3953 task.title,
3954 repo.display()
3955 );
3956
3957 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3958 Ok(owner) => owner,
3959 Err(e) => {
3960 task.hold_machine(Some(format!(
3961 "Cannot verify existing public owner: {e:#}; no attempt started."
3962 )));
3963 record(queue, task);
3964 return Vec::new();
3965 }
3966 };
3967 let resume_runs = public_owner
3968 .as_ref()
3969 .map(|s| vec![s.id.clone()])
3970 .unwrap_or_else(|| task.runs.clone());
3971
3972 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
3976 && previous
3977 .merge_decision
3978 .as_ref()
3979 .is_some_and(|decision| !decision.closed)
3980 && previous.status != RunStatus::Landing
3981 && !crate::merge_decision::resumable_hold(&previous)
3982 {
3983 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
3984 record(queue, task);
3985 return Vec::new();
3986 }
3987 let stop_owner = resume_runs
3988 .last()
3989 .and_then(|id| RunState::load(id).ok())
3990 .filter(|state| {
3991 crate::merge_decision::resumable_hold(state)
3992 || state.status == RunStatus::Landing
3993 && state.land_approval.as_ref().is_some_and(|binding| {
3994 Questions::open().get(&binding.question).is_ok_and(|q| {
3995 q.run == state.id
3996 && q.resolution()
3997 .as_deref()
3998 .is_some_and(crate::merge_decision::stops)
3999 })
4000 })
4001 });
4002 if let Some(owner) = stop_owner {
4003 let mut runner = match Runner::resume(&owner.id) {
4004 Ok(runner) => runner,
4005 Err(e) => {
4006 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4007 record(queue, task);
4008 return Vec::new();
4009 }
4010 };
4011 runner.on_pause(stop.pause());
4012 runner.watch_interrupt(interrupt_pause);
4013 let quota_before = runner.state.quota.clone();
4014 task.start(owner.id.clone());
4015 record(queue, task);
4016 lock(status).current.push(Current {
4017 task: task.id.clone(),
4018 run: owner.id,
4019 });
4020 runner.state.next_pass_task_entry = true;
4021 let result = runner.execute().await;
4022 return finish_attempt(
4023 opts.max_attempts,
4024 queue,
4025 task,
4026 &runner.state,
4027 "a_before,
4028 result,
4029 );
4030 }
4031
4032 if let Some(owner) = &public_owner
4033 && (task.fresh_start || !may_resume_run(owner, task))
4034 {
4035 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4036 record(queue, task);
4037 return Vec::new();
4038 }
4039
4040 let unfinished = (!task.fresh_start)
4041 .then(|| unfinished_run(&resume_runs, task.short()))
4042 .flatten()
4043 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4044 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4045 if !task.fresh_start
4046 && task.pr_recovery.is_none()
4047 && task.review_branch.is_none()
4048 && task.resume_override.is_none()
4049 && previous
4050 .as_ref()
4051 .is_some_and(|run| run.retired_to.is_some())
4052 && unfinished.is_none()
4053 {
4054 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4055 record(queue, task);
4056 return Vec::new();
4057 }
4058
4059 let saved_recovery = previous.as_ref().filter(|run| {
4060 task.replacement.is_none()
4061 && task.pr_recovery.as_ref().is_some_and(|selected| {
4062 selected.check_task(&task.id, &task.instruction).is_ok()
4063 && selected_starter(selected, Some(run), unfinished.as_deref())
4064 == Starter::Resume(run.id.clone())
4065 })
4066 });
4067
4068 let explicit = match &task.overrides {
4072 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4073 Some(_) => opts.config.as_deref(),
4074 None => opts.config.as_deref(),
4075 };
4076 let (mut config, layers) = if let Some(run) = saved_recovery {
4077 let mut saved = run.clone();
4080 if let Err(e) = saved.configure_recovery_resume() {
4081 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4082 record(queue, task);
4083 return Vec::new();
4084 }
4085 (saved.config, Vec::new())
4086 } else {
4087 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4088 Ok(loaded) => loaded,
4089 Err(e) => {
4090 let reason = format!("[config] {e:#}");
4091 record_config_failure(task, &e);
4092 task.last_error = Some(reason.clone());
4093 task.hold_machine(Some(reason));
4094 record(queue, task);
4095 return Vec::new();
4096 }
4097 };
4098 if let Err(e) = apply_options(&mut config, opts, task) {
4099 task.hold_manual(Some(format!(
4100 "[config] {e:#}; correct the merge override before releasing the task"
4101 )));
4102 record(queue, task);
4103 return Vec::new();
4104 }
4105 if let Some(selected) = &task.pr_recovery
4106 && let Err(e) = selected.configure(&mut config)
4107 {
4108 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4109 record(queue, task);
4110 return Vec::new();
4111 }
4112 (config, layers)
4113 };
4114 let config_evidence = crate::pr_recovery::ConfigEvidence {
4115 layers,
4116 overrides: task.overrides.clone(),
4117 daemon_merge: task
4118 .overrides
4119 .as_ref()
4120 .is_none_or(|o| o.inherit_daemon_options)
4121 .then(|| opts.merge.clone())
4122 .flatten(),
4123 };
4124 apply_task_implementers(&mut config, task);
4125 if let Some(replacement) = &task.replacement {
4126 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4127 old.liveness(false) == Liveness::Dead
4128 && old.merge_decision.as_ref().is_some_and(|d| {
4129 d.closed
4130 && d.question == replacement.question
4131 && d.replacement_run.as_ref().is_none_or(|run| {
4132 task.runs.last() == Some(run)
4133 && RunState::load(run).is_ok_and(|current| {
4134 current.replaces.as_ref().is_some_and(|pin| {
4135 pin.question == replacement.question
4136 && pin.agent == replacement.agent
4137 })
4138 })
4139 })
4140 })
4141 });
4142 if !safe {
4143 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4144 record(queue, task);
4145 return Vec::new();
4146 }
4147 match config.agent(&replacement.agent) {
4148 Ok(agent) if crate::agent::installed(agent) => {}
4149 _ => {
4150 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4151 record(queue, task);
4152 return Vec::new();
4153 }
4154 }
4155 config.graph.implementers = 1;
4156 config.graph.land_approval = true;
4157 config.merge.mode = MergeMode::Pr;
4158 config.graph.selected_implementer = Some(replacement.agent.clone());
4159 }
4160 let p = phrases(&config.graph.language);
4161 let start_failed = p.could_not_start;
4162 if let Ok(base) =
4167 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4168 .await
4169 && let Ok(head) =
4170 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4171 {
4172 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4173 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4174 let source = crate::git::rev_parse(&repo, branch)
4175 .await
4176 .unwrap_or_default();
4177 inputs.push_str(&format!(";review:{branch}:{source}"));
4178 }
4179 if let Some(reason) = task.recovery_exhausted(
4180 &head,
4181 &inputs,
4182 task.attempt_limit
4183 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4184 ) {
4185 task.hold_unrequested_retry(Some(reason));
4186 record(queue, task);
4187 return Vec::new();
4188 }
4189 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4190 }
4191
4192 if let Some(reason) = disk_gate(&repo, &config) {
4200 task.notice_cause("disk-space", None, None);
4201 task.last_error = Some(reason.clone());
4202 task.hold_machine(Some(reason.clone()));
4203 record(queue, task);
4204 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4205 notices::raise(
4208 Notice::warn(
4209 &format!("disk:{}", repo.display()),
4210 "A task was held for want of free disk space; free some, then release it from the queue.",
4211 )
4212 .link(Link::Task {
4213 id: task.id.clone(),
4214 }),
4215 );
4216 return Vec::new();
4217 }
4218
4219 let review_branch = take_review_branch(task, previous.as_ref());
4244 let branch_exists = match &review_branch {
4245 Some(branch) => crate::git::branch_exists(&repo, branch)
4246 .await
4247 .unwrap_or(false),
4248 None => false,
4249 };
4250 let proposed = match (&review_branch, &unfinished, &task.review_of) {
4254 (None, None, Some(branch)) => Starter::Review(branch.clone()),
4255 _ => choose_starter(
4256 review_branch.as_deref(),
4257 branch_exists,
4258 unfinished.as_deref(),
4259 ),
4260 };
4261 let retry_selected = task.replacement.is_some()
4262 && task.resumption.is_some()
4263 && !task.fresh_start
4264 && previous.as_ref().is_some_and(|run| {
4265 run.replaces.is_some()
4266 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4267 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4268 });
4269 let stop_pin = task
4270 .resume_override
4271 .as_ref()
4272 .filter(|pin| {
4273 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4274 q.node == land::APPROVAL_NODE
4275 && q.resolution()
4276 .as_deref()
4277 .is_some_and(crate::merge_decision::stops)
4278 })
4279 })
4280 .and_then(|pin| pin.pinned_run.clone());
4281 let starter = if let Some(owner) = &public_owner {
4282 Starter::Resume(owner.id.clone())
4283 } else if let Some(run) = stop_pin {
4284 Starter::Resume(run)
4287 } else if retry_selected {
4288 Starter::Resume(previous.as_ref().unwrap().id.clone())
4289 } else if let Some(selected) = &task.pr_recovery {
4290 if selected.check_task(&task.id, &task.instruction).is_ok() {
4291 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4292 } else {
4293 Starter::Review(selected.branch.clone())
4294 }
4295 } else {
4296 starter_with_implementers(task, previous.as_ref(), proposed)
4297 };
4298 if let Some(pin) = &task.resume_override
4301 && let Some(run) = &pin.pinned_run
4302 && starter != Starter::Resume(run.clone())
4303 {
4304 let run = run.clone();
4305 task.notice_cause("resume-owner", Some(&run), None);
4306 task.hold_machine(Some(format!(
4307 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4308 task.id
4309 )));
4310 record(queue, task);
4311 return Vec::new();
4312 }
4313 if task.replacement.is_some()
4314 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4315 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4316 {
4317 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4318 record(queue, task);
4319 return Vec::new();
4320 }
4321 if let Some(reason) = retained_attempt_hold(
4322 task,
4323 &starter,
4324 config.disk.retained_attempt_limit,
4325 RunState::load,
4326 ) {
4327 if let Starter::Review(branch) = &starter {
4330 task.review_branch = Some(branch.clone());
4331 }
4332 task.hold_unrequested_retry(Some(reason));
4333 record(queue, task);
4334 return Vec::new();
4335 }
4336 let attachments = match task_attachments(queue, task) {
4341 Ok(a) => a,
4342 Err(e) => {
4343 task.attempts += 1;
4344 task.fail_startup_target(
4345 "startup-attachments",
4346 e.name.clone(),
4347 format!("{start_failed}{e:#}"),
4348 opts.max_attempts,
4349 );
4350 record(queue, task);
4351 return Vec::new();
4352 }
4353 };
4354 let startup_pause = stop.pause();
4355 let startup_interrupt = interrupt_pause.clone();
4356 let wait = crate::repo_lock::WaitPolicy::new(
4357 Duration::from_secs(config.graph.timeout_resource_wait),
4358 move || startup_pause.parked() || startup_interrupt.parked(),
4359 );
4360 let started = wait.scope(Box::pin(async { match &starter {
4363 Starter::Review(_) if task.pr_recovery.is_some() => {
4364 let selected = task.pr_recovery.clone().expect("selected recovery");
4365 match selected.check_task(&task.id, &task.instruction) {
4366 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4367 .await
4368 .map(|mut r| {
4369 r.state.attachments = attachments.clone();
4370 r
4371 }),
4372 Err(e) => Err(e),
4373 }
4374 }
4375 Starter::Review(branch) => {
4376 tracing::info!(
4377 "task {} reopens `{branch}` as a review-only pass",
4378 task.short()
4379 );
4380 let takeover = crate::handover::Takeover {
4383 earlier: task.earlier_attempts().to_vec(),
4384 home: crate::run::home(),
4385 choice: take_divergence_answer(branch, &config.merge.remote, task),
4386 task_instruction: Some(instruction_for(task)),
4387 };
4388 Runner::review_taking_over(
4389 &repo,
4390 branch,
4391 config,
4392 Some(takeover),
4393 crate::run::Origin::queue(&task.id),
4394 )
4395 .await
4396 }
4397 Starter::Resume(id) => {
4398 tracing::info!("resuming run {id} rather than competing again");
4399 Runner::resume(id).and_then(|mut r| {
4400 if retry_selected {
4401 for candidate in &mut r.state.candidates {
4404 candidate.failed = None;
4405 candidate.empty = false;
4406 candidate.commits = 0;
4407 }
4408 r.state.status = RunStatus::Implementing;
4409 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4410 }
4411 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4412 r.state.sources = task.sources.clone();
4413 }
4414 if r.state.prep_failure.is_some() {
4415 let approval = task
4416 .resumption
4417 .as_ref()
4418 .context("preparation retry requires an explicit task release")?;
4419 r.retry_preparation(approval.at)?;
4420 }
4421 if let Some(instruction) =
4422 prepare_instruction(&starter, Some(&r.state.instruction), task)
4423 {
4424 r.state.instruction = instruction;
4425 }
4426 r.state.attachments = attachments.clone();
4427 if task.replacement.is_some() {
4428 r.state.config.merge.mode = MergeMode::Pr;
4429 r.state.config.graph.land_approval = true;
4430 }
4431 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4435 .overrides
4436 .as_ref()
4437 .and_then(|o| o.merge.as_deref())
4438 .and_then(|m| merge_mode(m).ok())
4439 {
4440 r.state.config.merge.mode = mode;
4441 }
4442 Ok(r)
4443 })
4444 }
4445 Starter::Start => {
4446 if let Some(branch) = &review_branch {
4447 tracing::warn!(
4448 "conductor chose review for task {} but branch `{branch}` no longer \
4449 exists; requeuing as a fresh competition instead",
4450 task.short()
4451 );
4452 }
4453 let instruction = prepare_instruction(&starter, None, task)
4454 .unwrap_or_else(|| task.instruction.clone());
4455 Runner::start_naming(
4456 &repo,
4457 instruction,
4458 &task.title,
4459 config,
4460 crate::run::Origin::queue(&task.id),
4461 )
4462 .await
4463 .map(|mut r| {
4464 r.state.attachments = attachments.clone();
4465 r.state.sources = task.sources.clone();
4466 r
4467 })
4468 }
4469 }})).await;
4470 let mut runner = match started {
4471 Ok(r) => r,
4472 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4473 task.last_error = Some(e.to_string());
4474 if stop.parking() || interrupt_pause.parked() {
4475 let attempts = task.attempts;
4476 task.park(e.to_string());
4477 task.attempts = attempts;
4479 } else {
4480 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4481 }
4482 record(queue, task);
4483 return Vec::new();
4484 }
4485 Err(e) if task.pr_recovery.is_some() => {
4489 task.startup_notice_cause("startup-public-head");
4490 task.attempts += 1;
4491 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4492 task.link_run(&partial.run);
4493 task.record_pass(&partial.run, 0, task.attempts, None);
4494 }
4495 task.last_error = Some(format!("{start_failed}{e:#}"));
4496 task.hold_unrequested_retry(Some(format!(
4497 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4498 )));
4499 record(queue, task);
4500 return Vec::new();
4501 }
4502 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4503 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4504 Some(r) => (p.handover_refused)(r),
4505 None => format!("{e:#}"),
4506 };
4507 task.startup_notice_cause("startup-handover");
4508 let reason = format!("{start_failed}{detail}");
4509 task.last_error = Some(reason.clone());
4510 let branch = match &starter {
4513 Starter::Review(branch) => Some(branch.clone()),
4514 _ => None,
4515 };
4516 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4520 record(queue, task);
4521 tracing::warn!(
4522 "holding {} for a branch it cannot take over: {e:#}",
4523 task.short()
4524 );
4525 return Vec::new();
4526 }
4527 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4532 let d = e
4533 .downcast_ref::<crate::reconcile::Diverged>()
4534 .expect("checked by the guard");
4535 let mut q = ask::Question::new(
4536 task.id.clone(),
4537 crate::reconcile::NODE.to_owned(),
4538 crate::reconcile::SEAT.to_owned(),
4539 d.summary(),
4540 d.detail(),
4541 d.choices(),
4542 );
4543 match Questions::open().put(&mut q) {
4544 Ok(()) => {
4545 task.last_error = Some(format!("{e:#}"));
4546 task.review_branch = Some(d.branch.clone());
4549 task.review_instruction = Some(task.instruction.clone());
4550 task.block(vec![q.id.clone()], Some(d.summary()));
4551 }
4552 Err(put) => {
4553 tracing::warn!("could not file the divergence question: {put:#}");
4554 task.attempts += 1;
4555 task.fail_startup(
4556 "startup-question-store",
4557 format!("{start_failed}{e:#}"),
4558 opts.max_attempts,
4559 );
4560 }
4561 }
4562 record(queue, task);
4563 return Vec::new();
4564 }
4565 Err(e) => {
4566 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4572 && let Starter::Review(branch) = &starter
4573 {
4574 task.review_branch = Some(branch.clone());
4575 task.review_instruction = Some(task.instruction.clone());
4576 task.last_error = Some(format!("{start_failed}{e:#}"));
4577 task.status = crate::queue::TaskStatus::Failed;
4578 record(queue, task);
4579 return Vec::new();
4580 }
4581 task.attempts += 1;
4582 task.last_error = Some(format!("{start_failed}{e:#}"));
4583 task.startup_notice_cause("startup-source");
4584 task.hold_unrequested_retry(Some(format!(
4585 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4586 )));
4587 record(queue, task);
4588 return Vec::new();
4589 }
4590 };
4591 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4592 runner.state.recovery_config = Some(config_evidence);
4593 }
4594 if let Some(replacement) = &task.replacement {
4595 if matches!(starter, Starter::Start) {
4596 runner.state.instruction.push_str(&format!(
4597 "\n\n# Replacement review context\n{}",
4598 replacement.context
4599 ));
4600 }
4601 runner.state.replaces = Some(replacement.clone());
4602 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4603 if let Ok(mut old) = RunState::load(&replacement.run)
4604 && let Some(d) = &mut old.merge_decision
4605 {
4606 d.replacement_run = Some(runner.state.id.clone());
4607 old.event(
4608 "replacement",
4609 format!(
4610 "replacement run {} owns the next attempt; old PR remains closed",
4611 runner.state.id
4612 ),
4613 );
4614 if let Err(e) = old.save() {
4615 task.hold_manual(Some(format!(
4616 "Could not link replacement history: {e}; no agents dispatched"
4617 )));
4618 record(queue, task);
4619 return Vec::new();
4620 }
4621 }
4622 }
4623 runner.state.task_instruction = Some(task.instruction.clone());
4624 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4627 if runner.state.origin_chat.is_none() {
4628 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4629 }
4630 runner.on_pause(stop.pause());
4632 runner.watch_interrupt(interrupt_pause);
4636
4637 let run = runner.state.id.clone();
4640 task.start(run.clone());
4641 record(queue, task);
4642 lock(status).current.push(Current {
4643 task: task.id.clone(),
4644 run,
4645 });
4646
4647 let quota_before = runner.state.quota.clone();
4650 runner.state.next_pass_task_entry = true;
4651 let result = runner.execute().await;
4652 finish_attempt(
4653 opts.max_attempts,
4654 queue,
4655 task,
4656 &runner.state,
4657 "a_before,
4658 result,
4659 )
4660}
4661
4662pub fn finish_attempt(
4667 max_attempts: usize,
4668 queue: &Queue,
4669 task: &mut Task,
4670 state: &RunState,
4671 quota_before: &[QuotaLoss],
4672 result: Result<()>,
4673) -> Vec<QuotaLoss> {
4674 let detail = match result {
4675 Ok(()) => describe(state),
4676 Err(e) => format!("{e:#}"),
4677 };
4678 let fresh = losses_this_attempt(quota_before, &state.quota);
4679 let verdict = Verdict {
4680 status: state.status,
4681 left_pr: state.pr.is_some(),
4684 quota_hit: !fresh.is_empty(),
4690 parked: state.parked,
4694 no_viable_candidates: state.viable().is_empty(),
4697 };
4698 if let Some(replacement) = &state.replaces
4699 && let Some(pr) = &state.pr
4700 && let Ok(mut old) = RunState::load(&replacement.run)
4701 && let Some(d) = &mut old.merge_decision
4702 {
4703 d.replacement_pr = Some(pr.url.clone());
4704 let _ = old.save();
4705 }
4706 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4707 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4708 task.hold_unrequested_retry(Some(format!(
4709 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4710 task.short()
4711 )));
4712 }
4713 if task.status == TaskStatus::Done {
4714 supersede_prior_runs(task, &crate::run::home());
4715 }
4716 record(queue, task);
4717 tracing::info!(
4718 "task {} is {} after run {} ({})",
4719 task.short(),
4720 task.status.as_str(),
4721 state.short(),
4722 label(state.status)
4723 );
4724 fresh
4725}
4726
4727pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4731 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4734 && !awaiting_resume_with(task, RunState::load)
4735 && withheld_text_wait(task).is_none()
4739 && !(task.fresh_start && task.replacement.is_some())
4740 {
4741 let why = task.last_error.clone().map_or_else(
4742 || "the run did not finish".to_owned(),
4743 |e| format!("the run did not finish: {e}"),
4744 );
4745 task.hold_unrequested_retry(Some(format!(
4746 "{why}. It was started by hand, so it is not retried \
4747 automatically; `magi task release` retries it."
4748 )));
4749 record(queue, task);
4750 }
4751}
4752
4753#[must_use]
4757pub fn loop_would_resume(run: &str) -> bool {
4758 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4759}
4760
4761#[must_use]
4771pub fn foreign_loop(
4772 reading: Option<&Reading>,
4773 now: Timestamp,
4774 own_pid: u32,
4775) -> Option<Option<u32>> {
4776 let reading = reading.filter(|r| r.running(now))?;
4777 match reading.pid {
4778 Some(pid) if pid == own_pid => None,
4779 pid => Some(pid),
4780 }
4781}
4782
4783fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4784 reading.worker_protocol != Some(1)
4785 && (foreign_loop(Some(reading), now, own_pid).is_some()
4786 || reading
4787 .pid
4788 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4789}
4790
4791pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4802 run_claimed_until(opts, queue, task, &Stop::new()).await;
4803}
4804
4805async fn run_claimed_until(
4806 opts: &Opts,
4807 queue: &Queue,
4808 task: &mut Task,
4809 stop: &Stop,
4810) -> Vec<QuotaLoss> {
4811 let status = Arc::new(Mutex::new(Status::new()));
4812 let mut quota = Box::pin(attempt(
4815 opts,
4816 queue,
4817 &status,
4818 stop,
4819 crate::graph::Pause::new(),
4820 task,
4821 ))
4822 .await;
4823 let mut resumes = 0;
4832 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
4833 let Some(timeout) = withheld_text_wait(task) else {
4834 break;
4835 };
4836 eprintln!(
4837 "waiting for the owner's answer on a withheld pull request text \
4838 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4839 );
4840 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4843 let ready = loop {
4844 match land_resume_state(task) {
4845 LandResume::StillWaiting
4846 if !stop.parking() && std::time::Instant::now() < limit =>
4847 {
4848 tokio::time::sleep(WITHHELD_POLL).await;
4849 }
4850 LandResume::Ready => break true,
4851 _ => break false,
4852 }
4853 };
4854 if !ready || stop.parking() {
4855 break;
4856 }
4857 resumes += 1;
4858 quota.extend(
4859 Box::pin(attempt(
4860 opts,
4861 queue,
4862 &status,
4863 stop,
4864 crate::graph::Pause::new(),
4865 task,
4866 ))
4867 .await,
4868 );
4869 }
4870 hold_if_runnable(queue, task);
4871 quota
4872}
4873
4874const MAX_WITHHELD_RESUMES: u32 = 3;
4877
4878const WITHHELD_POLL: Duration = Duration::from_secs(15);
4880
4881fn withheld_text_wait(task: &Task) -> Option<u64> {
4884 let id = task.runs.last()?;
4885 let s = RunState::load(id).ok()?;
4886 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4887 .then_some(s.config.graph.answer_timeout)
4888}
4889
4890fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4900 after
4901 .iter()
4902 .filter(|q| !before.contains(q))
4903 .cloned()
4904 .collect()
4905}
4906
4907fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4910 if quota.is_empty() {
4911 return None;
4912 }
4913 let with_hint = quota.iter().find(|q| q.reset.is_some());
4914 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4915 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4916 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4917 Some(
4918 now.checked_add(jiff::SignedDuration::from_secs(secs))
4919 .unwrap_or(Timestamp::MAX),
4920 )
4921}
4922
4923fn implementers_match(task: &Task, state: &RunState) -> bool {
4925 task.replacement.as_ref().map_or_else(
4926 || {
4927 task.implementers()
4928 .is_none_or(|n| n == state.config.graph.implementers)
4929 },
4930 |replacement| {
4931 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4932 && state.config.graph.implementers == 1
4933 },
4934 )
4935}
4936
4937fn starter_with_implementers(
4939 task: &Task,
4940 previous: Option<&RunState>,
4941 proposed: Starter,
4942) -> Starter {
4943 if previous.is_some_and(|s| !implementers_match(task, s)) {
4944 task.review_of
4945 .as_ref()
4946 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4947 } else {
4948 proposed
4949 }
4950}
4951
4952fn apply_task_implementers(config: &mut Config, task: &Task) {
4954 if let Some(n) = task.implementers() {
4955 config.graph.implementers = n;
4956 }
4957}
4958
4959#[cfg(test)]
4964fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4965 let explicit = match &task.overrides {
4966 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4967 _ => opts.config.as_deref(),
4968 };
4969 let (mut config, _) = Config::discover(repo, explicit)?;
4970 apply_options(&mut config, opts, task)?;
4971 Ok(config)
4972}
4973
4974fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
4975 if task
4976 .overrides
4977 .as_ref()
4978 .is_none_or(|o| o.inherit_daemon_options)
4979 && let Some(mode) = opts.merge.as_deref()
4980 {
4981 config.merge.mode = merge_mode(mode)?;
4982 }
4983 if let Some(o) = &task.overrides {
4984 o.apply(config);
4985 }
4986 Ok(())
4987}
4988
4989fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
4991 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
4992 if let Some(mode) = &opts.merge {
4993 config.merge.mode = merge_mode(mode)?;
4994 }
4995 Ok(config)
4996}
4997
4998async fn maybe_prune_cache_between_runs(
5003 repo: &Path,
5004 opts: &Opts,
5005 home: &Path,
5006 stop: &Stop,
5007 last_checked: &mut Option<Timestamp>,
5008 now: Timestamp,
5009) {
5010 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5011 return;
5012 }
5013 *last_checked = Some(now);
5014 let cfg = match prepare(repo, opts) {
5015 Ok(cfg) => cfg,
5016 Err(e) => {
5017 tracing::warn!("cache check: no config: {e:#}");
5018 return;
5019 }
5020 };
5021 match clean::prune_cache_if_over_limit(&cfg, home) {
5022 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5023 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5024 pruned.files,
5025 pruned.freed
5026 ),
5027 Ok(_) => {}
5028 Err(e) => {
5029 tracing::warn!("housekeep: prune cache: {e:#}");
5030 notices::raise_in(
5031 home,
5032 Notice::warn(
5033 "housekeep:cache",
5034 "Pruning the shared build cache failed; disk usage may keep growing.",
5035 ),
5036 );
5037 }
5038 }
5039}
5040
5041fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5045 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5046}
5047
5048async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5067 let cfg = match prepare(repo, opts) {
5068 Ok(cfg) => cfg,
5069 Err(e) => {
5070 tracing::warn!("housekeep: no config: {e:#}");
5071 return;
5072 }
5073 };
5074 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5083 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5084 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5089 let mut extra = Vec::new();
5090 if out.unreadable > 0 {
5091 extra.push(format!("{} unreadable", out.unreadable));
5092 }
5093 if out.orphaned_worktrees > 0 {
5094 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5095 }
5096 let detail = if extra.is_empty() {
5097 String::new()
5098 } else {
5099 format!(" ({})", extra.join(", "))
5100 };
5101 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5102 }
5103 if out.external_merges_recorded > 0 {
5104 tracing::info!(
5105 "housekeep: recorded {} run(s) as merged externally",
5106 out.external_merges_recorded
5107 );
5108 }
5109 if out.stale_pr_states_repaired > 0 {
5110 tracing::info!(
5111 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5112 out.stale_pr_states_repaired
5113 );
5114 }
5115 if out.cache_files > 0 {
5116 tracing::info!(
5117 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5118 out.cache_files,
5119 out.cache_freed
5120 );
5121 }
5122 if out.questions_abandoned > 0 {
5123 tracing::info!(
5124 "housekeep: abandoned {} question(s) left open by a finished run",
5125 out.questions_abandoned
5126 );
5127 }
5128}
5129
5130async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5139 let questions = Questions::at(home.join("questions"));
5140 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5141 if report.is_empty() {
5142 return;
5143 }
5144 if !report.quarantined.is_empty() {
5145 tracing::info!(
5146 "triage: held {} blocked task(s) whose blocked-on task or \
5147 question no longer exists: {}",
5148 report.quarantined.len(),
5149 report.quarantined.join(", ")
5150 );
5151 }
5152 if !report.resumed.is_empty() {
5153 tracing::info!(
5154 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5155 report.resumed.len(),
5156 report.resumed.join(", ")
5157 );
5158 }
5159 if !report.asked.is_empty() {
5160 tracing::info!(
5161 "triage: asked about {} held task(s): {}",
5162 report.asked.len(),
5163 report.asked.join(", ")
5164 );
5165 }
5166 if !report.answered.is_empty() {
5167 tracing::info!(
5168 "triage: applied {} operator answer(s): {}",
5169 report.answered.len(),
5170 report.answered.join(", ")
5171 );
5172 }
5173}
5174
5175pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5182 disk_gate_with(repo, config, crate::disk::free_bytes)
5183}
5184
5185fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5189 repo: &Path,
5190 config: &Config,
5191 free_bytes: F,
5192) -> Option<String> {
5193 let min = config.disk.min_free_bytes;
5194 if min == 0 {
5195 return None;
5196 }
5197 let mut paths = vec![repo.to_owned()];
5198 if let Some(cache) = config.cache_dir() {
5199 paths.push(cache);
5200 }
5201 if let Some(root) = &config.graph.worktree_root {
5202 paths.push(root.clone());
5203 }
5204 for path in paths {
5205 let probe = if path == repo {
5206 path.as_path()
5207 } else {
5208 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5209 };
5210 let reason = match free_bytes(probe) {
5211 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5212 Err(e) => Some(crate::disk::unmeasured_in(
5213 probe,
5214 &e,
5215 &config.graph.language,
5216 )),
5217 };
5218 if reason.is_some() {
5219 return reason;
5220 }
5221 }
5222 None
5223}
5224
5225const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5232
5233const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5237
5238fn quota_wait(
5247 reset_at: Option<Timestamp>,
5248 now: Timestamp,
5249 fallback: Duration,
5250 cap: Duration,
5251) -> Duration {
5252 match reset_at {
5253 Some(at) if at > now => {
5254 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5255 Duration::from_secs(secs).min(cap)
5256 }
5257 _ => fallback,
5258 }
5259}
5260
5261fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5275 parse_reset_hint_zoned(text, now)
5276 .or_else(|| parse_reset_hint_dated(text))
5277 .or_else(|| parse_reset_hint_relative(text, recorded))
5278}
5279
5280fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5284 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5285 let mut rest = rest.trim();
5286 if rest.is_empty() {
5287 return None;
5288 }
5289 let mut total: i64 = 0;
5290 let mut matched = false;
5291 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5292 if let Some((digits, tail)) = rest.split_once(unit)
5293 && !digits.is_empty()
5294 && digits.bytes().all(|b| b.is_ascii_digit())
5295 {
5296 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5297 rest = tail;
5298 matched = true;
5299 }
5300 }
5301 if !rest.is_empty() || !matched {
5302 return None;
5303 }
5304 recorded
5305 .checked_add(jiff::SignedDuration::from_secs(total))
5306 .ok()
5307}
5308
5309fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5313 let clock = clock.trim().to_lowercase();
5314 let (digits, pm) = clock
5315 .strip_suffix("am")
5316 .map(|d| (d, false))
5317 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5318 let (h, m) = digits.trim().split_once(':')?;
5319 let mut hour: i8 = h.trim().parse().ok()?;
5320 let minute: i8 = m.trim().parse().ok()?;
5321 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5322 return None;
5323 }
5324 if pm && hour != 12 {
5325 hour += 12;
5326 } else if !pm && hour == 12 {
5327 hour = 0;
5328 }
5329 Some((hour, minute))
5330}
5331
5332fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5337 let open = text.find('(')?;
5338 let close = text.rfind(')')?;
5339 if close <= open {
5340 return None;
5341 }
5342 let zone = text[open + 1..close].trim();
5343 let (hour, minute) = parse_12h_clock(&text[..open])?;
5344 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5345 let candidate = now
5346 .to_zoned(tz)
5347 .with()
5348 .hour(hour)
5349 .minute(minute)
5350 .second(0)
5351 .millisecond(0)
5352 .microsecond(0)
5353 .nanosecond(0)
5354 .build()
5355 .ok()?;
5356 let mut at = candidate.timestamp();
5357 if at <= now {
5358 at += jiff::SignedDuration::from_hours(24);
5359 }
5360 Some(at)
5361}
5362
5363fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5372 let words: Vec<&str> = text.split_whitespace().collect();
5373 if words.len() < 5 {
5374 return None;
5375 }
5376 (0..=words.len() - 5)
5377 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5378}
5379
5380fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5386 if trailing.is_some_and(|next| next.starts_with('(')) {
5387 return None;
5388 }
5389 let month = month_number(window[0])?;
5390 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5391 let day_digits = ["st", "nd", "rd", "th"]
5392 .iter()
5393 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5394 let day: i8 = day_digits.parse().ok()?;
5395 let year_token = window[2];
5396 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5397 return None;
5398 }
5399 let year: i16 = year_token.parse().ok()?;
5400 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5404 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5405 let date = jiff::civil::Date::new(year, month, day).ok()?;
5406 let candidate = date
5407 .at(hour, minute, 0, 0)
5408 .to_zoned(jiff::tz::TimeZone::UTC)
5409 .ok()?;
5410 Some(candidate.timestamp())
5411}
5412
5413fn month_number(name: &str) -> Option<i8> {
5416 const NAMES: [&str; 12] = [
5417 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5418 ];
5419 let lower = name.to_lowercase();
5420 NAMES
5421 .iter()
5422 .position(|n| *n == lower.as_str())
5423 .map(|i| i as i8 + 1)
5424}
5425
5426#[cfg(test)]
5428fn exhausted_review_budget(state: &RunState) -> bool {
5429 !crate::handover::can_resume(state)
5432 && state.status == RunStatus::Blocked
5433 && !state.released()
5434 && state.retired_to.is_none()
5435}
5436
5437fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5474 unfinished_run_with(runs, short, RunState::load)
5475}
5476
5477pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5480where
5481 F: FnMut(&str) -> Result<RunState>,
5482{
5483 let mut id = runs.last()?.clone();
5484 let mut visited = std::collections::BTreeSet::new();
5485 let mut repo = None;
5486 loop {
5487 if !runs.contains(&id) || !visited.insert(id.clone()) {
5488 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5489 return None;
5490 }
5491 let state = match load(&id) {
5492 Ok(state) if state.id == id => state,
5493 Ok(_) => return None,
5494 Err(e) => {
5495 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5496 return None;
5497 }
5498 };
5499 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5500 return None;
5501 }
5502 repo = Some(state.repo.clone());
5503 if let Some(authority) = state.retired_to {
5504 id = authority;
5505 continue;
5506 }
5507 return (crate::handover::can_resume(&state)
5508 && state.liveness(false) != crate::run::Liveness::Live)
5509 .then_some(id);
5510 }
5511}
5512
5513fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5521where
5522 F: FnOnce(&str) -> Result<RunState>,
5523{
5524 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5526 || task.fresh_start
5527 || task.review_branch.is_some()
5528 {
5529 return false;
5530 }
5531 let Some(id) = task.runs.last() else {
5532 return false;
5533 };
5534 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5535}
5536
5537#[derive(Debug, Clone, PartialEq, Eq)]
5540enum Starter {
5541 Review(String),
5544 Resume(String),
5546 Start,
5548}
5549
5550fn retained_attempt_hold(
5554 task: &Task,
5555 starter: &Starter,
5556 limit: usize,
5557 mut load: impl FnMut(&str) -> Result<RunState>,
5558) -> Option<String> {
5559 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5560 return None;
5561 }
5562 let mut retained = 0;
5563 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5564 let state = match load(id) {
5565 Ok(state) if state.id == *id => state,
5566 _ => {
5567 return Some(format!(
5568 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5569 ));
5570 }
5571 };
5572 if state.retired_to.is_some() || state.released() {
5573 continue;
5574 }
5575 retained += 1;
5578 }
5579 (retained >= limit).then(|| format!(
5580 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5581 ))
5582}
5583
5584fn selected_starter(
5586 selected: &crate::pr_recovery::Selection,
5587 previous: Option<&RunState>,
5588 unfinished: Option<&str>,
5589) -> Starter {
5590 if let Some(id) = unfinished
5591 && previous.is_some_and(|r| {
5592 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5593 })
5594 {
5595 Starter::Resume(id.to_owned())
5596 } else {
5597 Starter::Review(selected.branch.clone())
5598 }
5599}
5600
5601fn take_divergence_answer(
5605 branch: &str,
5606 remote: &str,
5607 task: &mut Task,
5608) -> Option<crate::reconcile::Choice> {
5609 let summary = crate::reconcile::summary_for(branch, remote);
5610 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5611 (a.question == summary)
5612 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5613 .flatten()
5614 .map(|c| (i, c))
5615 })?;
5616 task.answers.remove(idx);
5617 Some(choice)
5618}
5619
5620fn choose_starter(
5632 review_branch: Option<&str>,
5633 branch_exists: bool,
5634 unfinished: Option<&str>,
5635) -> Starter {
5636 match review_branch {
5637 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5638 Some(_) => Starter::Start,
5639 None => match unfinished {
5640 Some(id) => Starter::Resume(id.to_owned()),
5641 None => Starter::Start,
5642 },
5643 }
5644}
5645
5646fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5648 let branch = task.review_branch.take();
5649 let instruction = task.review_instruction.take();
5650 branch.filter(|branch| match &instruction {
5651 Some(authored) => *authored == task.instruction,
5652 None => previous.map_or_else(
5655 || task.review_of.as_ref() == Some(branch),
5656 |run| compatible_run_instruction(run, task),
5657 ),
5658 })
5659}
5660
5661fn may_resume_run(run: &RunState, task: &Task) -> bool {
5664 task.resume_override
5665 .as_ref()
5666 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5667 || compatible_run_instruction(run, task)
5668}
5669
5670pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5672 if let Some(authored) = &run.task_instruction {
5673 return compatible_instruction(authored, task);
5674 }
5675 if compatible_instruction(&run.instruction, task) {
5676 return true;
5677 }
5678 if run.reviewed_commits.is_some() {
5681 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5682 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5683 }
5684 if let Some(branch) = &task.review_of {
5685 let canonical = format!(
5686 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5687 );
5688 let generated = format!(
5689 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5690 );
5691 return task.instruction == canonical && run.instruction.starts_with(&generated);
5692 }
5693 }
5694 false
5695}
5696
5697pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5699 strip_answers_block(old, task) == task.instruction
5700}
5701
5702fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5705 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5706 return fallback.to_path_buf();
5707 }
5708 task.repo.clone()
5709}
5710
5711const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5715
5716fn answers_block(task: &Task, count: usize) -> String {
5718 let mut s = ANSWERS_HEADER.to_owned();
5719 for a in &task.answers[..count] {
5720 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5721 }
5722 s
5723}
5724
5725fn resumption_block(r: &crate::queue::Resumption) -> String {
5728 format!(
5729 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5730 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5731 r.generation, r.at, r.instruction, r.answers_through,
5732 )
5733}
5734
5735fn append_answers(base: &str, task: &Task) -> String {
5737 let mut s = base.to_owned();
5738 if !task.answers.is_empty() {
5739 s.push_str(&answers_block(task, task.answers.len()));
5740 }
5741 if let Some(r) = &task.resumption {
5742 s.push_str(&resumption_block(r));
5743 }
5744 s
5745}
5746
5747fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5751 let mut instruction = instruction;
5752 for approval in task
5753 .resumption
5754 .iter()
5755 .chain(task.resumption_history.iter().rev())
5756 {
5757 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5758 instruction = base;
5759 break;
5760 }
5761 }
5762 for count in (1..=task.answers.len()).rev() {
5763 let block = answers_block(task, count);
5764 if let Some(base) = instruction.strip_suffix(&block) {
5765 return base;
5766 }
5767 }
5768 instruction
5769}
5770
5771fn instruction_for(task: &Task) -> String {
5779 append_answers(&task.instruction, task)
5780}
5781
5782fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5794 append_answers(strip_answers_block(old_instruction, task), task)
5795}
5796
5797fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5799 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5800 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5801 if let Some(cause) = task.hold_notice_cause.as_mut() {
5802 cause.target = failure.map(|f| f.target.clone());
5803 }
5804}
5805
5806#[derive(Debug, thiserror::Error)]
5807#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5808struct MissingAttachment {
5809 name: String,
5810 path: PathBuf,
5811}
5812
5813fn task_attachments(
5815 queue: &Queue,
5816 task: &Task,
5817) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5818 let paths = queue.attachment_paths(task);
5819 for (name, path) in task.attachments.iter().zip(&paths) {
5820 if !path.is_file() {
5821 return Err(MissingAttachment {
5822 name: name.clone(),
5823 path: path.clone(),
5824 });
5825 }
5826 }
5827 Ok(paths)
5828}
5829
5830fn prepare_instruction(
5841 starter: &Starter,
5842 old_instruction: Option<&str>,
5843 task: &Task,
5844) -> Option<String> {
5845 match starter {
5846 Starter::Start => Some(instruction_for(task)),
5847 Starter::Resume(_) => Some(resumed_instruction(
5848 old_instruction.expect("a resumed run always has a prior instruction"),
5849 task,
5850 )),
5851 Starter::Review(_) => None,
5852 }
5853}
5854
5855fn record(queue: &Queue, task: &mut Task) {
5859 if let Err(e) = queue.put(task) {
5860 tracing::error!("could not record task {}: {e:#}", task.short());
5861 let notice = Notice::error(
5862 "loop:record",
5863 "The loop could not save a task's state; check the disk.",
5864 )
5865 .link(Link::Task {
5866 id: task.id.clone(),
5867 });
5868 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5870 Some(home) => notices::raise_in(home, notice),
5871 None => notices::raise(notice),
5872 }
5873 }
5874}
5875
5876fn runnable(queue: &Queue) -> Vec<Task> {
5882 let mut tasks: Vec<Task> = queue
5883 .list()
5884 .into_iter()
5885 .filter(|t| t.status.runnable())
5886 .collect();
5887 tasks.sort_unstable_by(crate::queue::dispatch_order);
5888 tasks
5889}
5890
5891fn describe(state: &RunState) -> String {
5905 let p = phrases(&state.config.graph.language);
5906 let mut detail = if state.status == RunStatus::Stalled {
5907 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5908 seats.sort_unstable();
5909 seats.dedup();
5910 if seats.is_empty() {
5911 p.quorum_lost.to_owned()
5912 } else {
5913 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5914 }
5915 } else {
5916 format!("{}{}", p.run_ended, state.status.display_label())
5917 };
5918 if let Some(last) = state.events.last() {
5919 let unanswered = state
5923 .reviews
5924 .last()
5925 .filter(|r| {
5926 state.status == RunStatus::Blocked
5927 && last.node == "review"
5928 && r.incomplete()
5929 && r.blocking == 0
5930 && r.round == state.config.graph.review_rounds
5931 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
5932 })
5933 .map(|r| (r.expected - r.answered, r.round));
5934 match unanswered {
5935 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
5936 detail.push_str(&format!(
5937 " ({}: {})",
5938 last.node,
5939 (p.reviewers_never_answered)(missing, rounds)
5940 ));
5941 }
5942 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
5943 }
5944 }
5945 detail.push_str(&format!(" [run {}]", state.id));
5946 detail
5947}
5948
5949const DIAGNOSTIC_MAX: usize = 4_000;
5955
5956const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
5961
5962fn diagnostic(state: &RunState) -> Option<String> {
5976 let mut parts: Vec<String> = Vec::new();
5977
5978 for o in state.gate.iter().filter(|o| !o.ok()) {
5980 parts.push(format!(
5981 "gate `{}` failed ({}):\n{}",
5982 o.command,
5983 notices::exit_status(o.code),
5984 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
5985 ));
5986 }
5987
5988 if let Some(last) = state
5991 .events
5992 .iter()
5993 .rev()
5994 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
5995 {
5996 parts.push(last.message.clone());
5997 }
5998
5999 if state.viable().is_empty() {
6006 for c in &state.candidates {
6007 if let Some(evidence) = &c.verified_noop {
6008 parts.push(format!(
6009 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6010 c.label
6011 ));
6012 } else if !c.summary.trim().is_empty() {
6013 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6014 } else if let Some(why) = &c.failed {
6015 parts.push(format!("candidate {}: {why}", c.label));
6016 }
6017 }
6018 }
6019
6020 if parts.is_empty() {
6021 return None;
6022 }
6023 Some(crate::run::tail(
6028 &parts.join("\n\n"),
6029 DIAGNOSTIC_MAX.saturating_sub(100),
6030 ))
6031}
6032
6033fn label(status: RunStatus) -> &'static str {
6041 status.as_str()
6042}
6043
6044pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6046 match mode {
6047 "none" => Ok(MergeMode::None),
6048 "local" => Ok(MergeMode::Local),
6049 "pr" => Ok(MergeMode::Pr),
6050 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6051 }
6052}
6053
6054fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6059 mutex
6060 .lock()
6061 .unwrap_or_else(std::sync::PoisonError::into_inner)
6062}
6063
6064#[cfg(test)]
6065mod tests {
6066 use super::*;
6067 use crate::queue::{Source, TaskStatus};
6068 use crate::run::{Candidate, CommandOutcome};
6069 use pretty_assertions::assert_eq;
6070
6071 fn task() -> Task {
6072 Task::new(
6073 "add retries".to_owned(),
6074 "add retries".to_owned(),
6075 PathBuf::from("/repo"),
6076 Source::Human,
6077 )
6078 }
6079
6080 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6081 let job = crate::worker::Job {
6082 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6083 sequence: 1,
6084 kind: "task".into(),
6085 key: format!("task:{}", request.task),
6086 payload: serde_json::to_value(request).unwrap(),
6087 generation: "a".repeat(64),
6088 version: None,
6089 status: crate::worker::Status::Starting,
6090 result: None,
6091 pid: None,
6092 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6093 finished_at: None,
6094 };
6095 let dir = home.join("workers/jobs").join(&job.id);
6096 std::fs::create_dir_all(&dir).unwrap();
6097 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6098 job
6099 }
6100
6101 #[test]
6102 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6103 let home = tempfile::tempdir().unwrap();
6104 let request = TaskWorker {
6105 opts: Opts::default(),
6106 queue: home.path().join("queue"),
6107 task: "startup".into(),
6108 lane: PermitKind::Ordinary,
6109 standalone: false,
6110 };
6111 let job = starting_task_reservation(home.path(), &request);
6112 task_worker_spawn_error(
6113 home.path(),
6114 &request,
6115 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6116 )
6117 .expect_err("our own reservation must not turn a failed spawn into a resync");
6118 let active = crate::worker::active(home.path()).unwrap();
6119 assert_eq!(active.len(), 1);
6120 assert_eq!(active[0].id, job.id);
6121 assert_eq!(active[0].status, crate::worker::Status::Starting);
6122 assert!(
6123 crate::worker::control(home.path(), &job.key)
6124 .unwrap()
6125 .is_none()
6126 );
6127 let notices = notices::Notices::at(home.path().join("notifications")).list();
6128 assert_eq!(notices.len(), 1);
6129 assert_eq!(notices[0].key, "loop:worker-startup");
6130 assert!(!notices[0].message.contains("private-error-detail"));
6131 }
6132
6133 #[test]
6134 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6135 let home = tempfile::tempdir().unwrap();
6136 let mut request = TaskWorker {
6137 opts: Opts::default(),
6138 queue: home.path().join("queue"),
6139 task: "handoff".into(),
6140 lane: PermitKind::Ordinary,
6141 standalone: true,
6142 };
6143 let job = starting_task_reservation(home.path(), &request);
6144 request.standalone = false;
6145 task_worker_spawn_error(
6146 home.path(),
6147 &request,
6148 anyhow::anyhow!("operation key is already reserved for a different request"),
6149 )
6150 .expect("a different standalone request is the handoff race, not our startup failure");
6151 let active = crate::worker::active(home.path()).unwrap();
6152 assert_eq!(active.len(), 1);
6153 assert_eq!(active[0].id, job.id);
6154 assert_eq!(active[0].payload, job.payload);
6155 assert!(
6156 notices::Notices::at(home.path().join("notifications"))
6157 .list()
6158 .is_empty()
6159 );
6160 }
6161
6162 #[test]
6163 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6164 let dir = tempfile::tempdir().unwrap();
6165 let queue = Queue::at(dir.path().join("queue"));
6166 let mut task = task();
6167 task.status = TaskStatus::Running;
6168 task.attempts = 1;
6169 queue.put(&mut task).unwrap();
6170 let live = std::collections::BTreeSet::from([task.id.clone()]);
6171 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6172 drop(queue.claim(&task.id).unwrap());
6173 let saved = queue.get(&task.id).unwrap();
6174 assert_eq!(saved.status, TaskStatus::Running);
6175 assert_eq!(saved.attempts, 1);
6176 assert!(saved.last_error.is_none());
6177 }
6178
6179 #[test]
6180 fn standalone_workers_do_not_consume_scheduler_lanes() {
6181 let request = |task: &str, lane, standalone| TaskWorker {
6182 opts: Opts::default(),
6183 queue: PathBuf::from("/queue"),
6184 task: task.to_owned(),
6185 lane,
6186 standalone,
6187 };
6188 let mut jobs = vec![
6189 request("ordinary", PermitKind::Ordinary, false),
6190 request("standalone-urgent", PermitKind::Urgent, true),
6191 request("standalone-ordinary", PermitKind::Ordinary, true),
6192 request("decision", PermitKind::None, false),
6193 ];
6194 assert_eq!(available_slots(&jobs, 2), (1, 1));
6195 jobs.push(request("urgent", PermitKind::Urgent, false));
6196 assert_eq!(available_slots(&jobs, 2), (1, 0));
6197 assert_eq!(available_slots(&jobs, 0), (0, 0));
6198 }
6199
6200 #[test]
6201 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6202 let now = Timestamp::now();
6203 let mut reading = Reading {
6204 pid: Some(std::process::id()),
6205 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6206 ..Reading::default()
6207 };
6208 assert!(!reading.running(now));
6209 assert!(legacy_scheduler_running(&reading, now, 0));
6210 reading.worker_protocol = Some(1);
6211 assert!(!legacy_scheduler_running(&reading, now, 0));
6212 }
6213
6214 #[test]
6215 fn selected_public_head_overrides_historical_resume_and_pin() {
6216 let mut t = task();
6217 let mut old = run_state(RunStatus::Blocked);
6218 let selection = crate::pr_recovery::Selection {
6219 url: "url".into(),
6220 branch: format!("magi/task-{}", t.id),
6221 head: "a".repeat(40),
6222 source_head: None,
6223 source_run: None,
6224 source_proof: None,
6225 disposition: Some(crate::pr_recovery::Disposition::Review),
6226 instruction: t.instruction.clone(),
6227 approved_at: Timestamp::now(),
6228 };
6229 t.resume_override = Some(crate::queue::OperatorResume {
6230 pinned_run: Some(old.id.clone()),
6231 question_id: "choice".into(),
6232 at: Timestamp::now(),
6233 conductor_rehold: None,
6234 forced: false,
6235 });
6236 assert_eq!(
6237 selected_starter(&selection, Some(&old), Some(&old.id)),
6238 Starter::Review(selection.branch.clone())
6239 );
6240 old.pr_recovery = Some(selection.clone());
6241 assert_eq!(
6242 selected_starter(&selection, Some(&old), Some(&old.id)),
6243 Starter::Review(selection.branch.clone())
6244 );
6245 old.reviewed_commits = Some(vec!["implementation".into()]);
6246 assert_eq!(
6247 selected_starter(&selection, Some(&old), Some(&old.id)),
6248 Starter::Resume(old.id.clone())
6249 );
6250 let mut changed = selection.clone();
6251 changed.head = "b".repeat(40);
6252 assert_eq!(
6253 selected_starter(&changed, Some(&old), Some(&old.id)),
6254 Starter::Review(changed.branch)
6255 );
6256 }
6257
6258 #[test]
6259 fn candidate_transplant_failure_is_one_actionable_hold() {
6260 let dir = tempfile::tempdir().unwrap();
6261 let q = Queue::at(dir.path().join("queue"));
6262 let mut t = task();
6263 let mut state = run_state(RunStatus::Blocked);
6264 state.event("prep", "cannot apply referenced commit: conflict");
6265 state.prep_failure = Some("conflict".into());
6266 t.start(state.id.clone());
6267 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6268 assert_eq!(t.attempts, 1);
6269 assert_eq!(t.status, TaskStatus::Held);
6270 assert!(t.operator_held());
6271 assert!(!t.status.runnable());
6272 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6273 let first = t.chat_notice_key().unwrap();
6274 t.chat_notices.push(first.clone());
6275 t.release_operator("test", None);
6276 state.id = "another-unprepared-run".into();
6277 t.start(state.id.clone());
6278 finish_attempt(
6279 3,
6280 &q,
6281 &mut t,
6282 &state,
6283 &[],
6284 Err(anyhow::anyhow!("still conflicts")),
6285 );
6286 assert_eq!(
6287 t.chat_notice_key().as_ref(),
6288 Some(&first),
6289 "no work happened"
6290 );
6291 t.release_operator("test", None);
6292 state.prep_failure = None;
6293 state.status = RunStatus::Failed;
6294 state.begin_pass();
6295 t.start(state.id.clone());
6296 finish_attempt(
6297 1,
6298 &q,
6299 &mut t,
6300 &state,
6301 &[],
6302 Err(anyhow::anyhow!("new gate failure")),
6303 );
6304 let failure = t.chat_notice_key().unwrap();
6305 assert_ne!(failure, first, "approved recovery reached a real failure");
6306 t.release_operator("test", None);
6307 state.begin_pass();
6308 t.start(state.id.clone());
6309 finish_attempt(
6310 1,
6311 &q,
6312 &mut t,
6313 &state,
6314 &[],
6315 Err(anyhow::anyhow!("new failure on resume")),
6316 );
6317 assert_ne!(
6318 t.chat_notice_key().as_ref(),
6319 Some(&failure),
6320 "a new settled pass"
6321 );
6322 }
6323
6324 #[test]
6325 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6326 for source in ["cli", "web"] {
6327 let mut t = task();
6328 let mut run = run_state(RunStatus::Blocked);
6329 run.instruction = "Run A original criteria".into();
6330 run.task_instruction = Some(run.instruction.clone());
6331 t.start(run.id.clone());
6332 t.release_operator("choice", Some("resume-question"));
6333 t.resume_override = Some(crate::queue::OperatorResume {
6334 question_id: "resume-question".into(),
6335 at: Timestamp::now(),
6336 conductor_rehold: None,
6337 forced: true,
6338 pinned_run: Some(run.id.clone()),
6339 });
6340 t.hold_machine(Some("not enough free space to start a run".into()));
6341 let mut unchanged = t.clone();
6342 unchanged
6343 .edit("title changed".into(), unchanged.instruction.clone())
6344 .unwrap();
6345 unchanged.release_operator(source, None);
6346 assert_eq!(
6347 unchanged
6348 .resume_override
6349 .as_ref()
6350 .unwrap()
6351 .pinned_run
6352 .as_deref(),
6353 Some(run.id.as_str())
6354 );
6355 assert!(!compatible_run_instruction(&run, &unchanged));
6356 assert!(may_resume_run(&run, &unchanged));
6357
6358 t.edit(
6359 "approved recovery".into(),
6360 "Reuse measured PR head B; fix Clippy; review".into(),
6361 )
6362 .unwrap();
6363 assert!(t.resume_override.is_none());
6364 t.release_operator(source, None);
6365 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6366 assert!(reloaded.resume_override.is_none());
6367 assert!(!may_resume_run(&run, &reloaded));
6368 let unfinished =
6369 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6370 .filter(|_| may_resume_run(&run, &reloaded));
6371 assert_eq!(
6372 choose_starter(None, false, unfinished.as_deref()),
6373 Starter::Start
6374 );
6375 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6376 assert_eq!(
6377 reloaded.resumption_history[0].question_id.as_deref(),
6378 Some("resume-question")
6379 );
6380
6381 unchanged.instruction = "Reuse measured PR head B".into();
6384 unchanged.hold_machine(Some("disk hold".into()));
6385 unchanged.release_operator(source, None);
6386 assert!(unchanged.resume_override.is_none());
6387 assert!(!may_resume_run(&run, &unchanged));
6388 }
6389 }
6390
6391 #[test]
6392 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6393 let mut t = task();
6394 let mut run = run_state(RunStatus::Blocked);
6395 t.review_of = Some("approved-head".into());
6396 t.instruction =
6397 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6398 .into();
6399 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6400 run.reviewed_commits = Some(vec!["Commit log".into()]);
6401 t.start(run.id.clone());
6402 t.hold_machine(Some("interrupted review".into()));
6403 t.release_operator("cli", None);
6404 assert!(
6405 compatible_run_instruction(&run, &t),
6406 "legacy explicit review remains resumable"
6407 );
6408 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6409 .filter(|_| compatible_run_instruction(&run, &t));
6410 assert_eq!(
6411 choose_starter(None, false, unfinished.as_deref()),
6412 Starter::Resume(run.id.clone())
6413 );
6414 run.task_instruction = Some(t.instruction.clone());
6415 assert!(compatible_run_instruction(&run, &t));
6416 let reloaded: RunState =
6417 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6418 assert!(compatible_run_instruction(&reloaded, &t));
6419 t.edit(
6420 "new approved head".into(),
6421 "Reuse measured PR head B".into(),
6422 )
6423 .unwrap();
6424 assert!(!compatible_run_instruction(&run, &t));
6425
6426 t.review_of = None;
6428 t.instruction = "Fix Clippy on approved head".into();
6429 run.task_instruction = None;
6430 run.instruction = format!(
6431 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6432 t.instruction
6433 );
6434 assert!(compatible_run_instruction(&run, &t));
6435 t.instruction.push('\n');
6436 assert!(
6437 compatible_run_instruction(&run, &t),
6438 "legacy generated review trimmed the original criteria"
6439 );
6440 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6441 let mut unchanged = t.clone();
6442 assert_eq!(
6443 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6444 Some("approved-head")
6445 );
6446 t.edit("new plan".into(), "Reuse measured PR head B".into())
6447 .unwrap();
6448 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6449 assert!(!compatible_run_instruction(&run, &t));
6450 t.review_branch = Some("approved-head".into());
6451 t.review_instruction = None;
6452 assert!(
6453 take_review_branch(&mut t, Some(&run)).is_none(),
6454 "legacy retained candidate cannot replace a revised approved head"
6455 );
6456 }
6457
6458 #[test]
6459 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6460 crate::run::pin_test_home();
6461 let dir = tempfile::tempdir().unwrap();
6462 let queue = Queue::at(dir.path().join("queue"));
6463 for resume in [true, false] {
6464 let state = run_state(RunStatus::Blocked);
6465 let mut t = task();
6466 t.start(state.id.clone());
6467 let action = if resume {
6468 ask::ChoiceAction::Resume {
6469 run: state.id.clone(),
6470 }
6471 } else {
6472 ask::ChoiceAction::Requeue
6473 };
6474 let mut q = action_question(&state.id, action);
6475 q.status = ask::QuestionStatus::Open;
6476 q.answer = None;
6477 q.answered_at = None;
6478 t.fail("run stopped with a question", 3);
6479 hold_if_runnable(&queue, &mut t);
6480 assert!(t.operator_decided_at.is_none());
6481 assert!(t.operator_held());
6482 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6483 let expected = if resume {
6484 ActionDecision::Resume(state.id.clone())
6485 } else {
6486 ActionDecision::Requeue
6487 };
6488 assert_eq!(
6489 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6490 expected
6491 );
6492 t.hold_manual(Some("later explicit hold".into()));
6493 assert_eq!(
6494 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6495 ActionDecision::Stale
6496 );
6497 }
6498 }
6499
6500 #[test]
6501 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6502 let mut t = task();
6503 t.answers.push(crate::queue::AnsweredQuestion {
6504 question: "Inspect the remote?".into(),
6505 answer: "Inspect then wait".into(),
6506 });
6507 let old = instruction_for(&t);
6508 assert!(compatible_instruction(&old, &t));
6509 t.instruction = "Reuse approved PR head and review".into();
6510 t.release_operator("cli", None);
6511 assert!(!compatible_instruction(&old, &t));
6512 let approved = instruction_for(&t);
6513 assert!(approved.contains("waiting premises are superseded"));
6514 assert!(compatible_instruction(&approved, &t));
6515 assert_eq!(resumed_instruction(&approved, &t), approved);
6516 t.release_operator("web", None);
6517 assert!(compatible_instruction(&approved, &t));
6518 assert_eq!(t.resumption_history.len(), 1);
6519 t.start("run".into());
6520 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6521 }
6522
6523 #[test]
6527 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
6528 let mut state = RunState::new(
6529 PathBuf::from("/repo"),
6530 "main".to_owned(),
6531 "abc1234".to_owned(),
6532 "task".to_owned(),
6533 Config::default(),
6534 );
6535 state.status = RunStatus::Blocked;
6536 state.resource_block = Some(crate::run::ResourceBlock {
6537 node: "graft".to_owned(),
6538 kind: "shared build cache".to_owned(),
6539 detail: "timed out".to_owned(),
6540 at: jiff::Timestamp::now(),
6541 });
6542 let mut t = task();
6543 t.start("20260902-000000-aaaa".to_owned());
6544 assert_eq!(t.attempts, 1);
6545 let verdict = Verdict {
6546 status: RunStatus::Blocked,
6547 left_pr: false,
6548 quota_hit: false,
6549 parked: false,
6550 no_viable_candidates: false,
6551 };
6552 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6553 assert_eq!(t.status, TaskStatus::Held);
6554 assert_eq!(t.attempts, 0, "the attempt is refunded");
6555 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6556 assert!(
6557 !t.operator_held(),
6558 "a machine hold, released without a human"
6559 );
6560
6561 let mut selected = task();
6562 selected.start("20260902-000000-cccc".into());
6563 selected.replacement = Some(crate::merge_decision::Replacement {
6564 question: "fixture".into(),
6565 run: "earlier".into(),
6566 pr: "https://example.invalid/pr/1".into(),
6567 agent: "selected".into(),
6568 context: "fixture".into(),
6569 });
6570 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6571 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6572 assert_eq!(selected.attempts, 0);
6573 assert!(
6574 !selected.operator_held(),
6575 "a selected agent also retains a machine resource hold"
6576 );
6577 assert!(
6578 selected
6579 .hold_reason
6580 .as_deref()
6581 .unwrap()
6582 .starts_with("[resource] ")
6583 );
6584
6585 let mut p = task();
6587 p.start("20260902-000000-bbbb".to_owned());
6588 settle_and_diagnose(
6589 &mut p,
6590 Verdict {
6591 parked: true,
6592 ..verdict
6593 },
6594 "parked",
6595 2,
6596 &state,
6597 );
6598 assert_eq!(p.status, TaskStatus::Parked);
6599 }
6600
6601 #[test]
6602 fn record_failure_notice_links_the_task() {
6603 let dir = tempfile::tempdir().unwrap();
6604 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6607 let queue = Queue::at(dir.path().join("queue"));
6608 let mut t = interrupt_task("rec-fail-1");
6609 record(&queue, &mut t);
6610
6611 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6612 assert_eq!(found.len(), 1);
6613 assert_eq!(found[0].key, "loop:record");
6614 assert_eq!(
6615 found[0].link,
6616 Some(Link::Task {
6617 id: "rec-fail-1".to_owned()
6618 })
6619 );
6620 }
6621
6622 fn interrupt_task(id: &str) -> Task {
6625 let mut t = task();
6626 t.id = id.to_owned();
6627 t.interrupt = true;
6628 t
6629 }
6630
6631 fn task_with_id(id: &str) -> Task {
6633 let mut t = task();
6634 t.id = id.to_owned();
6635 t
6636 }
6637
6638 fn urgent_task(id: &str) -> Task {
6640 let mut t = task();
6641 t.id = id.to_owned();
6642 t.urgent = true;
6643 t
6644 }
6645
6646 #[test]
6647 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6648 let dir = tempfile::tempdir().unwrap();
6649 let queue = Queue::at(dir.path().join("queue"));
6650 let mut parked = task_with_id("parked");
6651 parked.start("run".into());
6652 parked.park("upgrade");
6653 parked.record_pass_at(
6654 "run",
6655 0,
6656 Some(Timestamp::now()),
6657 1,
6658 Some(crate::run::ParkCause::Upgrade {
6659 from: "old".into(),
6660 to: None,
6661 }),
6662 );
6663 let mut fresh = task_with_id("fresh");
6664 fresh.priority = 100;
6665 let mut urgent_task = urgent_task("urgent");
6666 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6667 queue.put(t).unwrap();
6668 }
6669 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6670 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6671 let candidates = runnable(&queue);
6672 assert_eq!(candidates[0].id, parked.id);
6673 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6674 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6675 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6676 drop(first);
6677 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6678 let next = runnable(&queue)
6679 .into_iter()
6680 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6681 .unwrap();
6682 assert_eq!(next.task.id, fresh.id);
6683 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6684 drop(claimed_elsewhere);
6685 drop(next);
6686 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6687 }
6688
6689 #[test]
6690 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6691 let dir = tempfile::tempdir().unwrap();
6692 let queue = Queue::at(dir.path().join("queue"));
6693 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6694 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6695 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6696 let mut first = task();
6697 queue.put(&mut first).unwrap();
6698 let stale_plain = queue.get(&first.id).unwrap();
6699 first.set_urgent(true).unwrap();
6700 queue.put(&mut first).unwrap();
6701 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6702 let mut second = urgent_task("second");
6703 queue.put(&mut second).unwrap();
6704 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6705 drop(queue.claim(&second.id).unwrap());
6706 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6707 drop(dispatched);
6708 let stale_urgent = queue.get(&first.id).unwrap();
6709 first.set_urgent(false).unwrap();
6710 queue.put(&mut first).unwrap();
6711 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6712 assert_eq!(urgent.available_permits(), 1);
6713 drop(queue.claim(&first.id).unwrap());
6714 drop(occupied);
6715 }
6716
6717 #[test]
6718 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6719 let dir = tempfile::tempdir().unwrap();
6720 let queue = Queue::at(dir.path().join("queue"));
6721 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6722 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6723 let mut t = task();
6724 queue.put(&mut t).unwrap();
6725 for enabled in [true, false] {
6726 let stale = queue.get(&t.id).unwrap();
6727 t.set_urgent(enabled).unwrap();
6728 queue.put(&mut t).unwrap();
6729 assert_ne!(stale.urgent, enabled);
6730 let DispatchClaim {
6731 task: fresh,
6732 claim,
6733 permit,
6734 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6735 assert_eq!(fresh.urgent, enabled);
6736 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6737 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6738 assert!(queue.claim(&t.id).is_err());
6739 assert_eq!(fresh.attempts, stale.attempts);
6740 drop((claim, permit));
6741 assert_eq!(ordinary.available_permits(), 1);
6742 assert_eq!(urgent.available_permits(), 1);
6743 }
6744 t.set_urgent(true).unwrap();
6745 queue.put(&mut t).unwrap();
6746 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6747 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6748 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6749 drop(queue.claim(&t.id).unwrap());
6750 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6751 drop(occupied);
6752 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6753 drop(queue.claim(&t.id).unwrap());
6754 let held = queue.claim(&t.id).unwrap();
6755 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6756 drop(held);
6757 t.status = TaskStatus::Held;
6758 queue.put(&mut t).unwrap();
6759 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6760 drop(queue.claim(&t.id).unwrap());
6761 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6762 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6763 drop(queue.claim(&t.id).unwrap());
6764 assert_eq!(urgent.available_permits(), 1);
6765 }
6766
6767 #[test]
6772 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6773 assert_eq!(permit_kind(true, false), PermitKind::None);
6774 assert_eq!(permit_kind(true, true), PermitKind::None);
6775 }
6776
6777 #[test]
6782 fn permit_kind_separates_urgent_from_ordinary() {
6783 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6784 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6785 }
6786
6787 #[test]
6794 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6795 let cfg = Config::default();
6796 let repo = Path::new("/any/repo/path");
6797
6798 let reason =
6799 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
6800 assert!(reason.contains("1024"), "{reason}");
6801 assert!(
6802 reason.contains(&cfg.disk.min_free_bytes.to_string()),
6803 "{reason}"
6804 );
6805
6806 assert_eq!(
6807 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
6808 None,
6809 "exactly at the floor is open"
6810 );
6811 assert_eq!(
6812 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
6813 None,
6814 "comfortably above the floor is open"
6815 );
6816 }
6817
6818 #[test]
6819 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
6820 let mut cfg = Config::default();
6821 cfg.disk.min_free_bytes = 0;
6822 let repo = Path::new("/any/repo/path");
6823 assert_eq!(
6824 disk_gate_with(repo, &cfg, |_| Ok(0)),
6825 None,
6826 "a zero floor never measures at all"
6827 );
6828 }
6829
6830 #[test]
6831 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
6832 let cfg = Config::default();
6833 let repo = Path::new("/any/repo/path");
6834 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
6835 .expect("a measurement failure must close the gate, not open it");
6836 assert!(reason.contains("could not measure"), "{reason}");
6837 }
6838
6839 #[test]
6840 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
6841 let ordinary = task();
6842 let next = advance_interrupt(
6843 Interrupt::Idle,
6844 std::slice::from_ref(&ordinary.id),
6845 std::slice::from_ref(&ordinary),
6846 );
6847 assert_eq!(next, Interrupt::Idle);
6848 }
6849
6850 #[test]
6851 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
6852 let marked = interrupt_task("marked");
6855 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6856 assert_eq!(next, Interrupt::Idle);
6857 }
6858
6859 #[test]
6860 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
6861 let marked = interrupt_task("marked");
6862 let next = advance_interrupt(
6863 Interrupt::Idle,
6864 &["running".to_owned()],
6865 std::slice::from_ref(&marked),
6866 );
6867 assert_eq!(
6868 next,
6869 Interrupt::Parking {
6870 parked: vec!["running".to_owned()],
6871 interrupt_task: "marked".to_owned(),
6872 }
6873 );
6874 }
6875
6876 #[test]
6885 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
6886 let marked = interrupt_task("marked");
6887
6888 let two = advance_interrupt(
6889 Interrupt::Idle,
6890 &["a".to_owned(), "b".to_owned()],
6891 std::slice::from_ref(&marked),
6892 );
6893 assert_eq!(two, Interrupt::Idle);
6894
6895 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
6896 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
6897 }
6898
6899 #[test]
6900 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
6901 let state = Interrupt::Parking {
6902 parked: vec!["running".to_owned()],
6903 interrupt_task: "marked".to_owned(),
6904 };
6905 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
6907 assert_eq!(still_going, state);
6908
6909 let stopped_but_not_yet_dispatched =
6913 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
6914 assert_eq!(stopped_but_not_yet_dispatched, state);
6915
6916 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
6918 assert_eq!(
6919 dispatched,
6920 Interrupt::Running {
6921 parked: vec!["running".to_owned()],
6922 interrupt_task: "marked".to_owned(),
6923 }
6924 );
6925 }
6926
6927 #[test]
6928 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
6929 let state = Interrupt::Running {
6930 parked: vec!["running".to_owned()],
6931 interrupt_task: "marked".to_owned(),
6932 };
6933 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
6934 assert_eq!(still_running, state);
6935
6936 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
6943 assert_eq!(
6944 ended,
6945 Interrupt::Resuming {
6946 parked: vec!["running".to_owned()]
6947 }
6948 );
6949 }
6950
6951 #[test]
6952 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
6953 let state = Interrupt::Resuming {
6954 parked: vec!["running".to_owned()],
6955 };
6956 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
6957 assert_eq!(still_waiting, state);
6958
6959 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
6960 assert_eq!(dispatched, Interrupt::Idle);
6961 }
6962
6963 #[test]
6969 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
6970 {
6971 let state = Interrupt::Parking {
6972 parked: vec!["running".to_owned()],
6973 interrupt_task: "marked".to_owned(),
6974 };
6975 let next = advance_interrupt(state, &[], &[]);
6978 assert_eq!(
6979 next,
6980 Interrupt::Resuming {
6981 parked: vec!["running".to_owned()]
6982 },
6983 "abandoning the interrupt must not abandon the resume it owes"
6984 );
6985 }
6986
6987 #[test]
6990 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
6991 let state = Interrupt::Resuming {
6992 parked: vec!["running".to_owned()],
6993 };
6994 let next = advance_interrupt(state, &[], &[]);
6995 assert_eq!(
6996 next,
6997 Interrupt::Idle,
6998 "nothing is left to wait for; the loop must not stay wedged"
6999 );
7000 }
7001
7002 #[test]
7003 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7004 let marked = interrupt_task("marked");
7005 let next = advance_interrupt_tick(
7006 false,
7007 Interrupt::Idle,
7008 &["running".to_owned()],
7009 std::slice::from_ref(&marked),
7010 );
7011 assert_eq!(
7012 next,
7013 Interrupt::Idle,
7014 "an unmarked, unconfigured daemon must behave exactly as before"
7015 );
7016 }
7017
7018 #[test]
7019 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7020 let state = Interrupt::Parking {
7021 parked: vec!["running".to_owned()],
7022 interrupt_task: "marked".to_owned(),
7023 };
7024 let candidates = vec![interrupt_task("marked"), task()];
7025 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7026 assert!(
7027 allowed.is_empty(),
7028 "nothing may dispatch - not even the interrupt task itself - \
7029 until the parked run has actually stopped"
7030 );
7031 }
7032
7033 #[test]
7047 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7048 for state in [
7049 Interrupt::Parking {
7050 parked: vec!["running".to_owned()],
7051 interrupt_task: "marked".to_owned(),
7052 },
7053 Interrupt::Running {
7054 parked: vec!["running".to_owned()],
7055 interrupt_task: "marked".to_owned(),
7056 },
7057 Interrupt::Resuming {
7058 parked: vec!["running".to_owned()],
7059 },
7060 ] {
7061 let candidates = vec![
7062 interrupt_task("marked"),
7063 urgent_task("hot"),
7064 task_with_id("ordinary"),
7065 ];
7066 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7067 assert!(
7068 !allowed.iter().any(|t| t.id == "hot"),
7069 "an urgent candidate must wait out the same gate as anything \
7070 else while the run it would run alongside has not actually \
7071 left flight, for state {state:?}: {allowed:?}"
7072 );
7073 }
7074 }
7075
7076 #[test]
7082 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7083 let state = Interrupt::Resuming {
7084 parked: vec!["hot".to_owned()],
7085 };
7086 let candidates = vec![urgent_task("hot"), task()];
7087 let allowed = interrupt_gate(&state, &[], candidates);
7088 assert_eq!(
7089 allowed.iter().filter(|t| t.id == "hot").count(),
7090 1,
7091 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7092 );
7093 }
7094
7095 #[test]
7096 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7097 let state = Interrupt::Parking {
7098 parked: vec!["running".to_owned()],
7099 interrupt_task: "marked".to_owned(),
7100 };
7101 let other = task();
7102 let candidates = vec![interrupt_task("marked"), other.clone()];
7103 let allowed = interrupt_gate(&state, &[], candidates);
7104 assert_eq!(allowed.len(), 1);
7105 assert_eq!(allowed[0].id, "marked");
7106 }
7107
7108 #[test]
7109 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7110 let state = Interrupt::Running {
7111 parked: vec!["running".to_owned()],
7112 interrupt_task: "marked".to_owned(),
7113 };
7114 let candidates = vec![task(), task()];
7115 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7116 assert!(allowed.is_empty());
7117 }
7118
7119 #[test]
7126 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7127 let state = Interrupt::Resuming {
7128 parked: vec!["a".to_owned(), "c".to_owned()],
7129 };
7130 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7131 let allowed = interrupt_gate(&state, &[], candidates);
7132 assert_eq!(
7133 allowed.len(),
7134 1,
7135 "at most one candidate may be offered while resuming: {allowed:?}"
7136 );
7137 assert_eq!(allowed[0].id, "a");
7138 }
7139
7140 #[test]
7141 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7142 let state = Interrupt::Resuming {
7143 parked: vec!["a".to_owned()],
7144 };
7145 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7146 assert!(allowed.is_empty());
7147 }
7148
7149 #[test]
7155 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7156 let running = task(); let marked = interrupt_task("marked");
7158
7159 let mut state = Interrupt::Idle;
7160 let in_flight = vec![running.id.clone()];
7162 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7163 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7164 assert!(gated.is_empty(), "still waiting on `running` to park");
7165
7166 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7168 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7169 assert_eq!(
7170 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7171 vec!["marked"],
7172 "only the interrupt task may be offered to the dispatcher now"
7173 );
7174
7175 state = advance_interrupt_tick(
7177 true,
7178 state,
7179 &["marked".to_owned()],
7180 std::slice::from_ref(&running),
7181 );
7182 let gated = interrupt_gate(
7183 &state,
7184 &["marked".to_owned()],
7185 vec![marked.clone(), running.clone()],
7186 );
7187 assert!(
7188 gated.is_empty(),
7189 "the parked run must not be offered back while the interrupt \
7190 task is still running"
7191 );
7192
7193 let other = task_with_id("other");
7197 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7198 assert_eq!(
7199 state,
7200 Interrupt::Resuming {
7201 parked: vec![running.id.clone()]
7202 }
7203 );
7204 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7205 assert_eq!(
7206 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7207 vec![running.id.as_str()],
7208 "exactly the parked run resumes - not the unrelated task, even \
7209 though it was offered first"
7210 );
7211
7212 state = advance_interrupt_tick(
7216 true,
7217 state,
7218 std::slice::from_ref(&running.id),
7219 std::slice::from_ref(&other),
7220 );
7221 assert_eq!(state, Interrupt::Idle);
7222 let gated = interrupt_gate(
7223 &state,
7224 std::slice::from_ref(&running.id),
7225 vec![other.clone()],
7226 );
7227 assert_eq!(
7228 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7229 vec![other.id.as_str()],
7230 "ordinary dispatch is unrestricted again"
7231 );
7232 }
7233
7234 #[test]
7235 fn every_run_status_settles_the_task_it_came_from() {
7236 let table = [
7238 (RunStatus::Merged, TaskStatus::Done, 1),
7239 (RunStatus::Ready, TaskStatus::Done, 1),
7240 (RunStatus::Stalled, TaskStatus::Failed, 0),
7241 (RunStatus::Blocked, TaskStatus::Failed, 1),
7242 (RunStatus::Failed, TaskStatus::Failed, 1),
7243 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7244 (RunStatus::Prep, TaskStatus::Failed, 1),
7245 (RunStatus::Implementing, TaskStatus::Failed, 1),
7246 (RunStatus::Judging, TaskStatus::Failed, 1),
7247 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7248 (RunStatus::Voting, TaskStatus::Failed, 1),
7249 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7250 (RunStatus::Gating, TaskStatus::Failed, 1),
7251 ];
7252 for (run, want, attempts) in table {
7253 let mut t = task();
7254 t.start("20260902-000000-aaaa".to_owned());
7255 settle(
7256 &mut t,
7257 Verdict {
7258 status: run,
7259 left_pr: false,
7260 parked: false,
7261 quota_hit: matches!(run, RunStatus::Stalled),
7262 no_viable_candidates: false,
7263 },
7264 "why",
7265 2,
7266 );
7267 assert_eq!(t.status, want, "task status after {}", label(run));
7268 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7269 }
7270 }
7271
7272 #[test]
7273 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7274 let mut stalled = task();
7275 stalled.start("20260902-000000-aaaa".to_owned());
7276 settle(
7277 &mut stalled,
7278 Verdict {
7279 status: RunStatus::Stalled,
7280 left_pr: false,
7281 parked: false,
7282 quota_hit: true,
7283 no_viable_candidates: false,
7284 },
7285 "quota",
7286 1,
7287 );
7288 assert_eq!(stalled.attempts, 0);
7289 assert!(
7290 stalled.status.runnable(),
7291 "a machine problem must leave the task in line"
7292 );
7293
7294 let mut blocked = task();
7295 blocked.start("20260902-000000-aaaa".to_owned());
7296 settle(
7297 &mut blocked,
7298 Verdict {
7299 status: RunStatus::Blocked,
7300 left_pr: false,
7301 parked: false,
7302 quota_hit: false,
7303 no_viable_candidates: false,
7304 },
7305 "findings open",
7306 1,
7307 );
7308 assert_eq!(blocked.attempts, 1);
7309 assert_eq!(
7310 blocked.status,
7311 TaskStatus::Held,
7312 "the last attempt hands the task to a human"
7313 );
7314 }
7315
7316 #[test]
7317 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7318 let mut delivered = task();
7321 delivered.start("20260903-080619-01c2".to_owned());
7322 settle(
7323 &mut delivered,
7324 Verdict {
7325 status: RunStatus::Blocked,
7326 left_pr: true,
7327 parked: false,
7328 quota_hit: false,
7329 no_viable_candidates: false,
7330 },
7331 "no check status",
7332 4,
7333 );
7334 assert_eq!(
7335 delivered.status,
7336 TaskStatus::Held,
7337 "a pull request waiting on CI or a person is not a retryable failure"
7338 );
7339 assert!(
7340 !delivered.status.runnable(),
7341 "the loop must not pick this task up again"
7342 );
7343 assert_eq!(
7344 delivered.last_error.as_deref(),
7345 Some("no check status"),
7346 "the operator needs to be told what the gate was waiting for"
7347 );
7348
7349 let mut empty_handed = task();
7352 empty_handed.start("20260903-080619-01c2".to_owned());
7353 settle(
7354 &mut empty_handed,
7355 Verdict {
7356 status: RunStatus::Blocked,
7357 left_pr: false,
7358 parked: false,
7359 quota_hit: false,
7360 no_viable_candidates: false,
7361 },
7362 "findings open",
7363 4,
7364 );
7365 assert_eq!(empty_handed.status, TaskStatus::Failed);
7366 assert!(empty_handed.status.runnable());
7367 }
7368
7369 #[test]
7370 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7371 let mut noop = task();
7377 noop.start("20260912-131304-391f".to_owned());
7378 settle(
7379 &mut noop,
7380 Verdict {
7381 status: RunStatus::VerifiedNoop,
7382 left_pr: false,
7383 parked: false,
7384 quota_hit: false,
7385 no_viable_candidates: true,
7386 },
7387 "candidate A: already fixed by b32cfc4, on main",
7388 4,
7389 );
7390 assert_eq!(
7391 noop.status,
7392 TaskStatus::Held,
7393 "an unverified claim is a request for a human, not a failure"
7394 );
7395 assert!(
7396 !noop.status.runnable(),
7397 "the loop must not requeue this on the same unverified claim"
7398 );
7399 assert_eq!(noop.attempts, 1);
7404 }
7405
7406 #[test]
7407 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7408 let mut parked = task();
7413 parked.start("20260903-183634-2d98".to_owned());
7414 settle(
7415 &mut parked,
7416 Verdict {
7417 status: RunStatus::Implementing,
7418 left_pr: false,
7419 quota_hit: false,
7420 parked: true,
7421 no_viable_candidates: false,
7422 },
7423 "parked after `implementing`",
7424 2,
7425 );
7426 assert_eq!(parked.attempts, 0, "a park is refunded");
7427 assert!(
7428 parked.status.runnable(),
7429 "and the task stays in line so the next loop resumes its run"
7430 );
7431 assert_eq!(parked.status, TaskStatus::Parked);
7432 assert_eq!(
7433 parked.park_reason.as_deref(),
7434 Some("parked after `implementing`"),
7435 "the card says where it stopped"
7436 );
7437 assert_eq!(parked.last_error, None, "nothing failed");
7438
7439 let mut broken = task();
7443 broken.start("20260903-183634-2d98".to_owned());
7444 settle(
7445 &mut broken,
7446 Verdict {
7447 status: RunStatus::Implementing,
7448 left_pr: false,
7449 quota_hit: false,
7450 parked: false,
7451 no_viable_candidates: false,
7452 },
7453 "returned mid-flight",
7454 2,
7455 );
7456 assert_eq!(broken.attempts, 1);
7457 }
7458
7459 #[test]
7460 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7461 let mut flaky = task();
7466 flaky.start("20260903-123023-e633".to_owned());
7467 settle(
7468 &mut flaky,
7469 Verdict {
7470 status: RunStatus::Stalled,
7471 left_pr: false,
7472 parked: false,
7473 quota_hit: false,
7474 no_viable_candidates: false,
7475 },
7476 "verdict rests on 1 of 3 judges",
7477 2,
7478 );
7479 assert_eq!(
7480 flaky.attempts, 1,
7481 "flakiness spends an attempt, so `max_attempts` still bounds it"
7482 );
7483 assert!(flaky.status.runnable(), "and it is still worth retrying");
7484
7485 let mut limited = task();
7487 limited.start("20260903-123023-e633".to_owned());
7488 settle(
7489 &mut limited,
7490 Verdict {
7491 status: RunStatus::Stalled,
7492 left_pr: false,
7493 parked: false,
7494 quota_hit: true,
7495 no_viable_candidates: false,
7496 },
7497 "judge-2, judge-3 out of quota",
7498 2,
7499 );
7500 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7501 assert!(limited.status.runnable());
7502
7503 let mut worn = task();
7506 for _ in 0..2 {
7507 worn.release();
7508 }
7509 worn.start("20260903-123023-e633".to_owned());
7510 worn.attempts = 2;
7511 settle(
7512 &mut worn,
7513 Verdict {
7514 status: RunStatus::Stalled,
7515 left_pr: false,
7516 parked: false,
7517 quota_hit: false,
7518 no_viable_candidates: false,
7519 },
7520 "no quorum again",
7521 2,
7522 );
7523 assert_eq!(worn.status, TaskStatus::Held);
7524 assert!(!worn.status.runnable());
7525 }
7526
7527 #[test]
7528 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
7529 let mut wiped_out = task();
7536 wiped_out.start("20260907-025000-a1b2".to_owned());
7537 settle(
7538 &mut wiped_out,
7539 Verdict {
7540 status: RunStatus::Failed,
7541 left_pr: false,
7542 parked: false,
7543 quota_hit: true,
7544 no_viable_candidates: true,
7545 },
7546 "no candidate produced a change; nothing to judge",
7547 2,
7548 );
7549 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7550 assert!(
7551 wiped_out.status.runnable(),
7552 "a machine problem must leave the task in line"
7553 );
7554
7555 let mut partial_progress = task();
7561 partial_progress.start("20260907-025500-c3d4".to_owned());
7562 settle(
7563 &mut partial_progress,
7564 Verdict {
7565 status: RunStatus::Failed,
7566 left_pr: false,
7567 parked: false,
7568 quota_hit: true,
7569 no_viable_candidates: false,
7570 },
7571 "gate failed on the winning candidate",
7572 2,
7573 );
7574 assert_eq!(
7575 partial_progress.attempts, 1,
7576 "a candidate that actually produced a change spends the attempt \
7577 even though some other seat hit its quota"
7578 );
7579 assert!(partial_progress.status.runnable());
7580 }
7581
7582 #[test]
7583 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7584 let mut t = task();
7590 t.start("20260907-025000-a1b2".to_owned());
7591 let mut state = run_state(RunStatus::Failed);
7592 state.quota.push(QuotaLoss {
7593 seat: "cand-a".to_owned(),
7594 node: "implement".to_owned(),
7595 at: Timestamp::now(),
7596 reset: None,
7597 });
7598 assert!(
7599 state.viable().is_empty(),
7600 "no candidate was added, so nothing is viable"
7601 );
7602 reclaim(&mut t, Some(state), 2, "en");
7603 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7604 assert!(t.status.runnable());
7605 }
7606
7607 #[test]
7608 fn a_held_task_is_never_offered_to_the_loop() {
7609 let dir = tempfile::tempdir().unwrap();
7610 let queue = Queue::at(dir.path().to_path_buf());
7611 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7612 let mut t = task();
7613 t.id = format!("2026090{n}-000000-000{n}");
7614 t.priority = priority;
7615 queue.put(&mut t).unwrap();
7616 }
7617 let mut held = task();
7618 held.id = "20260909-000000-9999".to_owned();
7619 held.priority = 99;
7620 held.hold_machine(None);
7621 queue.put(&mut held).unwrap();
7622
7623 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7624 assert_eq!(order.len(), 3);
7625 assert!(!order.contains(&held.id));
7626 assert_eq!(
7627 order.first().cloned(),
7628 queue.next_runnable().map(|t| t.id),
7629 "the loop's first candidate is exactly what the queue offers"
7630 );
7631 assert_eq!(
7632 order,
7633 vec![
7634 "20260902-000000-0002".to_owned(),
7635 "20260903-000000-0003".to_owned(),
7636 "20260901-000000-0001".to_owned(),
7637 ],
7638 "priority first, then oldest, so nothing starves"
7639 );
7640 }
7641
7642 #[test]
7643 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7644 let dir = tempfile::tempdir().unwrap();
7645 let queue = Queue::at(dir.path().to_path_buf());
7646 let mut old = task();
7647 old.id = "20260101-000000-old0".to_owned();
7648 queue.put(&mut old).unwrap();
7649 let mut fresh = task();
7650 fresh.id = "20260101-000000-new0".to_owned();
7651 queue.put(&mut fresh).unwrap();
7652
7653 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7657 std::thread::sleep(Duration::from_millis(60));
7658 let live = queue.claim(&fresh.id).unwrap();
7659
7660 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7661 assert_eq!(swept, vec![old.id.clone()]);
7662 assert!(
7663 queue.claim(&old.id).is_ok(),
7664 "an unparseable lock older than the threshold is swept"
7665 );
7666 assert!(
7667 queue.claim(&fresh.id).is_err(),
7668 "a live pid protects its lock regardless of age"
7669 );
7670 drop(live);
7671 }
7672
7673 #[test]
7674 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7675 let dir = tempfile::tempdir().unwrap();
7678 let queue = Queue::at(dir.path().to_path_buf());
7679 let mut t = task();
7680 t.id = "20260101-000000-live".to_owned();
7681 queue.put(&mut t).unwrap();
7682
7683 let claim = queue.claim(&t.id).unwrap();
7684 std::thread::sleep(Duration::from_millis(60));
7685
7686 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7687 assert!(
7688 swept.is_empty(),
7689 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7690 );
7691 assert!(
7692 queue.claim(&t.id).is_err(),
7693 "the lock still protects its task"
7694 );
7695 drop(claim);
7696 }
7697
7698 fn injected_dead_pid() -> u32 {
7701 std::process::id().checked_add(1).unwrap_or(1)
7702 }
7703
7704 #[test]
7705 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7706 let dir = tempfile::tempdir().unwrap();
7707 let queue = Queue::at(dir.path().to_path_buf());
7708 let mut t = task();
7709 t.id = "20260101-000000-dead".to_owned();
7710 queue.put(&mut t).unwrap();
7711 let dead_pid = injected_dead_pid();
7712
7713 std::fs::write(
7718 dir.path().join(format!("{}.lock", t.id)),
7719 dead_pid.to_string(),
7720 )
7721 .unwrap();
7722
7723 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7724 pid != dead_pid
7725 });
7726 assert_eq!(
7727 swept,
7728 vec![t.id.clone()],
7729 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7730 );
7731 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7732 }
7733
7734 #[test]
7735 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7736 let dir = tempfile::tempdir().unwrap();
7737 let queue = Queue::at(dir.path().to_path_buf());
7738 let mut t = task();
7739 t.id = "20260101-000000-late".to_owned();
7740 queue.put(&mut t).unwrap();
7741 let dead_pid = injected_dead_pid();
7742
7743 assert!(
7746 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7747 "nothing has claimed the task yet"
7748 );
7749
7750 std::fs::write(
7753 dir.path().join(format!("{}.lock", t.id)),
7754 dead_pid.to_string(),
7755 )
7756 .unwrap();
7757
7758 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7762 pid != dead_pid
7763 });
7764 assert_eq!(swept, vec![t.id.clone()]);
7765 }
7766
7767 #[test]
7768 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7769 crate::run::pin_test_home();
7774 let dir = tempfile::tempdir().unwrap();
7775 let queue = Queue::at(dir.path().to_path_buf());
7776 let mut t = task();
7777 t.id = "20260101-000000-crsh".to_owned();
7778 t.status = TaskStatus::Running;
7779 t.attempts = 1;
7780 t.runs.push("20260904-000000-4043".to_owned());
7784 queue.put(&mut t).unwrap();
7785 let dead_pid = injected_dead_pid();
7786
7787 std::fs::write(
7790 dir.path().join(format!("{}.lock", t.id)),
7791 dead_pid.to_string(),
7792 )
7793 .unwrap();
7794
7795 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
7801 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
7802
7803 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7804 pid != dead_pid
7805 });
7806 assert_eq!(swept, vec![t.id.clone()]);
7807
7808 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
7809 assert_eq!(reclaimed, vec![t.id.clone()]);
7810 let after = queue.get(&t.id).unwrap();
7811 assert_eq!(
7812 after.status,
7813 TaskStatus::Held,
7814 "no run.json to recover from, so a human is asked"
7815 );
7816 assert_eq!(
7817 after.runs,
7818 vec!["20260904-000000-4043".to_owned()],
7819 "the crashed run's id is kept as evidence, not discarded"
7820 );
7821 }
7822
7823 #[test]
7824 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
7825 let dir = tempfile::tempdir().unwrap();
7826 let queue = Queue::at(dir.path().to_path_buf());
7827 let mut t = task();
7828 t.id = "20260101-000000-unknown".to_owned();
7829 queue.put(&mut t).unwrap();
7830 let dead_pid = injected_dead_pid();
7831 std::fs::write(
7832 dir.path().join(format!("{}.lock", t.id)),
7833 dead_pid.to_string(),
7834 )
7835 .unwrap();
7836
7837 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
7838 assert!(swept.is_empty(), "an unknown pid must keep its lock");
7839 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
7840 }
7841
7842 fn run_state_in(status: RunStatus, language: &str) -> RunState {
7843 let mut s = run_state(status);
7844 s.config.graph.language = language.to_owned();
7845 s
7846 }
7847
7848 fn unstarted_verdict(status: RunStatus) -> Verdict {
7849 Verdict {
7850 status,
7851 left_pr: false,
7852 quota_hit: false,
7853 parked: false,
7854 no_viable_candidates: false,
7855 }
7856 }
7857
7858 #[test]
7859 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
7860 let mut t = task();
7861 t.attempts = 1;
7862 settle_in(
7863 &mut t,
7864 unstarted_verdict(RunStatus::AlreadyInBase),
7865 "already in main",
7866 1,
7867 phrases("en"),
7868 );
7869 assert_eq!(t.status, TaskStatus::Done);
7870 assert_eq!(t.attempts, 0);
7871 }
7872
7873 #[test]
7874 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
7875 let reason = |language: &str| {
7876 let mut t = task();
7877 settle_in(
7878 &mut t,
7879 unstarted_verdict(RunStatus::Judging),
7880 "boom",
7881 1,
7882 phrases(language),
7883 );
7884 t.last_error.or(t.hold_reason).unwrap_or_default()
7885 };
7886 assert!(
7887 reason("en").starts_with("the graph stopped at `"),
7888 "{}",
7889 reason("en")
7890 );
7891 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
7892 assert!(reason("日本語").contains("boom"));
7893 assert_eq!(reason("fr"), reason("en"));
7894 }
7895
7896 #[test]
7897 fn describe_follows_the_run_language_and_keeps_the_run_id() {
7898 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
7899 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
7900 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
7901 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
7902 assert!(ja.contains("[run "), "{ja}");
7903 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
7904 assert!(ended.starts_with("run 終了: "), "{ended}");
7905 let mut de = run_state_in(RunStatus::Failed, "de");
7906 let mut en = run_state_in(RunStatus::Failed, "en");
7907 de.id = "same".to_owned();
7908 en.id = "same".to_owned();
7909 assert_eq!(describe(&de), describe(&en));
7910 }
7911
7912 #[test]
7913 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
7914 use crate::handover::Refused;
7915 let cases = [
7916 Refused::Foreign {
7917 branch: "b".into(),
7918 path: "/w/x".into(),
7919 why: "made by hand".into(),
7920 },
7921 Refused::Unsafe {
7922 branch: "b".into(),
7923 path: "/w/x".into(),
7924 why: "its worktree has uncommitted changes (a.rs)".into(),
7925 },
7926 Refused::ReleaseFailed {
7927 branch: "b".into(),
7928 path: "/w/x".into(),
7929 run: "ab12".into(),
7930 },
7931 ];
7932 for r in &cases {
7933 let en = (phrases("en").handover_refused)(r);
7934 assert_eq!(en, r.to_string());
7935 let ja = (phrases("ja").handover_refused)(r);
7936 assert!(
7937 !ja.contains("is checked out") && !ja.contains("try again"),
7938 "{ja}"
7939 );
7940 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
7941 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
7942 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
7943 }
7944 }
7945 assert!(phrases("en").handover_hint.contains("release the task"));
7946 assert!(phrases("ja").handover_hint.contains("解放"));
7947 }
7948
7949 #[test]
7950 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
7951 let build = |lang: &str| {
7952 let mut s = run_state_in(RunStatus::Blocked, lang);
7953 s.id = "same".to_owned();
7954 s.config.graph.review_rounds = 3;
7955 let mut r = review_round(3);
7956 r.expected = 3;
7957 r.answered = 1;
7958 s.reviews.push(r);
7959 s.event(
7960 "review",
7961 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
7962 );
7963 s
7964 };
7965 let en = describe(&build("en"));
7966 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
7967 let ja = describe(&build("ja"));
7968 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
7969 assert!(!ja.contains("never answered"), "{ja}");
7970 assert!(ja.contains("[run "), "{ja}");
7971
7972 let mut failed = build("ja");
7975 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
7976 command: "cargo test".to_owned(),
7977 code: Some(1),
7978 output_tail: String::new(),
7979 duration_ms: 0,
7980 resource_blocked: false,
7981 });
7982 failed.event("review", "stopped; e2e failed: cargo test");
7983 let ja = describe(&failed);
7984 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
7985 assert!(!ja.contains("席のレビュアー"), "{ja}");
7986 }
7987
7988 #[test]
7989 fn refusals_and_recovery_prose_follow_the_language() {
7990 let t = held_task_with("r1");
7991 let q = action_question(
7992 "r1",
7993 ask::ChoiceAction::Resume {
7994 run: "r1".to_owned(),
7995 },
7996 );
7997 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
7998 ActionDecision::Refuse(s) => s,
7999 other => panic!("{other:?}"),
8000 };
8001 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8002 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8003 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8004
8005 let mut held = task();
8006 reclaim(&mut held, None, 2, "ja");
8007 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8008 let mut held = task();
8009 reclaim(&mut held, None, 2, "xx");
8010 assert!(held.hold_reason.unwrap().contains("held for a human"));
8011 }
8012
8013 fn run_state(status: RunStatus) -> RunState {
8014 let mut state = RunState::new(
8015 PathBuf::from("/repo"),
8016 "main".to_owned(),
8017 "abc1234def".to_owned(),
8018 "add retries".to_owned(),
8019 Config::default(),
8020 );
8021 state.status = status;
8022 state
8023 }
8024
8025 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8026 Candidate {
8027 index: 0,
8028 label,
8029 agent: "claude".to_owned(),
8030 branch: format!("magi/x/{label}"),
8031 worktree: PathBuf::from("/repo"),
8032 summary: summary.to_owned(),
8033 stat: String::new(),
8034 files: 0,
8035 commits: usize::from(!empty),
8036 empty,
8037 failed: failed.map(str::to_owned),
8038 verified_noop: None,
8039 duration_ms: 0,
8040 folded: false,
8041 }
8042 }
8043
8044 #[test]
8045 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8046 let mut state = run_state(RunStatus::Blocked);
8047 state.gate = vec![
8048 CommandOutcome {
8049 command: "cargo make check".to_owned(),
8050 code: Some(0),
8051 output_tail: "ok".to_owned(),
8052 duration_ms: 0,
8053 resource_blocked: false,
8054 },
8055 CommandOutcome {
8056 command: "cargo test".to_owned(),
8057 code: Some(101),
8058 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8059 duration_ms: 0,
8060 resource_blocked: false,
8061 },
8062 ];
8063 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8064 assert!(d.contains("cargo test"), "{d}");
8065 assert!(
8066 !d.contains("cargo make check"),
8067 "a passing check is not a diagnostic: {d}"
8068 );
8069 assert!(d.contains("assertion failed"), "{d}");
8070 }
8071
8072 #[test]
8073 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8074 let mut state = run_state(RunStatus::Blocked);
8075 state.event(
8076 "land",
8077 "stopped: the fixer produced no commit while 2 check(s) were failing \
8078 (build, lint); stopping instead of looping on an unchanged tree",
8079 );
8080 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8081 assert!(d.contains("build"), "{d}");
8082 assert!(d.contains("lint"), "{d}");
8083 assert!(d.contains("fixer produced no commit"), "{d}");
8084 }
8085
8086 #[test]
8087 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8088 let state = run_state(RunStatus::VerifiedNoop);
8093 let d = describe(&state);
8094 assert!(
8095 d.contains("agent-verified no-op"),
8096 "expected the display label, not the wire spelling: {d}"
8097 );
8098 assert!(!d.contains("verified_noop"), "{d}");
8099 }
8100
8101 #[test]
8102 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8103 let mut state = run_state(RunStatus::Failed);
8109 state.candidates = vec![candidate(
8110 'A',
8111 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8112 true,
8113 None,
8114 )];
8115 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8116 assert!(d.contains("candidate A"), "{d}");
8117 assert!(d.contains("tagged v1.2.3"), "{d}");
8118 }
8119
8120 #[test]
8121 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8122 let mut state = run_state(RunStatus::Failed);
8123 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8124 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8125 assert!(d.contains("candidate A"), "{d}");
8126 assert!(d.contains("agent timed out"), "{d}");
8127 }
8128
8129 #[test]
8130 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8131 let mut state = run_state(RunStatus::Failed);
8134 state.candidates = vec![candidate('A', "did the work", false, None)];
8135 assert!(diagnostic(&state).is_none());
8136 }
8137
8138 #[test]
8139 fn diagnostic_is_bounded_however_much_a_run_printed() {
8140 let mut state = run_state(RunStatus::Blocked);
8141 state.gate = vec![
8142 CommandOutcome {
8143 command: "cargo test".to_owned(),
8144 code: Some(101),
8145 output_tail: "x".repeat(50_000),
8146 duration_ms: 0,
8147 resource_blocked: false,
8148 },
8149 CommandOutcome {
8150 command: "cargo clippy".to_owned(),
8151 code: Some(1),
8152 output_tail: "y".repeat(50_000),
8153 duration_ms: 0,
8154 resource_blocked: false,
8155 },
8156 ];
8157 state.candidates = vec![
8158 candidate('A', &"z".repeat(50_000), true, None),
8159 candidate('B', &"w".repeat(50_000), true, None),
8160 ];
8161 let d = diagnostic(&state).expect("plenty here to diagnose");
8162 assert!(
8163 d.len() <= DIAGNOSTIC_MAX,
8164 "diagnostic grew to {} bytes, unbounded",
8165 d.len()
8166 );
8167 }
8168
8169 #[test]
8170 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8171 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8172 let mut t = task();
8173 let mut state = run_state(status);
8174 t.pr_recovery = Some(crate::pr_recovery::Selection {
8175 url: "url".into(),
8176 branch: format!("magi/task-{}", t.id),
8177 head: "a".repeat(40),
8178 source_head: None,
8179 source_run: None,
8180 source_proof: None,
8181 disposition: Some(crate::pr_recovery::Disposition::Land),
8182 instruction: t.instruction.clone(),
8183 approved_at: Timestamp::now(),
8184 });
8185 state.pr_recovery = t.pr_recovery.clone();
8186 t.start(state.id.clone());
8187 settle_and_diagnose(
8188 &mut t,
8189 Verdict {
8190 status,
8191 left_pr: false,
8192 quota_hit: false,
8193 parked: false,
8194 no_viable_candidates: false,
8195 },
8196 "delivery stopped",
8197 5,
8198 &state,
8199 );
8200 assert_eq!(t.status, TaskStatus::Held);
8201 let reason = t.hold_reason.as_deref().unwrap();
8202 if status == RunStatus::Ready {
8203 assert!(reason.contains("Selected PR delivery is incomplete"));
8204 assert!(t.recovery_budget.is_none());
8205 } else {
8206 assert!(reason.contains("Selected recovery stopped"));
8207 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8208 }
8209 assert!(t.pr_recovery.is_some());
8210 }
8211 }
8212
8213 #[test]
8214 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8215 let mut state = run_state(RunStatus::Blocked);
8216 state.gate = vec![CommandOutcome {
8217 command: "cargo test".to_owned(),
8218 code: Some(101),
8219 output_tail: "assertion failed".to_owned(),
8220 duration_ms: 0,
8221 resource_blocked: false,
8222 }];
8223 let verdict = Verdict {
8224 status: RunStatus::Blocked,
8225 left_pr: false,
8226 quota_hit: false,
8227 parked: false,
8228 no_viable_candidates: false,
8229 };
8230
8231 let mut t = task();
8234 t.start("run-1".to_owned());
8235 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8236 assert_eq!(t.status, TaskStatus::Failed);
8237 assert!(t.diagnostic.is_none());
8238
8239 t.start("run-2".to_owned());
8242 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8243 assert_eq!(t.status, TaskStatus::Held);
8244 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8245 assert!(d.contains("cargo test"), "{d}");
8246 }
8247
8248 #[test]
8249 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8250 crate::run::pin_test_home();
8255 let home = crate::run::home();
8256 let state = run_state(RunStatus::VerifiedNoop);
8257 let mut q = ask::Question::new(
8258 state.id.clone(),
8259 "implement".to_owned(),
8260 "impl-A".to_owned(),
8261 "is this really a no-op?".to_owned(),
8262 String::new(),
8263 Vec::new(),
8264 );
8265 Questions::at(home.join("questions")).put(&mut q).unwrap();
8266
8267 let verdict = Verdict {
8268 status: RunStatus::VerifiedNoop,
8269 left_pr: false,
8270 quota_hit: false,
8271 parked: false,
8272 no_viable_candidates: false,
8273 };
8274 let mut t = task();
8275 t.start(state.id.clone());
8276 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8277
8278 assert_eq!(t.status, TaskStatus::Held);
8279 let reason = t.hold_reason.expect("a held task must record why");
8280 assert!(
8281 reason.starts_with("run ended agent-verified no-op"),
8282 "the original settle reason must survive unchanged: {reason}"
8283 );
8284 assert!(
8285 reason.contains(q.short()),
8286 "the open question's id must be named so the notice is actionable: {reason}"
8287 );
8288 }
8289
8290 #[test]
8291 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8292 crate::run::pin_test_home();
8293 let state = run_state(RunStatus::VerifiedNoop);
8294
8295 let verdict = Verdict {
8296 status: RunStatus::VerifiedNoop,
8297 left_pr: false,
8298 quota_hit: false,
8299 parked: false,
8300 no_viable_candidates: false,
8301 };
8302 let mut t = task();
8303 t.start(state.id.clone());
8304 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8305
8306 assert_eq!(t.status, TaskStatus::Held);
8307 assert_eq!(
8308 t.hold_reason.as_deref(),
8309 Some("run ended agent-verified no-op"),
8310 "nothing to append when the question was already answered or never asked"
8311 );
8312 }
8313
8314 #[test]
8315 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8316 crate::run::pin_test_home();
8317 let mut first = run_state(RunStatus::Blocked);
8318 first.id = "20260101-000000-sup1".to_owned();
8319 first.save().unwrap();
8320 let mut second = run_state(RunStatus::Merged);
8321 second.id = "20260101-000000-sup2".to_owned();
8322 second.save().unwrap();
8323
8324 let mut t = task();
8325 t.runs = vec![first.id.clone(), second.id.clone()];
8326 t.status = TaskStatus::Done;
8327
8328 supersede_prior_runs(&t, &crate::run::home());
8329
8330 assert_eq!(
8331 RunState::load(&first.id).unwrap().status,
8332 RunStatus::Superseded,
8333 "the first attempt's Blocked no longer needs anyone's attention"
8334 );
8335 assert_eq!(
8336 RunState::load(&second.id).unwrap().status,
8337 RunStatus::Merged,
8338 "the run that actually succeeded is left exactly as it was"
8339 );
8340 }
8341
8342 #[test]
8343 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8344 crate::run::pin_test_home();
8350 let mut first = run_state(RunStatus::Blocked);
8351 first.id = "20260101-000000-sup9".to_owned();
8352 first.driver_pid = Some(std::process::id());
8355 first.driver_started_at = Some(
8356 crate::proc::process_started_at(std::process::id())
8357 .expect("this test process's own start time must be queryable"),
8358 );
8359 first.save().unwrap();
8360 let mut second = run_state(RunStatus::Merged);
8361 second.id = "20260101-000000-supa".to_owned();
8362 second.save().unwrap();
8363
8364 let mut t = task();
8365 t.runs = vec![first.id.clone(), second.id.clone()];
8366 t.status = TaskStatus::Done;
8367
8368 supersede_prior_runs(&t, &crate::run::home());
8369
8370 assert_eq!(
8371 RunState::load(&first.id).unwrap().status,
8372 RunStatus::Blocked,
8373 "a live driver_pid means something is still actually working this run, \
8374 even though no daemon claims it - rewriting under it would just be \
8375 undone the next time that process saves"
8376 );
8377 }
8378
8379 #[test]
8380 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8381 let dir = tempfile::tempdir().unwrap();
8387 let home = dir.path().to_path_buf();
8388 let queue = Queue::at(dir.path().join("queue"));
8389
8390 let mut first = run_state(RunStatus::Blocked);
8391 first.id = "20260101-000000-supd".to_owned();
8392 first.driver_pid = Some(std::process::id());
8393 first.driver_started_at = Some(
8394 crate::proc::process_started_at(std::process::id())
8395 .expect("this test process's own start time must be queryable"),
8396 );
8397 first.save_under(&home).unwrap();
8398 let mut second = run_state(RunStatus::Merged);
8399 second.id = "20260101-000000-supe".to_owned();
8400 second.save_under(&home).unwrap();
8401
8402 let mut t = task();
8403 t.runs = vec![first.id.clone(), second.id.clone()];
8404 t.status = TaskStatus::Done;
8405 queue.put(&mut t).unwrap();
8406
8407 resweep_superseded_attempts(&queue, &home);
8408 assert_eq!(
8409 RunState::load_under(&first.id, &home).unwrap().status,
8410 RunStatus::Blocked,
8411 "still live on the first pass, so still untouched"
8412 );
8413
8414 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8420 stale.driver_started_at = Some("1".to_owned());
8421 stale.save_under(&home).unwrap();
8422
8423 resweep_superseded_attempts(&queue, &home);
8424 assert_eq!(
8425 RunState::load_under(&first.id, &home).unwrap().status,
8426 RunStatus::Superseded,
8427 "the second pass catches up what the first one correctly skipped"
8428 );
8429 }
8430
8431 #[test]
8432 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8433 crate::run::pin_test_home();
8434 let mut first = run_state(RunStatus::Blocked);
8435 first.id = "20260101-000000-sup3".to_owned();
8436 first.save().unwrap();
8437 let mut second = run_state(RunStatus::Blocked);
8438 second.id = "20260101-000000-sup4".to_owned();
8439 second.save().unwrap();
8440
8441 let mut t = task();
8442 t.runs = vec![first.id.clone(), second.id.clone()];
8443 t.status = TaskStatus::Failed;
8447
8448 supersede_prior_runs(&t, &crate::run::home());
8449
8450 assert_eq!(
8451 RunState::load(&first.id).unwrap().status,
8452 RunStatus::Blocked
8453 );
8454 assert_eq!(
8455 RunState::load(&second.id).unwrap().status,
8456 RunStatus::Blocked
8457 );
8458 }
8459
8460 #[test]
8461 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
8462 crate::run::pin_test_home();
8466 let mut first = run_state(RunStatus::Blocked);
8467 first.id = "20260101-000000-sup5".to_owned();
8468 first.save().unwrap();
8469
8470 let mut t = task();
8471 t.runs = vec![first.id.clone()];
8472 t.status = TaskStatus::Done;
8473
8474 supersede_prior_runs(&t, &crate::run::home());
8475
8476 assert_eq!(
8477 RunState::load(&first.id).unwrap().status,
8478 RunStatus::Blocked,
8479 "a single-attempt task has no earlier run to supersede"
8480 );
8481 }
8482
8483 #[test]
8484 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
8485 crate::run::pin_test_home();
8492 let mut first = run_state(RunStatus::Blocked);
8493 first.id = "20260101-000000-supb".to_owned();
8494 first.save().unwrap();
8495 let mut second = run_state(RunStatus::Failed);
8496 second.id = "20260101-000000-supc".to_owned();
8497 second.save().unwrap();
8498
8499 let mut t = task();
8500 t.runs = vec![first.id.clone(), second.id.clone()];
8501 t.status = TaskStatus::Done;
8502
8503 supersede_prior_runs(&t, &crate::run::home());
8504
8505 assert_eq!(
8506 RunState::load(&first.id).unwrap().status,
8507 RunStatus::Blocked,
8508 "the task's last attempt never landed, so there is nothing here \
8509 actually superseding it"
8510 );
8511 }
8512
8513 #[test]
8514 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
8515 crate::run::pin_test_home();
8519 let mut failed = run_state(RunStatus::Failed);
8520 failed.id = "20260101-000000-sup6".to_owned();
8521 failed.save().unwrap();
8522 let mut noop = run_state(RunStatus::VerifiedNoop);
8523 noop.id = "20260101-000000-sup7".to_owned();
8524 noop.save().unwrap();
8525 let mut winner = run_state(RunStatus::Ready);
8526 winner.id = "20260101-000000-sup8".to_owned();
8527 winner.save().unwrap();
8528
8529 let mut t = task();
8530 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8531 t.status = TaskStatus::Done;
8532
8533 supersede_prior_runs(&t, &crate::run::home());
8534
8535 assert_eq!(
8536 RunState::load(&failed.id).unwrap().status,
8537 RunStatus::Failed
8538 );
8539 assert_eq!(
8540 RunState::load(&noop.id).unwrap().status,
8541 RunStatus::VerifiedNoop
8542 );
8543 }
8544
8545 fn approval_question(run: &str) -> ask::Question {
8546 ask::Question::new(
8547 run.to_owned(),
8548 land::APPROVAL_NODE.to_owned(),
8549 "land".to_owned(),
8550 "merge?".to_owned(),
8551 String::new(),
8552 vec!["merge".to_owned(), "hold".to_owned()],
8553 )
8554 }
8555
8556 #[test]
8557 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8558 crate::run::pin_test_home();
8559 let mut state = run_state(RunStatus::Landing);
8560 state.id = "20260101-000000-fre1".to_owned();
8561 state.parked = true;
8562 state.save().unwrap();
8563 ask::Questions::open()
8564 .put(&mut approval_question(&state.id))
8565 .unwrap();
8566
8567 let mut t = task();
8568 t.runs.push(state.id.clone());
8569 assert_eq!(
8570 land_resume_state(&t),
8571 LandResume::StillWaiting,
8572 "nobody has answered and the timeout has not passed"
8573 );
8574 }
8575
8576 #[test]
8577 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8578 crate::run::pin_test_home();
8579 let mut state = run_state(RunStatus::Gating);
8580 state.id = "20260101-000000-gt01".to_owned();
8581 state.parked = true;
8582 state.config.graph.answer_timeout = 60;
8583 state.github_text = Some(crate::run::GithubTextAsk {
8584 rewrite: None,
8585 fingerprint: "f".into(),
8586 title: true,
8587 body: false,
8588 categories: vec!["title-language".into()],
8589 question: None,
8590 asks: 1,
8591 resolved: false,
8592 chosen_title: None,
8593 choice: None,
8594 });
8595 state.save().unwrap();
8596 let store = ask::Questions::open();
8597 let mut q = ask::Question::new(
8598 state.id.clone(),
8599 crate::github_text::ASK_NODE.to_owned(),
8600 crate::github_text::ASK_SEAT.to_owned(),
8601 "s".into(),
8602 String::new(),
8603 vec!["use fallback".into()],
8604 );
8605 store.put(&mut q).unwrap();
8606 let mut t = task();
8607 t.runs.push(state.id.clone());
8608 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8609 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8610 store.put(&mut q).unwrap();
8611 assert_eq!(land_resume_state(&t), LandResume::Ready);
8612 assert!(!store.get(&q.id).unwrap().status.open());
8613 state.github_text.as_mut().unwrap().resolved = true;
8615 state.save().unwrap();
8616 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8617 }
8618
8619 #[test]
8620 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8621 crate::run::pin_test_home();
8626 let mut state = run_state(RunStatus::Landing);
8627 state.id = "20260101-000000-exp1".to_owned();
8628 state.parked = true;
8629 state.config.graph.answer_timeout = 60;
8630 state.save().unwrap();
8631
8632 let store = ask::Questions::open();
8633 let mut q = approval_question(&state.id);
8634 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8635 store.put(&mut q).unwrap();
8636
8637 let mut t = task();
8638 t.runs.push(state.id.clone());
8639 assert_eq!(
8640 land_resume_state(&t),
8641 LandResume::Ready,
8642 "an expired question must not be waited on forever"
8643 );
8644
8645 let after = store.get(&q.id).unwrap();
8646 assert!(
8647 !after.status.open(),
8648 "the question is abandoned, not silently ignored"
8649 );
8650 assert!(
8651 after.resolution().is_none(),
8652 "an abandoned question is not read as a decision"
8653 );
8654 }
8655
8656 #[test]
8657 fn reclaim_settles_a_running_task_against_its_last_run() {
8658 let mut t = task();
8659 t.start("20260904-000000-4043".to_owned());
8660 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8661 assert_eq!(
8662 t.status,
8663 TaskStatus::Done,
8664 "a run that actually finished must not stay `running` forever"
8665 );
8666 }
8667
8668 #[test]
8669 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8670 let mut t = task();
8671 t.start("20260904-000000-4043".to_owned());
8672 t.last_error = Some("earlier trouble".to_owned());
8673 let mut state = run_state(RunStatus::Judging);
8674 state.parked = true;
8675 reclaim(&mut t, Some(state), 1, "en");
8676 assert_eq!(t.status, TaskStatus::Parked);
8677 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8678 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8679 assert!(t.park_reason.is_some());
8680 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8681 assert!(!t.fresh_start);
8682 assert!(t.status.runnable());
8683 }
8684
8685 #[test]
8686 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8687 let mut t = task();
8688 t.start("20260904-000000-4043".to_owned());
8689 t.last_error = Some("earlier trouble".to_owned());
8690 settle(
8691 &mut t,
8692 Verdict {
8693 status: RunStatus::Judging,
8694 left_pr: false,
8695 quota_hit: false,
8696 parked: true,
8697 no_viable_candidates: false,
8698 },
8699 "parked after `judging`",
8700 1,
8701 );
8702 assert_eq!(t.status, TaskStatus::Parked);
8703 assert_eq!(t.attempts, 0);
8704 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8705 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8706 assert_eq!(t.schema, crate::queue::SCHEMA);
8707 assert!(
8708 crate::notices::task_held(&t).is_none(),
8709 "a park is not news"
8710 );
8711 t.start("20260904-000000-4043".to_owned());
8712 assert_eq!(t.park_reason, None);
8713 }
8714
8715 #[test]
8716 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8717 let mut t = task();
8721 t.start("20260904-000000-4043".to_owned());
8722 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8723 assert_eq!(t.status, TaskStatus::Failed);
8724 assert!(t.status.runnable());
8725 }
8726
8727 #[test]
8728 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8729 let mut t = task();
8730 t.start("20260904-000000-4043".to_owned());
8731 assert_eq!(t.attempts, 1);
8732 let mut run = run_state(RunStatus::Implementing);
8733 run.id = "20260904-000000-4043".to_owned();
8734 run.begin_pass();
8735 let cause = crate::run::ParkCause::Upgrade {
8736 from: "0.121.0".to_owned(),
8737 to: Some("0.121.1".to_owned()),
8738 };
8739 run.end_pass_parked(cause.clone());
8740 run.parked = true;
8741 let verdict = Verdict {
8742 status: run.status,
8743 left_pr: false,
8744 quota_hit: false,
8745 parked: true,
8746 no_viable_candidates: false,
8747 };
8748 let mut orphan = t.clone();
8751 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8752 assert_eq!(orphan.status, TaskStatus::Parked);
8753 assert_eq!(orphan.attempts, 0);
8754 assert_eq!(
8755 orphan.upgrade_resume_key(),
8756 Some((run.passes[0].started_at, orphan.id.as_str()))
8757 );
8758 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8759 assert_eq!(t.attempts, 0);
8760 assert_eq!(t.passes.len(), 1);
8761 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8762 assert_eq!(t.passes[0].parked, Some(cause));
8763 assert_eq!(t.passes[0].pass_index, 0);
8764 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8765 assert!(t.upgrade_resume_key().is_some());
8766
8767 t.start("20260904-000000-4043".to_owned());
8770 run.begin_pass();
8771 run.status = RunStatus::Failed;
8772 let verdict = Verdict {
8773 status: RunStatus::Failed,
8774 left_pr: false,
8775 quota_hit: false,
8776 parked: false,
8777 no_viable_candidates: false,
8778 };
8779 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8780 assert_eq!(t.passes.len(), 2);
8781 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8782 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8783 assert_eq!(t.passes[1].pass_index, 1);
8784 assert_eq!(t.attempts, 1);
8785 }
8786
8787 #[test]
8788 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8789 let mut t = task();
8790 t.start("20260904-000000-4043".to_owned());
8791 reclaim(&mut t, None, 2, "en");
8792 assert_eq!(t.status, TaskStatus::Held);
8793 assert!(
8794 t.last_error
8795 .as_deref()
8796 .is_some_and(|e| e.contains("running")),
8797 "the operator needs to know why this task was held"
8798 );
8799 }
8800
8801 #[test]
8802 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
8803 let dir = tempfile::tempdir().unwrap();
8804 let queue = Queue::at(dir.path().to_path_buf());
8805
8806 let mut orphaned = task();
8808 orphaned.id = "20260904-000000-orph".to_owned();
8809 orphaned.status = TaskStatus::Running;
8810 orphaned.attempts = 1;
8811 queue.put(&mut orphaned).unwrap();
8812
8813 let mut alive = task();
8814 alive.id = "20260904-000000-live".to_owned();
8815 alive.status = TaskStatus::Running;
8816 alive.attempts = 1;
8817 queue.put(&mut alive).unwrap();
8818 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
8819
8820 let mut queued = task();
8821 queued.id = "20260904-000000-wait".to_owned();
8822 queue.put(&mut queued).unwrap();
8823
8824 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8825 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
8826
8827 assert_eq!(
8828 queue.get(&orphaned.id).unwrap().status,
8829 TaskStatus::Held,
8830 "nothing was driving it and there was no run to recover"
8831 );
8832 assert_eq!(
8833 queue.get(&alive.id).unwrap().status,
8834 TaskStatus::Running,
8835 "a live claim must protect the task it belongs to"
8836 );
8837 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
8838 }
8839
8840 fn read_run_under(home: &Path, id: &str) -> RunState {
8846 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
8847 serde_json::from_str(&body).unwrap()
8848 }
8849
8850 #[test]
8851 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
8852 let dir = tempfile::tempdir().unwrap();
8853 let home = dir.path().to_path_buf();
8854 let now = Timestamp::now();
8855 let overrun_seat = || crate::run::ActiveSeat {
8856 node: "implement".to_owned(),
8857 started_at: now - jiff::SignedDuration::new(21_000, 0),
8858 timeout_secs: 3_600,
8859 attempt: 0,
8860 task: None,
8861 command: None,
8862 index: None,
8863 total: None,
8864 };
8865
8866 let mut dead = run_state(RunStatus::Implementing);
8867 dead.id = "20260101-000000-dead".to_owned();
8868 dead.active.insert("impl-A".to_owned(), overrun_seat());
8869 dead.driver_pid = Some(4242);
8872 dead.save_under(&home).unwrap();
8873
8874 let mut alive = run_state(RunStatus::Implementing);
8877 alive.id = "20260101-000000-aliv".to_owned();
8878 alive.active.insert("impl-A".to_owned(), overrun_seat());
8879 alive.save_under(&home).unwrap();
8880 let mut status = Status::new();
8881 status.current = vec![Current {
8882 task: "20260101-000000-task".to_owned(),
8883 run: alive.id.clone(),
8884 }];
8885 write_status_to(&home.join("daemon.json"), &status).unwrap();
8886
8887 let questions = Questions::at(home.join("questions"));
8891 let mut q = ask::Question::new(
8892 dead.id.clone(),
8893 "implement".to_owned(),
8894 "impl-A".to_owned(),
8895 "Which storage backend?".to_owned(),
8896 String::new(),
8897 vec!["SQLite".to_owned(), "Redis".to_owned()],
8898 );
8899 questions.put(&mut q).unwrap();
8900
8901 let abandoned = reclaim_abandoned_runs_with(
8902 &home,
8903 now,
8904 |pid| if pid == 4242 { Some(false) } else { None },
8905 |_| panic!("a query answering Dead outright needs no identity corroboration"),
8906 );
8907 assert_eq!(abandoned, vec![dead.id.clone()]);
8908
8909 let reloaded = read_run_under(&home, &dead.id);
8910 assert_eq!(reloaded.status, RunStatus::Failed);
8911 assert!(reloaded.active.is_empty());
8912 assert!(
8913 !questions.get(&q.id).unwrap().status.open(),
8914 "the failed run's own open question must be settled in the same pass"
8915 );
8916
8917 let still_alive = read_run_under(&home, &alive.id);
8918 assert_eq!(
8919 still_alive.status,
8920 RunStatus::Implementing,
8921 "a live daemon's claim protects it"
8922 );
8923 assert!(!still_alive.active.is_empty());
8924 }
8925
8926 #[test]
8936 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
8937 let dir = tempfile::tempdir().unwrap();
8938 let home = dir.path().to_path_buf();
8939 let now = Timestamp::now();
8940
8941 let mut manual = run_state(RunStatus::Reviewing);
8942 manual.id = "20260101-000000-manl".to_owned();
8943 manual.active.insert(
8944 "review-1".to_owned(),
8945 crate::run::ActiveSeat {
8946 node: "review".to_owned(),
8947 started_at: now - jiff::SignedDuration::new(21_000, 0),
8948 timeout_secs: 3_600,
8949 attempt: 0,
8950 task: None,
8951 command: None,
8952 index: None,
8953 total: None,
8954 },
8955 );
8956 manual.driver_pid = Some(4242);
8960 manual.driver_started_at = Some("1790000000".to_owned());
8961 manual.save_under(&home).unwrap();
8962
8963 let abandoned = reclaim_abandoned_runs_with(
8964 &home,
8965 now,
8966 |pid| if pid == 4242 { Some(true) } else { None },
8967 |pid| {
8968 if pid == 4242 {
8969 Some("1790000000".to_owned())
8970 } else {
8971 None
8972 }
8973 },
8974 );
8975 assert!(
8976 abandoned.is_empty(),
8977 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
8978 );
8979
8980 let reloaded = read_run_under(&home, &manual.id);
8981 assert_eq!(reloaded.status, RunStatus::Reviewing);
8982 assert!(!reloaded.active.is_empty());
8983 }
8984
8985 #[test]
8986 fn an_already_claimed_task_is_skipped_rather_than_failed() {
8987 let dir = tempfile::tempdir().unwrap();
8988 let queue = Queue::at(dir.path().to_path_buf());
8989 let mut only = task();
8990 queue.put(&mut only).unwrap();
8991
8992 let _elsewhere = queue.claim(&only.id).unwrap();
8993 let candidates = runnable(&queue);
8994 assert_eq!(candidates.len(), 1, "the task is still runnable");
8995 assert!(
8996 queue.claim(&candidates[0].id).is_err(),
8997 "the loop cannot take a claim somebody else holds"
8998 );
8999
9000 let after = queue.get(&only.id).unwrap();
9001 assert_eq!(after.status, TaskStatus::Queued);
9002 assert_eq!(
9003 after.attempts, 0,
9004 "losing the race is not an attempt at the task"
9005 );
9006 assert_eq!(after.last_error, None);
9007 }
9008
9009 #[test]
9010 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9011 let dir = tempfile::tempdir().unwrap();
9012 let path = dir.path().join("daemon.json");
9013
9014 let mut status = Status::new();
9015 status.idle = false;
9016 status.completed = 7;
9017 status.current = vec![Current {
9018 task: "20260902-000000-t111".to_owned(),
9019 run: "20260902-000001-r111".to_owned(),
9020 }];
9021 write_status_to(&path, &status).unwrap();
9022 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9023 assert_eq!(first.schema, SCHEMA);
9024 assert_eq!(first.pid, std::process::id());
9025 assert!(!first.idle);
9026 assert_eq!(first.completed, 7);
9027 assert_eq!(first.current, status.current);
9028 assert!(
9029 !path.with_extension("json.tmp").exists(),
9030 "the temp file is renamed, not left behind"
9031 );
9032
9033 std::thread::sleep(Duration::from_millis(5));
9034 status.updated_at = Timestamp::now();
9035 status.polls = 3;
9036 write_status_to(&path, &status).unwrap();
9037 let second: Status =
9038 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9039 assert!(
9040 second.updated_at > first.updated_at,
9041 "a reader can only detect staleness if the heartbeat moves"
9042 );
9043 assert_eq!(
9044 second.started_at, first.started_at,
9045 "the start time is not a heartbeat"
9046 );
9047 assert_eq!(second.polls, 3);
9048 }
9049
9050 #[test]
9051 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9052 let dir = tempfile::tempdir().unwrap();
9053
9054 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9055
9056 let mut status = Status::new();
9057 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9058 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9059 let stale = read_status(dir.path()).unwrap();
9060 assert!(
9061 !stale.running(Timestamp::now()),
9062 "a minute without a heartbeat is a dead daemon, not a busy one"
9063 );
9064 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9065
9066 status.updated_at = Timestamp::now();
9067 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9068 let fresh = read_status(dir.path()).unwrap();
9069 assert!(fresh.running(Timestamp::now()));
9070 }
9071
9072 #[test]
9073 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9074 let dir = tempfile::tempdir().unwrap();
9075 let now = Timestamp::now();
9076 let mine = "20260903-080619-01c2";
9077
9078 assert!(
9079 !is_working_on(dir.path(), mine, now),
9080 "no status file means nobody is working on anything"
9081 );
9082
9083 let mut status = Status::new();
9084 status.current = vec![Current {
9085 task: "20260903-080340-0167".to_owned(),
9086 run: mine.to_owned(),
9087 }];
9088 status.updated_at = now;
9089 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9090 assert!(is_working_on(dir.path(), mine, now));
9091 assert!(
9092 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9093 "a daemon busy with one run is not working on another"
9094 );
9095
9096 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9099 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9100 assert!(
9101 !is_working_on(dir.path(), mine, now),
9102 "a stale heartbeat is a dead daemon, so its run is a leftover"
9103 );
9104 }
9105
9106 #[test]
9107 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9108 let dir = tempfile::tempdir().unwrap();
9109 let now = Timestamp::now();
9110
9111 assert!(
9112 !is_working_on_short(dir.path(), "01c2", now),
9113 "no status file means nobody is working on anything"
9114 );
9115
9116 let mut status = Status::new();
9117 status.current = vec![Current {
9118 task: "20260903-080340-0167".to_owned(),
9119 run: "20260903-080619-01c2".to_owned(),
9120 }];
9121 status.updated_at = now;
9122 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9123 assert!(
9124 is_working_on_short(dir.path(), "01c2", now),
9125 "the run's short id is the last block of its full id"
9126 );
9127 assert!(
9128 !is_working_on_short(dir.path(), "3cbf", now),
9129 "a daemon busy with one worktree bay is not working on another"
9130 );
9131 }
9132
9133 #[test]
9134 fn a_newer_status_file_still_yields_a_reading() {
9135 let dir = tempfile::tempdir().unwrap();
9136 std::fs::write(
9139 dir.path().join("daemon.json"),
9140 serde_json::json!({
9141 "schema": 2,
9142 "updated_at": Timestamp::now().to_string(),
9143 "idle": true,
9144 "surprise": { "nested": [1, 2, 3] },
9145 })
9146 .to_string(),
9147 )
9148 .unwrap();
9149
9150 let reading = read_status(dir.path()).expect("a forward-compatible read");
9151 assert!(reading.running(Timestamp::now()));
9152 assert!(reading.idle);
9153 assert!(reading.current.is_empty());
9154 }
9155
9156 #[test]
9157 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9158 let dir = tempfile::tempdir().unwrap();
9164 std::fs::write(
9165 dir.path().join("daemon.json"),
9166 serde_json::json!({
9167 "schema": 1,
9168 "pid": 4242,
9169 "updated_at": Timestamp::now().to_string(),
9170 "idle": false,
9171 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9172 "completed": 3,
9173 "polls": 9,
9174 })
9175 .to_string(),
9176 )
9177 .unwrap();
9178
9179 let reading = read_status(dir.path()).expect("an older shape must still parse");
9180 assert!(reading.running(Timestamp::now()));
9181 assert_eq!(
9182 reading.current,
9183 vec![Current {
9184 task: "20260902-140501-aaaa".to_owned(),
9185 run: "20260902-140502-bbbb".to_owned(),
9186 }]
9187 );
9188 }
9189
9190 #[test]
9191 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9192 let dir = tempfile::tempdir().unwrap();
9193 std::fs::write(
9194 dir.path().join("daemon.json"),
9195 serde_json::json!({
9196 "schema": 1,
9197 "updated_at": Timestamp::now().to_string(),
9198 "idle": true,
9199 "current": null,
9200 })
9201 .to_string(),
9202 )
9203 .unwrap();
9204 let with_null = read_status(dir.path()).expect("null must still parse");
9205 assert!(with_null.current.is_empty());
9206
9207 std::fs::write(
9208 dir.path().join("daemon.json"),
9209 serde_json::json!({
9210 "schema": 1,
9211 "updated_at": Timestamp::now().to_string(),
9212 "idle": true,
9213 })
9214 .to_string(),
9215 )
9216 .unwrap();
9217 let absent = read_status(dir.path()).expect("a missing field must still parse");
9218 assert!(absent.current.is_empty());
9219 }
9220
9221 #[test]
9222 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9223 let fallback = Path::new("/default");
9224 let mut blank = task();
9225 blank.repo = PathBuf::new();
9226 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9227 let mut dot = task();
9228 dot.repo = PathBuf::from(".");
9229 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9230 assert_eq!(
9231 repo_for(&task(), fallback),
9232 PathBuf::from("/repo"),
9233 "a task that names a repository keeps it"
9234 );
9235 }
9236
9237 #[test]
9238 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9239 let dir = tempfile::tempdir().unwrap();
9240 let path = dir.path().join("explicit.toml");
9241 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9242 let opts = Opts {
9243 config: Some(path),
9244 merge: Some("none".into()),
9245 ..Default::default()
9246 };
9247 let mut t = task();
9248 t.set_implementers(
9249 2,
9250 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9251 .unwrap(),
9252 )
9253 .unwrap();
9254 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9255 assert_eq!(cfg.graph.implementers, 2);
9256 assert_eq!(cfg.graph.judges, 1);
9257 assert_eq!(cfg.merge.mode, MergeMode::None);
9258 t.set_implementers(1, &cfg).unwrap();
9259 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9260 apply_task_implementers(&mut cfg, &t);
9261 assert_eq!(cfg.graph.implementers, 1);
9262 assert_eq!(cfg.graph.judges, 1);
9263 assert_eq!(cfg.merge.mode, MergeMode::None);
9264 }
9265
9266 #[test]
9267 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9268 let dir = tempfile::tempdir().unwrap();
9269 let repo = dir.path().join("repo");
9270 std::fs::create_dir(&repo).unwrap();
9271 std::fs::write(
9272 repo.join("magi.toml"),
9273 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9274 )
9275 .unwrap();
9276 let daemon_config = dir.path().join("daemon.toml");
9277 std::fs::write(
9278 &daemon_config,
9279 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9280 )
9281 .unwrap();
9282 let opts = Opts {
9283 config: Some(daemon_config),
9284 merge: Some("pr".into()),
9285 ..Default::default()
9286 };
9287 let q = Queue::at(dir.path().join("queue"));
9288 let validation: Config =
9289 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9290 .unwrap();
9291 for record in ["{}", r#"{"candidates":2}"#] {
9293 let overrides = serde_json::from_str(record).unwrap();
9294 let filing = crate::direct::Filing {
9295 instruction: "work".into(),
9296 title: "direct".into(),
9297 repo: repo.clone(),
9298 source: crate::queue::Source::Human,
9299 solo: false,
9300 overrides,
9301 review_of: None,
9302 sources: Default::default(),
9303 };
9304 let filed =
9305 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9306 let crate::direct::Filed::Standalone {
9307 mut task,
9308 claim: _claim,
9309 } = filed
9310 else {
9311 panic!("local filing")
9312 };
9313 let before = prepare_for(&repo, &opts, &task).unwrap();
9314 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9315 assert_eq!(
9316 before.merge.mode,
9317 MergeMode::None,
9318 "daemon --merge must not leak"
9319 );
9320 for n in [2, 1, 2] {
9321 task.set_implementers(n, &validation).unwrap();
9322 q.put(&mut task).unwrap();
9323 task = q.get(&task.id).unwrap();
9324 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9325 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9326 apply_task_implementers(&mut cfg, &task);
9327 assert_eq!(cfg.graph.implementers, usize::from(n));
9328 assert_eq!(cfg.graph.judges, 2);
9329 assert_eq!(cfg.merge.mode, MergeMode::None);
9330 }
9331 }
9332 }
9333
9334 #[test]
9335 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9336 let cfg: Config =
9337 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9338 .unwrap();
9339 let mut t = task();
9340 t.review_of = Some("review-target".into());
9341 t.hold_manual(Some("retry".into()));
9342 let mut previous = RunState::new(
9343 PathBuf::from("/repo"),
9344 "main".into(),
9345 "abc1234".into(),
9346 "review".into(),
9347 cfg.clone(),
9348 );
9349 previous.config.graph.implementers = 2;
9350 for count in [1, 3] {
9351 t.set_implementers(count, &cfg).unwrap();
9352 t.release();
9353 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9354 for proposed in [
9355 Starter::Resume(previous.id.clone()),
9356 Starter::Review("recovery-target".into()),
9357 ] {
9358 assert_eq!(
9359 starter_with_implementers(&t, Some(&previous), proposed),
9360 Starter::Review("review-target".into())
9361 );
9362 }
9363 let mut configured = cfg.clone();
9364 apply_task_implementers(&mut configured, &t);
9365 assert_eq!(configured.graph.implementers, usize::from(count));
9366 assert_eq!(previous.config.graph.implementers, 2);
9367 }
9368 }
9369
9370 #[test]
9371 fn task_implementers_dispatch_and_resume_agree() {
9372 let mut t = task();
9373 let cfg: crate::config::Config =
9374 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9375 .unwrap();
9376 let mut run = RunState::new(
9377 PathBuf::from("/repo"),
9378 "main".into(),
9379 "abc1234".into(),
9380 "work".into(),
9381 cfg.clone(),
9382 );
9383 run.config.graph.implementers = 2;
9384 for n in [2, 1, 3, 2] {
9385 t.set_implementers(n, &cfg).unwrap();
9386 let mut dispatched = cfg.clone();
9387 apply_task_implementers(&mut dispatched, &t);
9388 assert_eq!(dispatched.graph.implementers, usize::from(n));
9389 assert_eq!(implementers_match(&t, &run), n == 2);
9390 for proposed in [
9391 Starter::Resume(run.id.clone()),
9392 Starter::Review("old-branch".into()),
9393 ] {
9394 let expected = if n == 2 {
9395 proposed.clone()
9396 } else {
9397 Starter::Start
9398 };
9399 assert_eq!(
9400 starter_with_implementers(&t, Some(&run), proposed),
9401 expected
9402 );
9403 }
9404 assert_eq!(
9405 run.config.graph.implementers, 2,
9406 "history must not be rewritten"
9407 );
9408 }
9409 t.solo = false;
9410 t.overrides = None;
9411 assert!(implementers_match(&t, &run));
9412 }
9413
9414 #[test]
9415 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9416 let mut solo_cfg = Config::default();
9422 solo_cfg.graph.implementers = 3;
9423 let mut solo_task = task();
9424 solo_task.solo = true;
9425 apply_task_implementers(&mut solo_cfg, &solo_task);
9426 assert_eq!(solo_cfg.graph.implementers, 1);
9427
9428 let mut plain_cfg = Config::default();
9429 plain_cfg.graph.implementers = 3;
9430 let plain_task = task();
9431 assert!(!plain_task.solo);
9432 apply_task_implementers(&mut plain_cfg, &plain_task);
9433 assert_eq!(
9434 plain_cfg.graph.implementers, 3,
9435 "a task that did not ask to run alone keeps the config's candidates"
9436 );
9437 }
9438
9439 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9440 QuotaLoss {
9441 seat: seat.into(),
9442 node: "judge".into(),
9443 at: at.parse().unwrap(),
9444 reset: reset.map(str::to_string),
9445 }
9446 }
9447
9448 #[test]
9449 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
9450 let old: Vec<QuotaLoss> = (1..=4)
9451 .map(|i| {
9452 loss(
9453 &format!("judge-{i}"),
9454 "2026-09-23T05:23:00Z",
9455 Some("2:40pm (Asia/Tokyo)"),
9456 )
9457 })
9458 .collect();
9459 let fresh = losses_this_attempt(&old, &old);
9460 assert!(fresh.is_empty());
9461 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
9462 }
9464
9465 #[test]
9466 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
9467 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
9468 let now = Timestamp::now();
9469 let mut after = old.clone();
9470 after.push(loss("judge-2", &now.to_string(), None));
9471 let fresh = losses_this_attempt(&old, &after);
9472 assert_eq!(fresh, vec![after[1].clone()]);
9473 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
9474 assert_eq!(
9475 until,
9476 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
9477 );
9478 }
9479
9480 #[test]
9481 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
9482 let before = vec![
9485 loss("judge-1", "2026-09-23T05:23:00Z", None),
9486 loss("judge-2", "2026-09-23T05:24:00Z", None),
9487 ];
9488 let after = vec![
9489 loss("judge-2", "2026-09-23T05:24:00Z", None),
9490 loss("judge-1", "2026-09-24T01:00:00Z", None),
9491 ];
9492 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
9493 }
9494
9495 #[test]
9496 fn merge_overrides_are_parsed_or_refused() {
9497 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
9498 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
9499 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
9500 assert!(merge_mode("squash").is_err());
9501 }
9502
9503 #[test]
9504 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
9505 let now = Timestamp::now();
9506 let fallback = Duration::from_secs(300);
9507 let cap = Duration::from_secs(1800);
9508
9509 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
9511
9512 let soon = now + jiff::SignedDuration::from_secs(600);
9514 assert_eq!(
9515 quota_wait(Some(soon), now, fallback, cap),
9516 Duration::from_secs(600)
9517 );
9518
9519 let past = now - jiff::SignedDuration::from_secs(60);
9522 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
9523
9524 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
9527 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
9528 }
9529
9530 #[test]
9531 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9532 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9533
9534 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9535 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9536
9537 let already_past =
9541 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9542 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9543
9544 assert!(
9545 parse_reset_hint("session limit reached", now, now).is_none(),
9546 "free text with no recognised shape is not guessed at"
9547 );
9548 assert!(
9549 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9550 "an unresolvable zone name is not guessed at either"
9551 );
9552 }
9553
9554 #[test]
9555 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9556 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9557
9558 let at = parse_reset_hint(
9559 "You've hit your usage limit. Visit \
9560 https://chatgpt.com/codex/settings/usage to purchase more \
9561 credits or try again at Sep 19th, 2026 5:10 PM.",
9562 now,
9563 now,
9564 )
9565 .expect("the codex reset wording is a recognised shape");
9566 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9567
9568 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9573 .expect("an explicit year needs no rollover");
9574 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9575
9576 assert!(
9577 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9578 "a two-digit year is not the documented shape and is not guessed at"
9579 );
9580 assert!(
9581 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9582 "a four-letter month name is not the documented three-letter abbreviation"
9583 );
9584 assert!(
9585 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9586 "an explicit zone on the dated shape is a format nobody has \
9587 documented, and is refused rather than guessed at as UTC"
9588 );
9589 }
9590
9591 #[test]
9592 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9593 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9594 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9595
9596 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9597 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9598
9599 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9600 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9601
9602 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9603 assert!(
9604 parse_reset_hint(bad, now, recorded).is_none(),
9605 "{bad:?} must not be guessed at"
9606 );
9607 }
9608 }
9609
9610 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9614 let config = dir.join("magi.toml");
9615 std::fs::write(
9616 &config,
9617 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9618 )
9619 .unwrap();
9620 let opts = Opts {
9621 poll: Duration::from_secs(30),
9622 config: Some(config),
9623 repo: dir.join("repo"),
9627 ..Opts::default()
9628 };
9629 let home = dir.join("home");
9638 let stamps = home.join("service-ticks");
9641 std::fs::create_dir_all(&stamps).unwrap();
9642 for service in ["waiter", "deputies", "release-watch"] {
9643 std::fs::write(
9644 stamps.join(service),
9645 Timestamp::now().as_second().to_string(),
9646 )
9647 .unwrap();
9648 }
9649 for operation in [
9650 "conductor",
9651 "maintenance",
9652 "housekeeping",
9653 "disk-pressure",
9654 "cache",
9655 "fetch",
9656 ] {
9657 std::fs::write(
9658 stamps.join(format!("daemon-{operation}")),
9659 serde_json::to_vec(&Timestamp::now()).unwrap(),
9660 )
9661 .unwrap();
9662 }
9663 let worktrees = dir.join("wt");
9664 (
9665 opts,
9666 Queue::at(dir.join("queue")),
9667 home.join("daemon.json"),
9668 home,
9669 worktrees,
9670 )
9671 }
9672
9673 #[test]
9674 fn a_stop_is_idempotent_and_once_set_stays_set() {
9675 let stop = Stop::new();
9676 assert!(!stop.stopped());
9677
9678 stop.stop();
9679 assert!(stop.stopped());
9680 stop.stop();
9681 assert!(stop.stopped(), "a second stop is not a toggle");
9682
9683 let shared = stop.clone();
9684 assert!(
9685 shared.stopped(),
9686 "a clone is the same stop; that is how the loop and its caller share one"
9687 );
9688 }
9689
9690 #[test]
9691 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9692 let stop = Stop::new();
9693 stop.busy.store(1, Ordering::SeqCst);
9694 assert!(
9695 !stop.finishing(),
9696 "a busy loop nobody has asked to stop is just running"
9697 );
9698
9699 stop.stop();
9700 assert!(
9701 stop.finishing(),
9702 "a stop asked for mid-run has not landed until the run is settled"
9703 );
9704
9705 stop.busy.store(0, Ordering::SeqCst);
9706 assert!(
9707 !stop.finishing(),
9708 "once the run is settled the stop has landed and there is nothing to finish"
9709 );
9710 }
9711
9712 #[test]
9713 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9714 let stop = Stop::new();
9715 stop.busy.store(2, Ordering::SeqCst);
9716 stop.stop();
9717 assert!(stop.finishing(), "two runs still in flight");
9718
9719 stop.busy.store(1, Ordering::SeqCst);
9720 assert!(
9721 stop.finishing(),
9722 "one run finished, but a sibling is still working"
9723 );
9724
9725 stop.busy.store(0, Ordering::SeqCst);
9726 assert!(
9727 !stop.finishing(),
9728 "the last run out is what actually lands the stop"
9729 );
9730 }
9731
9732 #[tokio::test]
9733 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9734 let dir = tempfile::tempdir().unwrap();
9735 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9736 let stop = Stop::new();
9737 stop.stop();
9738
9739 let began = std::time::Instant::now();
9740 tokio::time::timeout(
9741 Duration::from_secs(2),
9742 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9743 )
9744 .await
9745 .expect("a stopped loop must return, not sit out its poll interval")
9746 .expect("the loop's own setup and teardown must not fail");
9747 assert!(
9748 began.elapsed() < opts.poll,
9749 "returned only after {:?}, which is a poll interval, not a stop",
9750 began.elapsed()
9751 );
9752 }
9753
9754 #[tokio::test]
9755 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9756 let dir = tempfile::tempdir().unwrap();
9757 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9758 let stop = Stop::new();
9759
9760 let asker = {
9763 let stop = stop.clone();
9764 tokio::spawn(async move {
9765 tokio::time::sleep(Duration::from_millis(20)).await;
9766 stop.stop();
9767 })
9768 };
9769
9770 let began = std::time::Instant::now();
9771 tokio::time::timeout(
9772 Duration::from_secs(2),
9773 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9774 )
9775 .await
9776 .expect("a stop asked for while idle must wake the wait")
9777 .expect("the loop's own setup and teardown must not fail");
9778 asker.await.unwrap();
9779 assert!(
9780 began.elapsed() < opts.poll,
9781 "returned only after {:?}, so the stop waited on the sleep",
9782 began.elapsed()
9783 );
9784 }
9785
9786 #[tokio::test]
9787 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9788 let dir = tempfile::tempdir().unwrap();
9789 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9790 let stop = Stop::new();
9791 stop.stop();
9792
9793 tokio::time::timeout(
9794 Duration::from_secs(2),
9795 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9796 )
9797 .await
9798 .expect("a stopped loop must return")
9799 .expect("the loop's own setup and teardown must not fail");
9800
9801 assert!(
9802 home.is_dir(),
9803 "the loop did publish a status file, so its removal is the teardown and not an absence"
9804 );
9805 assert!(
9806 !status_file.exists(),
9807 "a stopped loop clears its status file"
9808 );
9809 assert!(
9810 read_status(&home).is_none(),
9811 "a reader must see no daemon at all, not a heartbeat that merely stopped"
9812 );
9813 }
9814
9815 #[tokio::test]
9816 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
9817 let dir = tempfile::tempdir().unwrap();
9818 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
9819
9820 let mut settled = RunState::new(
9821 dir.path().join("repo"),
9822 "main".to_owned(),
9823 "abc1234".to_owned(),
9824 "fixture".to_owned(),
9825 Config::default(),
9826 );
9827 settled.status = RunStatus::Ready;
9828 let run_dir = home.join("runs").join(&settled.id);
9829 std::fs::create_dir_all(&run_dir).unwrap();
9830 std::fs::write(
9831 run_dir.join("run.json"),
9832 serde_json::to_string_pretty(&settled).unwrap(),
9833 )
9834 .unwrap();
9835 let questions = Questions::at(home.join("questions"));
9836 let mut question = ask::Question::new(
9837 settled.id.clone(),
9838 "review".to_owned(),
9839 "reviewer-1".to_owned(),
9840 "Continue?".to_owned(),
9841 String::new(),
9842 Vec::new(),
9843 );
9844 questions.put(&mut question).unwrap();
9845
9846 run_service_worker(
9847 &home,
9848 serde_json::to_value(ServiceWorker {
9849 opts,
9850 queue: queue.root().to_path_buf(),
9851 worktrees_root: worktrees,
9852 operation: "housekeeping".to_owned(),
9853 })
9854 .unwrap(),
9855 )
9856 .await
9857 .unwrap();
9858
9859 assert_eq!(
9860 questions.get(&question.id).unwrap().status,
9861 ask::QuestionStatus::Abandoned,
9862 "the independent housekeeping worker settles ended-run questions"
9863 );
9864 }
9865
9866 #[test]
9867 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
9868 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9869
9870 assert!(
9871 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
9872 "never checked before: due at once"
9873 );
9874
9875 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
9876 assert!(
9877 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
9878 "well inside the interval: not due yet"
9879 );
9880
9881 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
9882 assert!(
9883 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
9884 "exactly at the edge: not yet due, same convention as `clean::due`"
9885 );
9886
9887 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9888 assert!(
9889 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
9890 "past the interval: due again"
9891 );
9892 }
9893
9894 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
9899 let config = dir.join("magi.toml");
9900 std::fs::write(
9906 &config,
9907 format!(
9908 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
9909 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
9910 cache_dir.display()
9911 ),
9912 )
9913 .unwrap();
9914 Opts {
9915 config: Some(config),
9916 repo: dir.join("repo"),
9917 ..Opts::default()
9918 }
9919 }
9920
9921 #[tokio::test]
9922 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
9923 let dir = tempfile::tempdir().unwrap();
9924 let home = dir.path().join("home");
9925 let cache_dir = dir.path().join("cache");
9926 std::fs::create_dir_all(&cache_dir).unwrap();
9927 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9928 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
9929
9930 let running = Stop::new();
9933 let mut last_checked = None;
9934 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
9935 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
9936 .await;
9937 assert_eq!(
9938 crate::disk::dir_size(&cache_dir),
9939 0,
9940 "over the cap on the first check ever: pruned at once, no idle queue required"
9941 );
9942 assert_eq!(last_checked, Some(t0));
9943
9944 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
9946 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
9947 maybe_prune_cache_between_runs(
9948 &opts.repo,
9949 &opts,
9950 &home,
9951 &running,
9952 &mut last_checked,
9953 too_soon,
9954 )
9955 .await;
9956 assert_eq!(
9957 crate::disk::dir_size(&cache_dir),
9958 10,
9959 "too soon since the last check: left alone rather than rescanned every call"
9960 );
9961 assert_eq!(
9962 last_checked,
9963 Some(t0),
9964 "an idle check does not reset the clock"
9965 );
9966
9967 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
9969 maybe_prune_cache_between_runs(
9970 &opts.repo,
9971 &opts,
9972 &home,
9973 &running,
9974 &mut last_checked,
9975 due_again,
9976 )
9977 .await;
9978 assert_eq!(
9979 crate::disk::dir_size(&cache_dir),
9980 0,
9981 "due again: pruned back under the cap"
9982 );
9983 }
9984
9985 #[tokio::test]
9993 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
9994 let dir = tempfile::tempdir().unwrap();
9995 let home = dir.path().join("home");
9996 let cache_dir = dir.path().join("cache");
9997 std::fs::create_dir_all(&cache_dir).unwrap();
9998 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
9999 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10000
10001 let stop = Stop::new();
10002 stop.stop();
10003 assert!(
10004 !stop.finishing(),
10005 "no run is in flight at a between-runs boundary, so nothing else \
10006 would tell the operator this stop had not taken effect yet"
10007 );
10008
10009 let mut last_checked = None;
10010 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10011 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10012 .await;
10013 assert_eq!(
10014 crate::disk::dir_size(&cache_dir),
10015 10,
10016 "over its cap, and due for the first check ever, but a stop outranks \
10017 it: the cap is a standing policy the next start measures again"
10018 );
10019 assert_eq!(
10020 last_checked, None,
10021 "a check that never happened must not claim the interval"
10022 );
10023 }
10024
10025 #[tokio::test]
10028 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10029 let dir = tempfile::tempdir().unwrap();
10030 let cache_dir = dir.path().join("cache");
10031 std::fs::create_dir_all(&cache_dir).unwrap();
10032 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10033
10034 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10035 opts.poll = Duration::from_millis(20);
10036 opts.max_attempts = 1_000;
10037
10038 let queue = Queue::at(dir.path().join("queue"));
10039 for i in 0..128 {
10040 let mut task = Task::new(
10041 format!("queued work {i}"),
10042 format!("queued work {i}"),
10043 opts.repo.clone(),
10044 Source::Human,
10045 );
10046 queue.put(&mut task).unwrap();
10047 }
10048
10049 let home = dir.path().join("home");
10050 let worktrees = dir.path().join("wt");
10051 run_service_worker(
10052 &home,
10053 serde_json::to_value(ServiceWorker {
10054 opts,
10055 queue: queue.root().to_path_buf(),
10056 worktrees_root: worktrees,
10057 operation: "cache".to_owned(),
10058 })
10059 .unwrap(),
10060 )
10061 .await
10062 .unwrap();
10063 let after = queue.list();
10064 assert_eq!(after.len(), 128);
10065 assert!(
10066 after
10067 .iter()
10068 .all(|task| task.status.runnable() && task.attempts == 0)
10069 );
10070
10071 assert_eq!(
10072 crate::disk::dir_size(&cache_dir),
10073 0,
10074 "an oversized cache must not be left to grow unboundedly just because the \
10075 queue kept the loop busy the whole time"
10076 );
10077 }
10078
10079 #[test]
10080 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10081 let dir = tempfile::tempdir().unwrap();
10082 let queue = Queue::at(dir.path().join("queue"));
10083 let questions = Questions::at(dir.path().join("questions"));
10084 let mut task = task();
10085 queue.put(&mut task).unwrap();
10086
10087 let mut task_question = ask::Question::new(
10088 task.id.clone(),
10089 crate::conduct::NODE.to_owned(),
10090 "conduct".to_owned(),
10091 "Which backend?".to_owned(),
10092 String::new(),
10093 Vec::new(),
10094 );
10095 questions.put(&mut task_question).unwrap();
10096 task.block(vec![task_question.id.clone()], None);
10097 queue.put(&mut task).unwrap();
10098
10099 let mut run_question = ask::Question::new(
10100 "20260101-000000-run1".to_owned(),
10101 "review".to_owned(),
10102 "reviewer-1".to_owned(),
10103 "Run question".to_owned(),
10104 String::new(),
10105 Vec::new(),
10106 );
10107 questions.put(&mut run_question).unwrap();
10108
10109 let mut coincidental = ask::Question::new(
10114 task.id.clone(),
10115 "review".to_owned(),
10116 "reviewer-1".to_owned(),
10117 "Unrelated review question".to_owned(),
10118 String::new(),
10119 Vec::new(),
10120 );
10121 questions.put(&mut coincidental).unwrap();
10122
10123 reconcile_task_questions(&queue, &questions);
10124 assert!(questions.get(&task_question.id).unwrap().status.open());
10125 assert!(questions.get(&run_question.id).unwrap().status.open());
10126 assert!(questions.get(&coincidental.id).unwrap().status.open());
10127
10128 task.release();
10129 queue.put(&mut task).unwrap();
10130 reconcile_task_questions(&queue, &questions);
10131 assert_eq!(
10132 questions.get(&task_question.id).unwrap().status,
10133 ask::QuestionStatus::Abandoned
10134 );
10135 assert!(
10136 questions.get(&run_question.id).unwrap().status.open(),
10137 "run questions remain the run janitor's responsibility"
10138 );
10139 assert!(
10140 questions.get(&coincidental.id).unwrap().status.open(),
10141 "a non-conductor question must not be abandoned just because its \
10142 run id coincides with a task id"
10143 );
10144 }
10145
10146 #[test]
10147 fn a_freshly_started_running_task_is_never_stalled() {
10148 let dir = tempfile::tempdir().unwrap();
10149 let mut t = task();
10150 t.start("run-1".to_owned());
10151 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10154 }
10155
10156 #[test]
10157 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10158 let dir = tempfile::tempdir().unwrap();
10159 let mut t = task();
10160 t.start("run-1".to_owned());
10161 t.updated_at = Timestamp::now()
10162 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10163 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10164 assert_eq!(
10165 stalled_tasks(
10166 &Queue::at(dir.path().join("q")),
10167 dir.path(),
10168 Timestamp::now()
10169 )
10170 .len(),
10171 0,
10172 "the task was never written to this queue"
10173 );
10174 }
10175
10176 #[test]
10177 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10178 let dir = tempfile::tempdir().unwrap();
10179 let mut t = task();
10180 t.id = "20260903-080340-0167".to_owned();
10181 t.start("20260903-080619-01c2".to_owned());
10182 t.updated_at = Timestamp::now()
10183 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10184
10185 let mut status = Status::new();
10186 status.current = vec![Current {
10187 task: t.id.clone(),
10188 run: "20260903-080619-01c2".to_owned(),
10189 }];
10190 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10191
10192 assert!(
10193 !is_stalled(&t, dir.path(), Timestamp::now()),
10194 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10195 );
10196 }
10197
10198 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10202 let path = queue.path_of(id);
10203 let body = std::fs::read_to_string(&path).unwrap();
10204 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10205 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10206 v["updated_at"] = serde_json::Value::String(old.to_string());
10207 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10208 }
10209
10210 #[test]
10211 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10212 let dir = tempfile::tempdir().unwrap();
10225 let queue = Queue::at(dir.path().join("queue"));
10226 let home = dir.path().join("home");
10227
10228 let mut t = task();
10229 t.id = "20260101-000001-lock".to_owned();
10230 t.start("run-1".to_owned());
10231 queue.put(&mut t).unwrap();
10232 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10233 std::fs::write(
10234 dir.path().join("queue").join(format!("{}.lock", t.id)),
10235 "not a pid",
10236 )
10237 .unwrap();
10238
10239 let now = Timestamp::now();
10240 assert!(
10241 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10242 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10243 and reclaim must leave the task alone"
10244 );
10245 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10246
10247 let stalled = stalled_tasks(&queue, &home, now);
10248 assert_eq!(
10249 stalled.len(),
10250 1,
10251 "reclaim's inability to claim it yet must not hide it from the conductor"
10252 );
10253 assert_eq!(stalled[0].id, t.id);
10254 }
10255
10256 #[test]
10257 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10258 let dir = tempfile::tempdir().unwrap();
10259 crate::run::set_home(dir.path().join("run-home"));
10260 let queue = Queue::at(dir.path().join("queue"));
10261 let home = dir.path().join("home");
10262 let questions = Questions::at(dir.path().join("questions"));
10263
10264 let mut t = task();
10265 t.id = "20260101-000003-dead".to_owned();
10266 t.start("missing-run".to_owned());
10267 queue.put(&mut t).unwrap();
10268 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10269
10270 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10273 assert_eq!(
10274 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10275 [&t.id]
10276 );
10277 assert_eq!(
10278 reclaim_orphaned_running(&queue, 2, &Default::default()),
10279 [t.id.clone()]
10280 );
10281 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10282
10283 crate::conduct::apply(
10286 &queue,
10287 &questions,
10288 &crate::conduct::Verdict {
10289 decisions: vec![crate::conduct::Decision {
10290 id: t.id.clone(),
10291 recovery: Some(crate::conduct::Recovery::Requeue),
10292 ..crate::conduct::Decision::default()
10293 }],
10294 },
10295 )
10296 .unwrap();
10297 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10298 }
10299
10300 #[test]
10301 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10302 let dir = tempfile::tempdir().unwrap();
10303 let queue = Queue::at(dir.path().join("queue"));
10304 let home = dir.path().join("home");
10305
10306 let mut fresh = task();
10307 fresh.id = "20260101-000001-aaaa".to_owned();
10308 fresh.start("run-1".to_owned());
10309 queue.put(&mut fresh).unwrap();
10310
10311 let mut old = task();
10312 old.id = "20260101-000002-bbbb".to_owned();
10313 old.start("run-2".to_owned());
10314 queue.put(&mut old).unwrap();
10315 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10316
10317 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10318 assert_eq!(stalled.len(), 1);
10319 assert_eq!(stalled[0].id, old.id);
10320 }
10321
10322 #[test]
10323 fn queued_and_finished_task_views_partition_by_status() {
10324 let dir = tempfile::tempdir().unwrap();
10325 let queue = Queue::at(dir.path().join("queue"));
10326
10327 let mut queued = task();
10328 queued.id = "20260101-000001-aaaa".to_owned();
10329 queue.put(&mut queued).unwrap();
10330
10331 let mut failed = task();
10332 failed.id = "20260101-000002-bbbb".to_owned();
10333 failed.start("run-1".to_owned());
10334 failed.fail("gate red", 5);
10335 queue.put(&mut failed).unwrap();
10336
10337 let mut held = task();
10338 held.id = "20260101-000003-cccc".to_owned();
10339 held.hold_machine(None);
10340 queue.put(&mut held).unwrap();
10341
10342 let mut running = task();
10343 running.id = "20260101-000004-dddd".to_owned();
10344 running.start("run-2".to_owned());
10345 queue.put(&mut running).unwrap();
10346
10347 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
10348 assert_eq!(queued_ids, [queued.id.clone()]);
10349
10350 let mut finished_ids: Vec<String> =
10351 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
10352 finished_ids.sort_unstable();
10353 let mut want = vec![failed.id.clone(), held.id.clone()];
10354 want.sort_unstable();
10355 assert_eq!(finished_ids, want);
10356 }
10357
10358 #[test]
10359 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
10360 for short in [false, true] {
10361 let dir = tempfile::tempdir().unwrap();
10362 let queue = Queue::at(dir.path().join("queue"));
10363 let questions = ask::Questions::at(dir.path().join("questions"));
10364 let mut running = task();
10365 running.id = "20261013-192213-feb2".into();
10366 running.start("existing-run".into());
10367 queue.put(&mut running).unwrap();
10368 let mut middle = task();
10369 middle.id = "20261013-193611-ca5b".into();
10370 middle.block(
10371 vec![if short {
10372 "feb2".into()
10373 } else {
10374 running.id.clone()
10375 }],
10376 None,
10377 );
10378 queue.put(&mut middle).unwrap();
10379 let mut waiting = task();
10380 waiting.id = "20261013-202849-ee37".into();
10381 waiting.block(
10382 vec![if short {
10383 "ca5b".into()
10384 } else {
10385 middle.id.clone()
10386 }],
10387 Some("wait for implementation".into()),
10388 );
10389 queue.put(&mut waiting).unwrap();
10390 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
10391 let inv = crate::blockers::Inventory::load(&queue, &questions);
10392 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
10393 assert!(inv.stuck().is_empty());
10394 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10395 assert!(report.quarantined.is_empty());
10396 assert!(report.asked.is_empty());
10397 resolve_blockers(&queue, &questions);
10398 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10399 assert!(questions.list().is_empty());
10400 running.succeed();
10403 queue.put(&mut running).unwrap();
10404 resolve_blockers(&queue, &questions);
10405 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
10406 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10407 middle = queue.get(&middle.id).unwrap();
10408 middle.succeed();
10409 queue.put(&mut middle).unwrap();
10410 resolve_blockers(&queue, &questions);
10411 let after = queue.get(&waiting.id).unwrap();
10412 assert_eq!(after.status, TaskStatus::Queued);
10413 assert!(after.blocked_by.is_empty());
10414 assert_eq!(after.instruction, waiting.instruction);
10415 }
10416 }
10417
10418 #[test]
10419 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
10420 for future_schema in [false, true] {
10421 let name = "20261013-193611-ca5b";
10422 let mut future = task();
10423 future.id = name.into();
10424 future.schema = crate::queue::SCHEMA + 1;
10425 let contents = if future_schema {
10426 serde_json::to_string(&future).unwrap()
10427 } else {
10428 "corrupt".into()
10429 };
10430 let dir = tempfile::tempdir().unwrap();
10431 let queue = Queue::at(dir.path().join("queue"));
10432 let questions = ask::Questions::at(dir.path().join("questions"));
10433 let mut waiting = task();
10434 waiting.id = "20261013-202849-ee37".into();
10435 waiting.block(vec!["ca5b".into()], None);
10436 queue.put(&mut waiting).unwrap();
10437 std::fs::write(queue.path_of(name), contents).unwrap();
10438 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
10439 assert!(
10440 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
10441 );
10442 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10443 assert!(report.asked.is_empty());
10444 resolve_blockers(&queue, &questions);
10445 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10446 assert!(questions.list().is_empty());
10447 }
10448 let dir = tempfile::tempdir().unwrap();
10449 let queue = Queue::at(dir.path().join("queue"));
10450 let questions = ask::Questions::at(dir.path().join("questions"));
10451 let mut waiting = task();
10452 waiting.id = "20261013-202849-ee37".into();
10453 waiting.block(vec!["ca5b".into()], None);
10454 queue.put(&mut waiting).unwrap();
10455 let mut done = task();
10456 done.id = "20261013-193611-ca5b".into();
10457 done.succeed();
10458 queue.put(&mut done).unwrap();
10459 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
10462 assert!(queue.get("ca5b").is_err());
10463 assert_eq!(
10464 crate::queue::dependency(&queue, &questions, "ca5b"),
10465 crate::queue::Dependency::Ambiguous
10466 );
10467 resolve_blockers(&queue, &questions);
10468 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10469 assert!(report.asked.is_empty());
10470 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10471 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
10472 }
10473
10474 #[test]
10475 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
10476 for held in [false, true] {
10477 for question in [false, true] {
10478 for reference in ["ca5b", "20261013-193611-ca5b"] {
10479 let dir = tempfile::tempdir().unwrap();
10480 let queue = Queue::at(dir.path().join("queue"));
10481 let questions = ask::Questions::at(dir.path().join("questions"));
10482 let mut prerequisite = task();
10483 prerequisite.id = "20261013-193611-ca5b".into();
10484 prerequisite.start("existing-run".into());
10485 queue.put(&mut prerequisite).unwrap();
10486 let mut waiting = task();
10487 waiting.id = "20261013-202849-ee37".into();
10488 if held {
10489 waiting.hold_machine(Some("waiting for separate consent".into()));
10490 }
10491 queue.put(&mut waiting).unwrap();
10492 let reason = "Deleting saved data requires separate owner consent";
10493 crate::conduct::apply(
10494 &queue,
10495 &questions,
10496 &crate::conduct::Verdict {
10497 decisions: vec![crate::conduct::Decision {
10498 id: waiting.id.clone(),
10499 blocked_by: vec![reference.into()],
10500 question: question.then(|| "May the saved data be deleted?".into()),
10501 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
10502 reason: Some(reason.into()),
10503 ..crate::conduct::Decision::default()
10504 }],
10505 },
10506 )
10507 .unwrap();
10508 let before = queue.get(&waiting.id).unwrap();
10509 if question {
10510 assert_eq!(before.status, TaskStatus::Blocked);
10511 let q = questions.get(&before.blocked_by[0]).unwrap();
10512 assert_eq!(q.summary, "May the saved data be deleted?");
10513 assert!(q.status.open());
10514 } else {
10515 assert_eq!(before.status, TaskStatus::Held);
10516 let recorded_reason = before.hold_reason.as_deref().unwrap();
10517 assert!(recorded_reason.starts_with(reason));
10518 if held {
10519 assert!(recorded_reason.contains("waiting for separate consent"));
10520 }
10521 }
10522 prerequisite.succeed();
10523 queue.put(&mut prerequisite).unwrap();
10524 resolve_blockers(&queue, &questions);
10525 let after = queue.get(&waiting.id).unwrap();
10526 assert_eq!(after.status, before.status);
10527 assert_eq!(after.blocked_by, before.blocked_by);
10528 assert_eq!(after.hold_reason, before.hold_reason);
10529 assert_eq!(after.instruction, waiting.instruction);
10530 if question {
10531 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10532 }
10533 assert!(
10534 queue.next_runnable().is_none(),
10535 "dependency completion is not consent"
10536 );
10537 }
10538 }
10539 }
10540 }
10541
10542 #[test]
10543 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10544 let dir = tempfile::tempdir().unwrap();
10545 let queue = Queue::at(dir.path().join("queue"));
10546 let questions = ask::Questions::at(dir.path().join("questions"));
10547
10548 let mut dep = task();
10549 dep.id = "20260101-000001-dep0".to_owned();
10550 dep.succeed();
10551 queue.put(&mut dep).unwrap();
10552
10553 let mut still_going = task();
10554 still_going.id = "20260101-000002-dep1".to_owned();
10555 queue.put(&mut still_going).unwrap();
10556
10557 let mut blocked = task();
10558 blocked.id = "20260101-000003-main".to_owned();
10559 blocked.block(
10560 vec![dep.id.clone(), still_going.id.clone()],
10561 Some("waits on both".to_owned()),
10562 );
10563 queue.put(&mut blocked).unwrap();
10564
10565 resolve_blockers(&queue, &questions);
10566
10567 let after = queue.get(&blocked.id).unwrap();
10568 assert_eq!(
10569 after.status,
10570 TaskStatus::Blocked,
10571 "one dependency is still outstanding"
10572 );
10573 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10574 }
10575
10576 #[test]
10577 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10578 let dir = tempfile::tempdir().unwrap();
10579 let queue = Queue::at(dir.path().join("queue"));
10580 let questions = ask::Questions::at(dir.path().join("questions"));
10581
10582 let mut q = crate::ask::Question::new(
10583 "20260101-000001-main".to_owned(),
10584 crate::conduct::NODE.to_owned(),
10585 "conduct".to_owned(),
10586 "Which backend?".to_owned(),
10587 String::new(),
10588 Vec::new(),
10589 );
10590 questions.put(&mut q).unwrap();
10591 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10592 .unwrap();
10593 questions.put(&mut q).unwrap();
10594
10595 let mut blocked = task();
10596 blocked.id = "20260101-000001-main".to_owned();
10597 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10598 queue.put(&mut blocked).unwrap();
10599
10600 resolve_blockers(&queue, &questions);
10601
10602 let after = queue.get(&blocked.id).unwrap();
10603 assert_eq!(
10604 after.status,
10605 TaskStatus::Queued,
10606 "the only blocker resolved"
10607 );
10608 assert_eq!(after.answers.len(), 1);
10609 assert_eq!(after.answers[0].question, "Which backend?");
10610 assert_eq!(after.answers[0].answer, "SQLite");
10611
10612 let instruction = instruction_for(&after);
10614 assert!(instruction.contains("Which backend?"));
10615 assert!(instruction.contains("SQLite"));
10616 }
10617
10618 #[test]
10619 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10620 let dir = tempfile::tempdir().unwrap();
10621 let queue = Queue::at(dir.path().join("queue"));
10622 let questions = ask::Questions::at(dir.path().join("questions"));
10623
10624 let mut q = crate::ask::Question::new(
10625 "20260101-000001-main".to_owned(),
10626 crate::conduct::NODE.to_owned(),
10627 "conduct".to_owned(),
10628 "Is the setup done?".to_owned(),
10629 String::new(),
10630 Vec::new(),
10631 );
10632 q.abandon("no answer within 60s of asking");
10633 questions.put(&mut q).unwrap();
10634
10635 let mut blocked = task();
10636 blocked.id = "20260101-000001-main".to_owned();
10637 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10638 queue.put(&mut blocked).unwrap();
10639
10640 resolve_blockers(&queue, &questions);
10641
10642 let after = queue.get(&blocked.id).unwrap();
10643 assert_eq!(
10644 after.status,
10645 TaskStatus::Held,
10646 "never left blocked on nothing"
10647 );
10648 assert!(!after.operator_held(), "a machine hold, for triage");
10649 assert!(
10650 after
10651 .hold_reason
10652 .as_deref()
10653 .unwrap_or_default()
10654 .contains("went unanswered")
10655 );
10656 }
10657
10658 #[test]
10659 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10660 let dir = tempfile::tempdir().unwrap();
10666 let queue = Queue::at(dir.path().join("queue"));
10667 let questions = ask::Questions::at(dir.path().join("questions"));
10668
10669 let mut q = crate::ask::Question::new(
10670 "20260101-000001-main".to_owned(),
10671 crate::conduct::NODE.to_owned(),
10672 "conduct".to_owned(),
10673 "How should this be handled?".to_owned(),
10674 String::new(),
10675 Vec::new(),
10676 );
10677 questions.put(&mut q).unwrap();
10678 q.answer(crate::ask::Answer::Text(
10679 "leave it held, a human will look at it later".to_owned(),
10680 ))
10681 .unwrap();
10682 questions.put(&mut q).unwrap();
10683
10684 let mut held = task();
10685 held.id = "20260101-000001-main".to_owned();
10686 held.hold_machine(Some("out of attempts".to_owned()));
10687 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10688 queue.put(&mut held).unwrap();
10689
10690 resolve_blockers(&queue, &questions);
10691
10692 let after = queue.get(&held.id).unwrap();
10693 assert_eq!(after.status, TaskStatus::Held);
10694 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10695 assert_eq!(
10696 after.answers[0].answer,
10697 "leave it held, a human will look at it later"
10698 );
10699 }
10700
10701 #[test]
10702 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10703 let dir = tempfile::tempdir().unwrap();
10704 let queue = Queue::at(dir.path().join("queue"));
10705 let questions = ask::Questions::at(dir.path().join("questions"));
10706 let mut dep = task();
10707 dep.id = "20260101-000001-gone".to_owned();
10708 queue.put(&mut dep).unwrap();
10709 let mut blocked = task();
10710 blocked.id = "20260101-000003-main".to_owned();
10711 blocked.block(vec![dep.id.clone()], None);
10712 queue.put(&mut blocked).unwrap();
10713
10714 let claim = queue.claim(&blocked.id).unwrap();
10716 queue.remove(&dep.id, false, &questions).unwrap();
10717 drop(claim);
10718 resolve_blockers(&queue, &questions);
10719
10720 let after = queue.get(&blocked.id).unwrap();
10721 assert_eq!(after.status, TaskStatus::Queued);
10722 assert!(after.blocked_by.is_empty());
10723 }
10724
10725 #[test]
10726 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10727 let dir = tempfile::tempdir().unwrap();
10733 let queue = Queue::at(dir.path().join("queue"));
10734 let questions = ask::Questions::at(dir.path().join("questions"));
10735
10736 let mut still_going = task();
10737 still_going.id = "20260101-000002-dep1".to_owned();
10738 queue.put(&mut still_going).unwrap();
10739
10740 let mut blocked = task();
10741 blocked.id = "20260101-000003-main".to_owned();
10742 blocked.block(
10743 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10744 Some("waits on both".to_owned()),
10745 );
10746 queue.put(&mut blocked).unwrap();
10747
10748 resolve_blockers(&queue, &questions);
10749
10750 let after = queue.get(&blocked.id).unwrap();
10751 assert_eq!(
10752 after.status,
10753 TaskStatus::Held,
10754 "a missing dependency must not leave the task blocked forever"
10755 );
10756 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10757 assert!(after.blocked_by.is_empty());
10758 let reason = after.hold_reason.as_deref().unwrap_or_default();
10759 assert!(
10760 reason.contains("20260101-000001-gone"),
10761 "the missing id must be named so an operator can tell what happened: {reason}"
10762 );
10763 assert!(
10764 reason.contains(&still_going.id),
10765 "the still-valid dependency must not silently vanish from the record: {reason}"
10766 );
10767 }
10768
10769 #[test]
10770 fn instruction_for_is_unchanged_without_any_answers() {
10771 let t = task();
10772 assert_eq!(instruction_for(&t), t.instruction);
10773 }
10774
10775 #[test]
10776 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10777 let dir = tempfile::tempdir().unwrap();
10778 let q = Queue::at(dir.path().join("queue"));
10779 let src = dir.path().join("shot.png");
10780 std::fs::write(&src, "x").unwrap();
10781 let mut t = task();
10782 q.attach(&mut t, &[src]).unwrap();
10783 let paths = task_attachments(&q, &t).unwrap();
10784 assert_eq!(paths.len(), 1);
10785 assert!(paths[0].is_absolute() && paths[0].is_file());
10786 std::fs::remove_file(&paths[0]).unwrap();
10787 let err = task_attachments(&q, &t).unwrap_err().to_string();
10788 assert!(err.contains("shot.png"), "{err}");
10789 }
10790
10791 #[tokio::test]
10792 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
10793 let dir = tempfile::tempdir().unwrap();
10794 let queue_path = dir.path().join("queue");
10795 let queue = Queue::at(queue_path.clone());
10796 let mut t = task();
10797 t.source = Source::Agent {
10798 run: "originating-chat".into(),
10799 node: crate::queue::CHAT_NODE.into(),
10800 };
10801 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
10802 stage: "config-fetch",
10803 target: "origin/main".into(),
10804 });
10805 record_config_failure(&mut t, &fetch);
10806 t.hold_machine(Some(format!("{fetch:#}")));
10807 let fetch_key = t.chat_notice_due().unwrap();
10808 t.chat_notices.push(fetch_key.clone());
10809 queue.put(&mut t).unwrap();
10810
10811 let queue = Queue::at(queue_path);
10812 let mut t = queue.get(&t.id).unwrap();
10813 let a = dir.path().join("a.toml");
10814 let b = dir.path().join("b.toml");
10815 for (path, text, new_cause) in [
10816 (&a, "[graph", true),
10817 (&a, "[verify", false),
10818 (&a, "[graph]\nimplementers = 'invalid-type'", true),
10819 (&b, "[graph]\nimplementers = 'invalid-type'", true),
10820 ] {
10821 t.release_operator("test", None);
10822 std::fs::write(path, text).unwrap();
10823 let error = Config::discover_fetched(dir.path(), Some(path))
10824 .await
10825 .unwrap_err();
10826 record_config_failure(&mut t, &error);
10827 t.hold_machine(Some(format!("{error:#}")));
10828 let due = t.chat_notice_due();
10829 assert_eq!(due.is_some(), new_cause, "{error:#}");
10830 if let Some(key) = due {
10831 assert_ne!(key, fetch_key);
10832 t.chat_notices.push(key);
10833 }
10834 }
10835 t.succeed();
10836 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
10837 }
10838
10839 #[test]
10840 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
10841 let dir = tempfile::tempdir().unwrap();
10842 let queue_path = dir.path().join("queue");
10843 let queue = Queue::at(queue_path.clone());
10844 let mut t = task();
10845 t.source = crate::queue::Source::Agent {
10846 run: "originating-chat".into(),
10847 node: crate::queue::CHAT_NODE.into(),
10848 };
10849 let a = dir.path().join("a.png");
10850 let b = dir.path().join("b.png");
10851 std::fs::write(&a, "a").unwrap();
10852 std::fs::write(&b, "b").unwrap();
10853 queue.attach(&mut t, &[a, b]).unwrap();
10854 let paths = task_attachments(&queue, &t).unwrap();
10855 for path in &paths {
10856 std::fs::remove_file(path).unwrap();
10857 }
10858 let missing = task_attachments(&queue, &t).unwrap_err();
10859 t.attempts += 1;
10860 let reason = missing.to_string();
10861 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10862 let first = t.chat_notice_due().expect("first attachment notice");
10863 t.chat_notices.push(first.clone());
10864 queue.put(&mut t).unwrap();
10865
10866 let queue = Queue::at(queue_path.clone());
10867 let mut t = queue.get(&t.id).unwrap();
10868 assert!(t.chat_notice_due().is_none(), "restart does not replay");
10869 t.release_operator("test", None);
10870 t.attempts += 1;
10871 let missing = task_attachments(&queue, &t).unwrap_err();
10872 t.fail_startup_target(
10873 "startup-attachments",
10874 missing.name,
10875 "same missing attachment with regenerated wording",
10876 1,
10877 );
10878 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
10879
10880 std::fs::write(&paths[0], "restored").unwrap();
10881 t.release_operator("test", None);
10882 t.attempts += 1;
10883 let missing = task_attachments(&queue, &t).unwrap_err();
10884 let reason = missing.to_string();
10885 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
10886 let second = t
10887 .chat_notice_due()
10888 .expect("different attachment must notify");
10889 assert_ne!(first, second);
10890 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
10891 }
10892
10893 #[test]
10894 fn resumed_instruction_is_unchanged_without_any_answers() {
10895 let t = task();
10896 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
10897 }
10898
10899 #[test]
10900 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
10901 let mut t = task();
10902 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10903 let old = t.instruction.clone();
10907
10908 let refreshed = resumed_instruction(&old, &t);
10909 assert!(refreshed.starts_with(&old), "the original text is kept");
10910 assert!(refreshed.contains("Which backend?"));
10911 assert!(refreshed.contains("SQLite"));
10912 }
10913
10914 #[test]
10915 fn resumed_instruction_keeps_an_original_answers_heading() {
10916 let mut t = task();
10917 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
10918 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10919
10920 let refreshed = resumed_instruction(&t.instruction, &t);
10921
10922 assert!(
10923 refreshed.starts_with(&t.instruction),
10924 "an answers heading in the original instruction is not the appended block"
10925 );
10926 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
10927 assert!(refreshed.contains("Which backend?"));
10928 assert!(refreshed.contains("SQLite"));
10929
10930 let repeated = resumed_instruction(&refreshed, &t);
10931 assert_eq!(
10932 repeated, refreshed,
10933 "only the final appended block is refreshed"
10934 );
10935 }
10936
10937 #[test]
10938 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
10939 let mut t = task();
10940 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10941
10942 let once = resumed_instruction(&t.instruction, &t);
10946 let twice = resumed_instruction(&once, &t);
10947 assert_eq!(once, twice);
10948 assert_eq!(once.matches("Which backend?").count(), 1);
10949
10950 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
10952 let refreshed = resumed_instruction(&once, &t);
10953 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
10954 assert!(refreshed.contains("Which backend?"));
10955 assert!(refreshed.contains("Which cache?"));
10956 }
10957
10958 #[test]
10959 fn prepare_instruction_covers_all_three_starters() {
10960 let mut t = task();
10961 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
10962
10963 assert_eq!(
10966 prepare_instruction(&Starter::Start, None, &t),
10967 Some(instruction_for(&t))
10968 );
10969
10970 let old = t.instruction.clone();
10973 assert_eq!(
10974 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
10975 Some(resumed_instruction(&old, &t))
10976 );
10977
10978 assert_eq!(
10982 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
10983 None
10984 );
10985 }
10986
10987 #[test]
10988 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
10989 assert_eq!(
10990 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
10991 Starter::Review("magi/eba2/A".to_owned())
10992 );
10993 }
10994
10995 #[test]
10996 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
10997 assert_eq!(
10998 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
10999 Starter::Start,
11000 "a vanished review branch must not fall back to resuming the old run either"
11001 );
11002 }
11003
11004 #[test]
11005 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11006 let mut t = task();
11007 t.start("old-run".to_owned());
11008 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11009 t.release();
11010 let branch = t.review_branch.take();
11011 assert_eq!(
11012 choose_starter(branch.as_deref(), true, Some("old-run")),
11013 Starter::Review("magi/eba2/A".to_owned()),
11014 "a review wins over resuming the old run"
11015 );
11016 }
11017
11018 #[test]
11019 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11020 assert_eq!(
11021 choose_starter(None, false, Some("some-run")),
11022 Starter::Resume("some-run".to_owned())
11023 );
11024 assert_eq!(choose_starter(None, false, None), Starter::Start);
11025 }
11026
11027 #[test]
11028 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11029 let mut released = task();
11030 released.start("stalled-run".to_owned());
11031 released.requeue();
11032 let unfinished = (!released.fresh_start)
11033 .then(|| Some("stalled-run".to_owned()))
11034 .flatten();
11035 assert_eq!(
11036 choose_starter(None, false, unfinished.as_deref()),
11037 Starter::Start,
11038 "release keeps run history but must not resume it"
11039 );
11040 assert_eq!(released.runs, ["stalled-run"]);
11041 }
11042
11043 fn stopped_public_run(status: RunStatus) -> RunState {
11044 let mut state = run_state(status);
11045 state.driver_exited = true;
11046 state.config.graph.land = true;
11047 state.config.graph.review_rounds = 1;
11048 state.reviews = vec![review_round(1)];
11049 state.merge = Some(crate::run::MergeOutcome {
11050 mode: MergeMode::Pr,
11051 ok: false,
11052 detail: "landing stopped".into(),
11053 empty: false,
11054 });
11055 state.pr = Some(crate::run::PrRecord {
11056 url: "https://example.test/pull/1".into(),
11057 number: 1,
11058 state: "open".into(),
11059 checks: "red".into(),
11060 round: 1,
11061 rounds: 1,
11062 red_at_merge: Vec::new(),
11063 });
11064 state
11065 }
11066
11067 #[test]
11068 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11069 for status in [RunStatus::Blocked, RunStatus::Gating] {
11070 let home = tempfile::tempdir().unwrap();
11071 let mut state = stopped_public_run(status);
11072 state.save_under(home.path()).unwrap();
11073 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11074 assert!(!exhausted_review_budget(&state));
11075 let mut task = held_task_with(&state.id);
11076 task.release();
11077 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11078 assert_eq!(
11079 choose_starter(None, false, unfinished.as_deref()),
11080 Starter::Resume(state.id.clone())
11081 );
11082 let question = action_question(&state.id, resume_action(&state.id));
11083 assert_eq!(
11084 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11085 ActionDecision::Resume(state.id.clone())
11086 );
11087 assert_eq!(
11089 crate::resume::validate(home.path(), &state.id).unwrap().id,
11090 state.id
11091 );
11092 let queue = Queue::at(home.path().join("queue"));
11093 let mut controlled = Task::new(
11094 "resume land".into(),
11095 state.instruction.clone(),
11096 state.repo.clone(),
11097 Source::Agent {
11098 run: "talk".into(),
11099 node: crate::queue::CHAT_NODE.into(),
11100 },
11101 );
11102 controlled.runs.push(state.id.clone());
11103 controlled.hold_machine(Some("land stopped".into()));
11104 queue.put(&mut controlled).unwrap();
11105 let resumed =
11106 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11107 assert_eq!(resumed.status, TaskStatus::Queued);
11108 assert_eq!(
11109 resumed.resume_override.unwrap().pinned_run,
11110 Some(state.id.clone())
11111 );
11112 assert_eq!(resumed.attempts, 0);
11113 }
11114 }
11115
11116 #[test]
11117 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11118 for review_rounds in [0, 1] {
11119 let home = tempfile::tempdir().unwrap();
11120 let mut state = run_state(RunStatus::Blocked);
11121 state.config.graph.review_rounds = review_rounds;
11122 if review_rounds > 0 {
11123 let mut review = review_round(1);
11124 review.clean = true;
11125 state.reviews = vec![review];
11126 }
11127 state.gate_ran = true;
11128 state.gate = vec![crate::run::CommandOutcome {
11129 command: "verify".into(),
11130 code: Some(1),
11131 output_tail: "gate failed after repair budget exhausted".into(),
11132 duration_ms: 0,
11133 resource_blocked: false,
11134 }];
11135 state.save_under(home.path()).unwrap();
11136 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11137 assert!(exhausted_review_budget(&state));
11138 let mut task = held_task_with(&state.id);
11139 task.release();
11140 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11141 let question = action_question(&state.id, resume_action(&state.id));
11142 assert_ne!(
11143 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11144 ActionDecision::Resume(state.id.clone())
11145 );
11146 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11147
11148 state.gate_ran = false;
11150 state.gate.clear();
11151 assert!(!exhausted_review_budget(&state));
11152 assert_eq!(
11153 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11154 Some(state.id)
11155 );
11156 }
11157 }
11158
11159 #[test]
11160 fn resource_blocked_verification_resumes_the_same_run() {
11161 for (kind, stage) in [
11162 ("cache", "clean"),
11163 ("coordination", "clean"),
11164 ("cache", "review-e2e"),
11165 ("coordination", "no-reviews"),
11166 ("coordination", "handoff"),
11167 ] {
11168 let home = tempfile::tempdir().unwrap();
11169 let mut state = run_state(RunStatus::Blocked);
11170 state.config.graph.review_rounds = 1;
11171 let mut review = review_round(1);
11172 review.clean = true;
11173 state.reviews = vec![review];
11174 if stage == "review-e2e" {
11175 let review = &mut state.reviews[0];
11176 review.clean = false;
11177 review.blocking = 1;
11178 review.e2e = vec![crate::run::CommandOutcome {
11179 command: "verify".into(),
11180 code: None,
11181 output_tail: "resource wait expired".into(),
11182 duration_ms: 0,
11183 resource_blocked: true,
11184 }];
11185 } else if stage == "no-reviews" {
11186 state.config.graph.review_rounds = 0;
11187 state.reviews.clear();
11188 } else if stage == "handoff" {
11189 let review = &mut state.reviews[0];
11192 review.clean = false;
11193 review.blocking = 1;
11194 review.e2e = vec![crate::run::CommandOutcome {
11195 command: "verify".into(),
11196 code: Some(0),
11197 output_tail: String::new(),
11198 duration_ms: 0,
11199 resource_blocked: false,
11200 }];
11201 }
11202 state.gate_ran = false;
11203 state.resource_block = Some(crate::run::ResourceBlock {
11204 node: if stage == "review-e2e" {
11205 "review"
11206 } else {
11207 "gate"
11208 }
11209 .into(),
11210 kind: kind.into(),
11211 detail: "resource wait expired".into(),
11212 at: jiff::Timestamp::now(),
11213 });
11214 state.save_under(home.path()).unwrap();
11215 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11216 assert!(!exhausted_review_budget(&state));
11217 let mut task = held_task_with(&state.id);
11218 task.release();
11219 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11220 assert_eq!(
11221 choose_starter(None, false, unfinished.as_deref()),
11222 Starter::Resume(state.id.clone())
11223 );
11224 let question = action_question(&state.id, resume_action(&state.id));
11225 assert_eq!(
11226 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11227 ActionDecision::Resume(state.id.clone())
11228 );
11229 assert_eq!(
11230 crate::resume::validate(home.path(), &state.id).unwrap().id,
11231 state.id
11232 );
11233 let queue = Queue::at(home.path().join("queue"));
11234 let mut controlled = Task::new(
11235 "resume verification".into(),
11236 state.instruction.clone(),
11237 state.repo.clone(),
11238 Source::Agent {
11239 run: "talk".into(),
11240 node: crate::queue::CHAT_NODE.into(),
11241 },
11242 );
11243 controlled.runs.push(state.id.clone());
11244 controlled.hold_machine(Some("verification resource unavailable".into()));
11245 queue.put(&mut controlled).unwrap();
11246 let resumed =
11247 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11248 assert_eq!(resumed.status, TaskStatus::Queued);
11249 assert_eq!(
11250 resumed.resume_override.unwrap().pinned_run,
11251 Some(state.id.clone())
11252 );
11253 assert_eq!(resumed.attempts, 0);
11254
11255 state.config.graph.review_rounds = 1;
11257 let mut unresolved = review_round(1);
11258 unresolved.blocking = 1;
11259 state.reviews = vec![unresolved];
11260 state.resource_block = None;
11261 state.save_under(home.path()).unwrap();
11262 assert!(exhausted_review_budget(&state));
11263 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11264 }
11265 }
11266
11267 #[tokio::test]
11268 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11269 let home = tempfile::tempdir().unwrap();
11270 let mut owner = stopped_public_run(RunStatus::Blocked);
11271 owner.save_under(home.path()).unwrap();
11272 let history = vec![owner.id.clone()];
11273 let repo = owner.repo.clone();
11274 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11275 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11276 let initial =
11277 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11278 assert!(
11279 initial.is_pending(),
11280 "a busy record must not hold or resume the task"
11281 );
11282 owner.retired_to = Some("20260101-000001-bbbb".into());
11283 owner.save_under(home.path()).unwrap();
11284 drop(lock);
11285 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11286 .await
11287 .expect("ownership lock released")
11288 .expect("read settled owner");
11289 assert!(
11290 result.is_none(),
11291 "the retired owner must not be resumed from a stale read"
11292 );
11293 }
11294
11295 #[tokio::test]
11296 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11297 let home = tempfile::tempdir().unwrap();
11298 let mut owner = stopped_public_run(RunStatus::Blocked);
11299 let repo = tempfile::tempdir().unwrap();
11300 crate::git::git(repo.path(), &["init"]).await.unwrap();
11301 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11302 owner.id = "20260101-000000-aaaa".into();
11303 owner.save_under(home.path()).unwrap();
11304 let mut successor = run_state(RunStatus::Blocked);
11305 successor.id = "20260101-000001-bbbb".into();
11306 successor.save_under(home.path()).unwrap();
11307 let runs = vec![owner.id.clone(), successor.id.clone()];
11308 assert_eq!(
11309 public_resume_owner(home.path(), &runs, &owner.repo)
11310 .await
11311 .unwrap()
11312 .unwrap()
11313 .id,
11314 owner.id
11315 );
11316 let subdir = owner.repo.join("nested");
11317 std::fs::create_dir(&subdir).unwrap();
11318 assert_eq!(
11319 public_resume_owner(home.path(), &runs, &subdir)
11320 .await
11321 .unwrap()
11322 .unwrap()
11323 .id,
11324 owner.id
11325 );
11326 #[cfg(unix)]
11327 {
11328 let alias = home.path().join("repo-alias");
11329 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
11330 assert_eq!(
11331 public_resume_owner(home.path(), &runs, &alias.join("nested"))
11332 .await
11333 .unwrap()
11334 .unwrap()
11335 .id,
11336 owner.id
11337 );
11338 }
11339 let different = tempfile::tempdir().unwrap();
11340 crate::git::git(different.path(), &["init"]).await.unwrap();
11341 assert!(
11342 public_resume_owner(home.path(), &runs, different.path())
11343 .await
11344 .is_err()
11345 );
11346 std::fs::remove_file(
11347 home.path()
11348 .join("runs")
11349 .join(&successor.id)
11350 .join("run.json"),
11351 )
11352 .unwrap();
11353 assert!(
11354 public_resume_owner(home.path(), &runs, &owner.repo)
11355 .await
11356 .is_err()
11357 );
11358 }
11359
11360 #[test]
11361 fn an_ordinary_release_keeps_a_resumable_run_available() {
11362 let mut released = task();
11363 released.start("stalled-run".to_owned());
11364 released.release();
11365 let unfinished = (!released.fresh_start)
11366 .then(|| Some("stalled-run".to_owned()))
11367 .flatten();
11368 assert_eq!(
11369 choose_starter(None, false, unfinished.as_deref()),
11370 Starter::Resume("stalled-run".to_owned()),
11371 "manual release must preserve the normal resume path"
11372 );
11373 }
11374
11375 #[test]
11376 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
11377 let mut state = run_state(RunStatus::Blocked);
11378 state.config.graph.review_rounds = 3;
11379 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
11380 assert!(exhausted_review_budget(&state));
11381
11382 state.reviews.pop();
11384 assert!(!exhausted_review_budget(&state));
11385
11386 let mut stalled = run_state(RunStatus::Stalled);
11389 stalled.config.graph.review_rounds = 1;
11390 stalled.reviews = vec![review_round(1)];
11391 assert!(!exhausted_review_budget(&stalled));
11392 }
11393
11394 fn review_round(round: usize) -> crate::run::ReviewRound {
11395 crate::run::ReviewRound {
11396 round,
11397 head: "deadbeef".to_owned(),
11398 verified_head: None,
11399 verified_at: None,
11400 reviews: Vec::new(),
11401 e2e: Vec::new(),
11402 verify_retried: false,
11403 e2e_deferred: false,
11404 e2e_defer_reason: None,
11405 fix: None,
11406 blocking: 0,
11407 answered: 1,
11408 expected: 1,
11409 clean: false,
11410 progressed: true,
11411 vote_split: false,
11412 reconsideration: Vec::new(),
11413 verdict: None,
11414 }
11415 }
11416
11417 #[test]
11418 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
11419 let mut run = RunState::new(
11420 PathBuf::from("/repo"),
11421 "main".to_owned(),
11422 "abc1234def".to_owned(),
11423 "add retries".to_owned(),
11424 Config::default(),
11425 );
11426 run.status = RunStatus::Judging;
11427 run.parked = true;
11428 let mut task = Task::new(
11429 "add retries".to_owned(),
11430 "add retries".to_owned(),
11431 PathBuf::from("/repo"),
11432 crate::queue::Source::Human,
11433 );
11434 task.status = TaskStatus::Failed;
11435 task.runs = vec![run.id.clone()];
11436 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
11437 assert!(with(&task, &run), "parked after judging is the case");
11438 let mut parked_task = task.clone();
11439 parked_task.status = TaskStatus::Parked;
11440 assert!(with(&parked_task, &run), "the Parked status is the case");
11441
11442 let mut not_parked = run.clone();
11443 not_parked.parked = false;
11444 not_parked.status = RunStatus::Stalled;
11445 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
11446
11447 let mut fresh = task.clone();
11448 fresh.fresh_start = true;
11449 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
11450
11451 let mut review = task.clone();
11452 review.review_branch = Some("magi/x/A".to_owned());
11453 assert!(!with(&review, &run), "review is ranked before resume");
11454
11455 let mut held = task.clone();
11456 held.status = TaskStatus::Held;
11457 assert!(!with(&held, &run), "a hold stays visible to the conductor");
11458
11459 let mut released = run.clone();
11460 released.released_to = Some("20260901-000000-new1".to_owned());
11461 assert!(!with(&task, &released), "nothing left to resume into");
11462
11463 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
11464 "unreadable"
11465 )));
11466 }
11467
11468 #[test]
11469 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
11470 let mut released = RunState::new(
11471 PathBuf::from("/repo"),
11472 "main".to_owned(),
11473 "abc1234def".to_owned(),
11474 "add retries".to_owned(),
11475 Config::default(),
11476 );
11477 released.status = RunStatus::Blocked;
11478 assert_eq!(
11479 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11480 Some(released.id.clone())
11481 );
11482 released.released_to = Some("20260901-000000-new1".to_owned());
11483 assert_eq!(
11484 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11485 None,
11486 "there is nothing left to resume it into"
11487 );
11488 }
11489
11490 #[test]
11491 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
11492 let mut exhausted = RunState::new(
11501 PathBuf::from("/repo"),
11502 "main".to_owned(),
11503 "abc1234def".to_owned(),
11504 "add retries".to_owned(),
11505 Config::default(),
11506 );
11507 exhausted.status = RunStatus::Blocked;
11508 exhausted.config.graph.review_rounds = 1;
11509 exhausted.reviews = vec![review_round(1)];
11510
11511 assert_eq!(
11512 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
11513 None,
11514 "an exhausted `Blocked` run must not be offered as resumable"
11515 );
11516
11517 let mut has_budget_left = RunState::new(
11520 PathBuf::from("/repo"),
11521 "main".to_owned(),
11522 "abc1234def".to_owned(),
11523 "add retries".to_owned(),
11524 Config::default(),
11525 );
11526 has_budget_left.status = RunStatus::Blocked;
11527 has_budget_left.config.graph.review_rounds = 3;
11528 has_budget_left.reviews = vec![review_round(1)];
11529
11530 assert_eq!(
11531 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
11532 Ok(has_budget_left.clone())
11533 }),
11534 Some(has_budget_left.id.clone())
11535 );
11536 }
11537
11538 #[test]
11539 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
11540 let mut run = RunState::new(
11541 PathBuf::from("/repo"),
11542 "main".to_owned(),
11543 "abc1234def".to_owned(),
11544 "add retries".to_owned(),
11545 Config::default(),
11546 );
11547 run.status = RunStatus::Blocked;
11548 run.config.graph.review_rounds = 1;
11549 run.reviews = vec![review_round(1)];
11550 run.pr_publication_pending = true;
11551 assert_eq!(
11552 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11553 Some(run.id.clone()),
11554 "a spent review budget must not turn a failed publication into a re-competition"
11555 );
11556 }
11557
11558 #[test]
11559 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11560 let mut task = Task::new(
11561 "recover work".into(),
11562 "recover work".into(),
11563 PathBuf::from("/repo"),
11564 crate::queue::Source::Human,
11565 );
11566 let mut history = std::collections::BTreeMap::new();
11567 for index in 0..8 {
11568 let mut run = run_state(RunStatus::Blocked);
11569 run.id = format!("20260101-000000-{index:04}");
11570 run.base_commit = format!("upstream-{index}");
11571 if index > 0 {
11573 run.candidates.clear();
11574 }
11575 task.runs.extend([run.id.clone(), run.id.clone()]);
11576 history.insert(run.id.clone(), run);
11577 task.requeue();
11578 }
11579 let check = |task: &Task, starter: &Starter, limit| {
11580 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11581 };
11582 assert!(
11583 check(&task, &Starter::Start, 8)
11584 .unwrap()
11585 .contains("8 unretired")
11586 );
11587 assert!(check(&task, &Starter::Start, 9).is_none());
11588 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11589 task.release_operator("owner", None);
11590 assert_eq!(task.attempts, 0);
11591 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11592 assert!(check(&restored, &Starter::Start, 8).is_some());
11593 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11594 assert!(check(&restored, &Starter::Start, 0).is_none());
11595 }
11596
11597 #[test]
11598 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11599 let mut task = Task::new(
11600 "recover work".into(),
11601 "recover work".into(),
11602 PathBuf::from("/repo"),
11603 crate::queue::Source::Human,
11604 );
11605 let mut run = run_state(RunStatus::Superseded);
11606 run.candidates.clear();
11607 task.runs.push(run.id.clone());
11608 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11609 run.retired_to = Some("authority".into());
11610 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11611 run.retired_to = None;
11612 run.released_to = Some("authority".into());
11613 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11614 assert!(
11615 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11616 .unwrap()
11617 .contains("Cannot verify")
11618 );
11619 assert!(
11621 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11622 panic!("resume must not load historical attempts")
11623 })
11624 .is_none()
11625 );
11626 }
11627
11628 #[test]
11629 fn retired_resume_requires_a_valid_same_repository_authority() {
11630 let mut source = run_state(RunStatus::Blocked);
11631 let mut empty = run_state(RunStatus::Blocked);
11632 empty.retired_to = Some(source.id.clone());
11633 let runs = vec![source.id.clone(), empty.id.clone()];
11634 let resolve = |source: &RunState, empty: &RunState| {
11635 unfinished_run_with(&runs, "fixture", |id| {
11636 if id == source.id {
11637 Ok(source.clone())
11638 } else if id == empty.id {
11639 Ok(empty.clone())
11640 } else {
11641 bail!("unreadable authority")
11642 }
11643 })
11644 };
11645 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11646 assert_eq!(
11647 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11648 Starter::Resume(source.id.clone())
11649 );
11650 source.retired_to = Some(empty.id.clone());
11651 assert!(
11652 resolve(&source, &empty).is_none(),
11653 "cycles do not revive obsolete runs"
11654 );
11655 source.retired_to = None;
11656 source.repo = PathBuf::from("/another-repository");
11657 assert!(resolve(&source, &empty).is_none());
11658 source.repo = empty.repo.clone();
11659 source.released_to = Some("later".into());
11660 assert!(resolve(&source, &empty).is_none());
11661 empty.retired_to = Some("unlisted-run".into());
11662 assert!(resolve(&source, &empty).is_none());
11663 }
11664
11665 #[test]
11666 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11667 let mut older_stalled = RunState::new(
11675 PathBuf::from("/repo"),
11676 "main".to_owned(),
11677 "abc1234def".to_owned(),
11678 "add retries".to_owned(),
11679 Config::default(),
11680 );
11681 older_stalled.status = RunStatus::Stalled;
11682
11683 let mut newest_exhausted = RunState::new(
11684 PathBuf::from("/repo"),
11685 "main".to_owned(),
11686 "abc1234def".to_owned(),
11687 "add retries".to_owned(),
11688 Config::default(),
11689 );
11690 newest_exhausted.status = RunStatus::Blocked;
11691 newest_exhausted.config.graph.review_rounds = 1;
11692 newest_exhausted.reviews = vec![review_round(1)];
11693
11694 assert_eq!(
11695 unfinished_run_with(
11696 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11697 "t",
11698 |_| Ok(newest_exhausted.clone())
11699 ),
11700 None,
11701 "the newest run is exhausted, so nothing here is worth resuming - \
11702 least of all the older, already-superseded run"
11703 );
11704 }
11705
11706 #[test]
11707 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11708 assert_eq!(
11709 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11710 Err(anyhow::anyhow!("fixture is absent"))
11711 }),
11712 None
11713 );
11714 }
11715
11716 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11717 let mut q = ask::Question::new(
11718 run.to_owned(),
11719 "implement".to_owned(),
11720 "impl-A".to_owned(),
11721 "continue?".to_owned(),
11722 String::new(),
11723 vec!["resume で続行する".to_owned(), "other".to_owned()],
11724 );
11725 q.actions.insert("resume で続行する".to_owned(), action);
11726 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11727 .unwrap();
11728 q
11729 }
11730
11731 fn held_task_with(run: &str) -> Task {
11732 let mut t = task();
11733 t.runs = vec![run.to_owned()];
11734 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11735 t
11736 }
11737
11738 fn resume_action(run: &str) -> ask::ChoiceAction {
11739 ask::ChoiceAction::Resume { run: run.into() }
11740 }
11741
11742 #[test]
11743 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11744 for parked in [false, true] {
11745 let mut t = task();
11746 t.start("A".into());
11747 t.start("B".into());
11748 assert_eq!(t.runs, vec!["A", "B"]);
11749 t.start("A".into());
11750 let mut resumed = run_state(RunStatus::Blocked);
11751 resumed.id = "A".into();
11752 resumed.parked = parked;
11753 resumed.driver_exited = true;
11754 t.hold_machine(Some("resumed run stopped".into()));
11755 let q = action_question("A", resume_action("A"));
11756 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11757 assert_eq!(
11758 decide_action(&t, &q, &PHRASES_EN, |id| {
11759 assert_eq!(id, "A");
11760 Ok(resumed.clone())
11761 }),
11762 ActionDecision::Resume("A".into())
11763 );
11764 assert_eq!(
11765 action_standing(&t, &action_question("B", resume_action("B"))),
11766 ActionStanding::Stale
11767 );
11768 t.release_operator("cli", None);
11769 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11770 assert_eq!(id, "A");
11771 Ok(resumed.clone())
11772 });
11773 assert_eq!(
11774 choose_starter(None, false, unfinished.as_deref()),
11775 Starter::Resume("A".into())
11776 );
11777 assert_eq!(t.runs, vec!["B", "A"]);
11778 }
11779 }
11780
11781 #[test]
11782 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
11783 let mut state = crate::land::tests::repair_state(true, true);
11784 state.status = RunStatus::Landing;
11785 state.parked = true;
11786 state.driver_exited = true;
11787 let mut q = ask::Question::new(
11788 state.id.clone(),
11789 land::APPROVAL_NODE.into(),
11790 "land".into(),
11791 "Merge?".into(),
11792 String::new(),
11793 vec![land::APPROVE.into(), land::HOLD.into()],
11794 );
11795 q.id = state.id.clone();
11796 state.land_approval = Some(crate::run::LandApproval {
11797 question: q.id.clone(),
11798 head: "current-head".into(),
11799 });
11800 assert!(land::upgrade_repair_choice(&mut q, &state));
11801 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11802 let mut task = held_task_with(&state.id);
11803 task.status = TaskStatus::Parked;
11804 assert_eq!(
11805 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11806 ActionDecision::Resume(state.id.clone())
11807 );
11808 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11809 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
11810 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
11811 assert_eq!(
11812 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11813 ActionDecision::Resume(state.id.clone()),
11814 "a repair answer newer than the manual hold resumes the existing owner"
11815 );
11816 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11817 for answered_at in [q.answered_at, None] {
11818 let mut older = q.clone();
11819 older.answered_at = answered_at;
11820 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
11821 panic!("a later manual hold must prevent dispatch")
11822 });
11823 let ActionDecision::Refuse(reason) = refused else {
11824 panic!("expected an actionable refusal, got {refused:?}");
11825 };
11826 assert!(reason.contains("later operator decision"));
11827 assert!(reason.contains(&format!("magi task release {}", task.id)));
11828 }
11829 task.operator_decided_at = None;
11830 let mut superseded = state.clone();
11831 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
11832 assert_eq!(
11833 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
11834 ActionDecision::Stale
11835 );
11836 for case in 1..4 {
11837 let mut changed = state.clone();
11838 match case {
11839 1 => changed.status = RunStatus::Blocked,
11840 2 => changed.released_to = Some("new-owner".into()),
11841 _ => changed.land_approval.as_mut().unwrap().head.clear(),
11842 }
11843 assert!(matches!(
11844 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
11845 ActionDecision::Refuse(_)
11846 ));
11847 }
11848 assert!(matches!(
11849 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
11850 ActionDecision::Refuse(_)
11851 ));
11852 task.runs.push("replacement".into());
11853 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
11854 assert_eq!(
11855 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
11856 "old owner must not load"
11857 )),
11858 ActionDecision::Stale
11859 );
11860 }
11861
11862 #[test]
11863 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
11864 let mut state = crate::land::tests::repair_state(true, true);
11865 state.status = RunStatus::Landing;
11866 state.parked = true;
11867 state.driver_exited = true;
11868 let mut q = ask::Question::new(
11869 state.id.clone(),
11870 land::APPROVAL_NODE.into(),
11871 "land".into(),
11872 "Merge?".into(),
11873 String::new(),
11874 vec![land::APPROVE.into(), land::HOLD.into()],
11875 );
11876 q.id = state.id.clone();
11877 state.land_approval = Some(crate::run::LandApproval {
11878 question: q.id.clone(),
11879 head: "current-head".into(),
11880 });
11881 state.save().unwrap();
11882 land::upgrade_repair_choice(&mut q, &state);
11883 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11884 Questions::open().put(&mut q).unwrap();
11885 let tmp = tempfile::tempdir().unwrap();
11886 let queue = Queue::at(tmp.path().join("queue"));
11887 let mut task = held_task_with(&state.id);
11888 task.status = TaskStatus::Parked;
11889 queue.put(&mut task).unwrap();
11890 let empty = Arc::new(tokio::sync::Semaphore::new(0));
11891 let free = Arc::new(tokio::sync::Semaphore::new(1));
11892 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
11893 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
11894 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
11895 assert!(claim.permit.is_some());
11896 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
11897 drop(claim);
11898 Questions::open()
11899 .update(&q.id, |q| {
11900 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
11901 Ok(())
11902 })
11903 .unwrap();
11904 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
11905 assert!(claim.permit.is_none());
11906 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
11907 }
11908
11909 #[test]
11910 fn decide_action_resumes_only_the_latest_resumable_run() {
11911 let t = held_task_with("r1");
11912 let q = action_question("r1", resume_action("r1"));
11913 let load = |s: RunState| move |_: &str| Ok(s);
11914 assert_eq!(
11915 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
11916 ActionDecision::Resume("r1".into())
11917 );
11918 let q_other = action_question("r1", resume_action("r0"));
11920 assert!(matches!(
11921 decide_action(
11922 &t,
11923 &q_other,
11924 &PHRASES_EN,
11925 load(run_state(RunStatus::Blocked))
11926 ),
11927 ActionDecision::Refuse(_)
11928 ));
11929 assert!(matches!(
11931 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
11932 ActionDecision::Refuse(_)
11933 ));
11934 let mut released = run_state(RunStatus::Blocked);
11936 released.released_to = Some("elsewhere".into());
11937 assert!(matches!(
11938 decide_action(&t, &q, &PHRASES_EN, load(released)),
11939 ActionDecision::Refuse(_)
11940 ));
11941 assert!(matches!(
11943 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
11944 ActionDecision::Refuse(_)
11945 ));
11946 }
11947
11948 #[test]
11949 fn decide_action_ignores_a_question_about_an_earlier_run() {
11950 let mut t = held_task_with("r1");
11951 t.runs.push("r2".to_owned());
11952 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11953 assert_eq!(
11954 decide_action(
11955 &t,
11956 &action_question("r1", ask::ChoiceAction::Done),
11957 &PHRASES_EN,
11958 never
11959 ),
11960 ActionDecision::Stale
11961 );
11962 }
11963
11964 #[test]
11965 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
11966 let mut t = held_task_with("r1");
11967 let never = |_: &str| -> Result<RunState> { bail!("not read") };
11968 assert_eq!(
11969 decide_action(
11970 &t,
11971 &action_question("r1", ask::ChoiceAction::Requeue),
11972 &PHRASES_EN,
11973 never
11974 ),
11975 ActionDecision::Requeue
11976 );
11977 let done_q = action_question("r1", ask::ChoiceAction::Done);
11978 assert_eq!(
11979 decide_action(&t, &done_q, &PHRASES_EN, never),
11980 ActionDecision::Done
11981 );
11982 t.mark_action_applied(&done_q.id);
11983 assert_eq!(
11984 decide_action(&t, &done_q, &PHRASES_EN, never),
11985 ActionDecision::Skip
11986 );
11987
11988 let mut plain = action_question("r1", ask::ChoiceAction::Done);
11990 plain.actions.clear();
11991 assert_eq!(
11992 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
11993 ActionDecision::Skip
11994 );
11995 let mut running = held_task_with("r1");
11997 running.status = TaskStatus::Running;
11998 assert_eq!(
11999 decide_action(
12000 &running,
12001 &action_question("r1", ask::ChoiceAction::Done),
12002 &PHRASES_EN,
12003 never
12004 ),
12005 ActionDecision::Skip
12006 );
12007 }
12008
12009 #[test]
12010 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12011 crate::run::pin_test_home();
12012 for legacy in [false, true] {
12013 let dir = tempfile::tempdir().unwrap();
12014 let queue = Queue::at(dir.path().join("queue"));
12015 let questions = Questions::at(dir.path().join("questions"));
12016 let run = "20260101-000000-hold";
12017 let mut task = held_task_with(run);
12018 task.status = TaskStatus::Parked;
12019 let mut q = ask::Question::new(
12020 run.into(),
12021 land::APPROVAL_NODE.into(),
12022 "land".into(),
12023 "Repair?".into(),
12024 String::new(),
12025 vec![land::REPAIR.into()],
12026 );
12027 q.actions.insert(land::REPAIR.into(), resume_action(run));
12028 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12029 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12031 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12032 questions.put(&mut q).unwrap();
12033 task.hold_manual(Some("Keep this PR paused".into()));
12034 if legacy {
12035 task.hold_source = None;
12036 }
12037 queue.put(&mut task).unwrap();
12038
12039 apply_task_choice_actions(&queue, &questions, dir.path());
12040 let after = queue.get(&task.id).unwrap();
12041 assert!(after.operator_held());
12042 assert_eq!(after.hold_source, task.hold_source);
12043 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12044 assert_eq!(after.runs, task.runs);
12045 assert_eq!(after.instruction, task.instruction);
12046 assert!(after.resume_override.is_none());
12047 assert!(after.action_applied(&q.id));
12048 let reason = after.hold_reason.as_deref().unwrap();
12049 assert!(reason.starts_with("Keep this PR paused"));
12050 assert!(reason.contains("later operator decision"));
12051 assert!(reason.contains(&format!("magi task release {}", task.id)));
12052
12053 for recovery in [
12054 crate::conduct::Recovery::Review,
12055 crate::conduct::Recovery::Requeue,
12056 ] {
12057 crate::conduct::apply(
12058 &queue,
12059 &questions,
12060 &crate::conduct::Verdict {
12061 decisions: vec![crate::conduct::Decision {
12062 id: task.id.clone(),
12063 recovery: Some(recovery),
12064 ..Default::default()
12065 }],
12066 },
12067 )
12068 .unwrap();
12069 }
12070 apply_task_choice_actions(&queue, &questions, dir.path());
12073 let still_held = queue.get(&task.id).unwrap();
12074 assert!(still_held.operator_held());
12075 assert_eq!(still_held.hold_reason, after.hold_reason);
12076 assert_eq!(still_held.runs, task.runs);
12077 assert!(still_held.review_branch.is_none());
12078 assert!(!still_held.fresh_start);
12079 assert!(queue.next_runnable().is_none());
12080 }
12081 }
12082
12083 #[test]
12084 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
12085 for held in [false, true] {
12086 for retry in [false, true] {
12087 let dir = tempfile::tempdir().unwrap();
12088 let queue = Queue::at(dir.path().join("queue"));
12089 let questions = Questions::at(dir.path().join("questions"));
12090 let home = dir.path().join("home");
12091 let mut t = task();
12092 if held {
12093 t.hold_machine(Some("driver state unknown".into()));
12094 }
12095 queue.put(&mut t).unwrap();
12096 let verdict = serde_json::from_value(serde_json::json!({
12097 "decisions": [{
12098 "id": t.id,
12099 "question": "Driver state unknown; retry the checks or hold?",
12100 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
12101 "actions": {
12102 "daemon: recheck and retry": {"do": "requeue"},
12103 "daemon: hold the task": {"do": "hold"}
12104 }
12105 }]
12106 }))
12107 .unwrap();
12108 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
12109 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12110 let mut q = questions.list().pop().unwrap();
12111 let choice = if retry {
12112 "daemon: recheck and retry"
12113 } else {
12114 "daemon: hold the task"
12115 };
12116 assert_eq!(q.actions.len(), 2);
12117 q.answer(ask::Answer::Choice(choice.into())).unwrap();
12118 questions.put(&mut q).unwrap();
12119
12120 questions.beat(&q.id, ask::WaiterKind::Deputy);
12122 for _ in 0..2 {
12123 resolve_blockers(&queue, &questions);
12124 apply_task_choice_actions(&queue, &questions, &home);
12125 let waiting = queue.get(&t.id).unwrap();
12126 assert_eq!(waiting.status, TaskStatus::Blocked);
12127 assert_eq!(waiting.blocked_by, [q.id.clone()]);
12128 assert!(waiting.answers.is_empty());
12129 assert!(!waiting.action_applied(&q.id));
12130 assert!(runnable(&queue).is_empty());
12131 let free = Arc::new(tokio::sync::Semaphore::new(1));
12132 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
12133 }
12134 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12135
12136 resolve_blockers(&queue, &questions);
12138 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12139 apply_task_choice_actions(&queue, &questions, &home);
12140 let after = queue.get(&t.id).unwrap();
12141 assert_eq!(
12142 after.status,
12143 if retry {
12144 TaskStatus::Queued
12145 } else {
12146 TaskStatus::Held
12147 }
12148 );
12149 assert!(after.action_applied(&q.id));
12150 assert_eq!(after.operator_held(), !retry);
12151 assert_eq!(after.answers.len(), 1);
12152
12153 let mut changed = after;
12155 if retry {
12156 changed.hold_manual(Some("later hold".into()));
12157 } else {
12158 changed.release_operator("test", None);
12159 }
12160 let expected = changed.status;
12161 queue.put(&mut changed).unwrap();
12162 apply_task_choice_actions(&queue, &questions, &home);
12163 assert_eq!(queue.get(&t.id).unwrap().status, expected);
12164 }
12165 }
12166 }
12167
12168 #[test]
12169 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
12170 let dir = tempfile::tempdir().unwrap();
12171 let queue = Queue::at(dir.path().join("queue"));
12172 let questions = Questions::at(dir.path().join("questions"));
12173 let home = dir.path().join("home");
12174 let mut state = run_state(RunStatus::Blocked);
12175 state.id = "20260101-000000-act1".to_owned();
12176 state.save_under(&home).unwrap();
12177
12178 let mut t = held_task_with(&state.id);
12179 queue.put(&mut t).unwrap();
12180 let mut q = action_question(&state.id, resume_action(&state.id));
12181 questions.put(&mut q).unwrap();
12182
12183 apply_task_choice_actions(&queue, &questions, &home);
12184 let after = queue.get(&t.id).unwrap();
12185 assert_eq!(after.status, TaskStatus::Queued);
12186 assert!(!after.fresh_start);
12187 assert!(after.action_applied(&q.id));
12188 let pin = after.resume_override.clone().unwrap();
12189 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
12190 assert!(pin.forced);
12191
12192 let mut again = queue.get(&t.id).unwrap();
12194 again.hold_machine(Some("later".into()));
12195 queue.put(&mut again).unwrap();
12196 apply_task_choice_actions(&queue, &questions, &home);
12197 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12198 }
12199
12200 #[test]
12201 fn a_delivered_answer_is_still_acted_on_exactly_once() {
12202 let dir = tempfile::tempdir().unwrap();
12203 let queue = Queue::at(dir.path().join("queue"));
12204 let questions = Questions::at(dir.path().join("questions"));
12205 let home = dir.path().join("home");
12206 let mut state = run_state(RunStatus::Blocked);
12207 state.id = "20260101-000000-act2".to_owned();
12208 state.save_under(&home).unwrap();
12209
12210 let mut t = held_task_with(&state.id);
12211 queue.put(&mut t).unwrap();
12212 let mut q = action_question(&state.id, resume_action(&state.id));
12213 q.answer_delivered = true;
12214 questions.put(&mut q).unwrap();
12215
12216 apply_task_choice_actions(&queue, &questions, &home);
12219 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12220 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
12221
12222 let mut again = queue.get(&t.id).unwrap();
12224 again.hold_machine(Some("later".into()));
12225 queue.put(&mut again).unwrap();
12226 apply_task_choice_actions(&queue, &questions, &home);
12227 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12228 }
12229
12230 #[test]
12231 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
12232 let dir = tempfile::tempdir().unwrap();
12233 let queue = Queue::at(dir.path().join("queue"));
12234 let questions = Questions::at(dir.path().join("questions"));
12235 let home = dir.path().join("home");
12236 let mut state = run_state(RunStatus::Blocked);
12237 state.id = "20260101-000000-act3".to_owned();
12238 state.save_under(&home).unwrap();
12239 let mut t = held_task_with(&state.id);
12240 queue.put(&mut t).unwrap();
12241 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
12242 questions.put(&mut q).unwrap();
12243
12244 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
12245 apply_task_choice_actions(&queue, &questions, &home);
12246 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12247
12248 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12249 apply_task_choice_actions(&queue, &questions, &home);
12250 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12251 }
12252
12253 #[test]
12254 fn action_standing_tells_the_reasons_apart() {
12255 let mut t = held_task_with("r1");
12256 let q = action_question("r1", ask::ChoiceAction::Requeue);
12257 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12258 let mut plain = q.clone();
12259 plain.actions.clear();
12260 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
12261
12262 t.mark_action_applied(&q.id);
12264 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
12265
12266 let mut t = held_task_with("r1");
12268 t.start("r2".to_owned());
12269 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
12270 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
12272 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12273 t.status = TaskStatus::Blocked;
12274 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12275
12276 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
12278 c.node = crate::conduct::NODE.to_owned();
12279 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
12280 assert!(!ActionStanding::Busy.daemon_owns());
12281 }
12282
12283 #[test]
12284 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
12285 let dir = tempfile::tempdir().unwrap();
12286 let queue = Queue::at(dir.path().join("queue"));
12287 let questions = Questions::at(dir.path().join("questions"));
12288 let home = dir.path().join("home");
12289 let mut state = run_state(RunStatus::Blocked);
12290 state.id = "20260101-000000-act4".to_owned();
12291 state.save_under(&home).unwrap();
12292 let mut t = held_task_with(&state.id);
12293 t.status = TaskStatus::Running;
12294 queue.put(&mut t).unwrap();
12295 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
12296 q.cwd = Some(dir.path().display().to_string());
12297 questions.put(&mut q).unwrap();
12298
12299 let running = queue.get(&t.id).unwrap();
12302 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
12303 assert!(matches!(
12304 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
12305 crate::waiter::Action::Deliver(_)
12306 ));
12307 apply_task_choice_actions(&queue, &questions, &home);
12309 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
12310 let mut back = queue.get(&t.id).unwrap();
12311 back.hold_machine(Some("later".into()));
12312 queue.put(&mut back).unwrap();
12313 apply_task_choice_actions(&queue, &questions, &home);
12314 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
12315 let mut again = queue.get(&t.id).unwrap();
12316 again.hold_machine(Some("again".into()));
12317 queue.put(&mut again).unwrap();
12318 apply_task_choice_actions(&queue, &questions, &home);
12319 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12320 }
12321}