1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.runs.last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id,
519 run: task.runs.last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if q.chosen_action().is_none() {
887 ActionStanding::NoAction
888 } else if task.action_applied(&q.id) {
889 ActionStanding::Applied
890 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 ActionStanding::Stale
892 } else if matches!(
893 task.status,
894 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
895 ) {
896 ActionStanding::Busy
897 } else {
898 ActionStanding::Pending
899 }
900}
901
902#[derive(Debug, Clone, PartialEq, Eq)]
904enum ActionDecision {
905 Skip,
907 Resume(String),
909 Requeue,
911 Hold,
913 Done,
915 Stale,
918 Refuse(String),
921}
922
923fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
931where
932 F: FnOnce(&str) -> Result<RunState>,
933{
934 let Some(action) = q.chosen_action() else {
935 return ActionDecision::Skip;
936 };
937 if task.action_applied(&q.id)
938 || matches!(
939 task.status,
940 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
941 )
942 {
943 return ActionDecision::Skip;
944 }
945 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
948 return ActionDecision::Stale;
949 }
950 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
951 let stopping = q.node == land::APPROVAL_NODE
952 && q.resolution()
953 .as_deref()
954 .is_some_and(crate::merge_decision::stops);
955 let decision_at = if repair || stopping {
959 q.answered_at.unwrap_or(q.asked_at)
960 } else {
961 q.asked_at
962 };
963 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
964 return if repair || stopping {
965 ActionDecision::Refuse(format!(
966 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
967 if repair { "repair" } else { "stop" },
968 q.id,
969 q.run,
970 task.id
971 ))
972 } else {
973 ActionDecision::Stale
974 };
975 }
976 match action {
977 a if a.maintenance() => ActionDecision::Skip,
978 ask::ChoiceAction::ResumeRun { .. }
979 | ask::ChoiceAction::ReleaseTask { .. }
980 | ask::ChoiceAction::RerunChecks { .. }
981 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
982 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
983 ask::ChoiceAction::Hold => ActionDecision::Hold,
984 ask::ChoiceAction::Done => ActionDecision::Done,
985 ask::ChoiceAction::Resume { run } => {
986 if task.runs.last() != Some(run) {
987 return ActionDecision::Refuse((p.resume_not_latest)(
988 q.short(),
989 ask::short_id(run),
990 ));
991 }
992 match load(run) {
993 Ok(s) if stopping => {
994 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
995 || !s
996 .land_approval
997 .as_ref()
998 .is_some_and(|binding| binding.question == q.id)
999 {
1000 ActionDecision::Stale
1001 } else if s.liveness(false) == Liveness::Live {
1002 ActionDecision::Skip
1003 } else if s.status == RunStatus::Landing
1004 && !s.released()
1005 && s.retired_to.is_none()
1006 {
1007 ActionDecision::Resume(run.clone())
1008 } else {
1009 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1010 }
1011 }
1012 Ok(s)
1013 if q.node == land::APPROVAL_NODE
1014 && q.resolution().as_deref() == Some(land::REPAIR) =>
1015 {
1016 if s.merge_repairs.iter().any(|r| r.question == q.id)
1017 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1018 {
1019 ActionDecision::Stale
1022 } else if s.liveness(false) == Liveness::Live {
1023 ActionDecision::Skip
1024 } else if s.status != RunStatus::Landing
1025 || s.released()
1026 || s.reviews.is_empty()
1027 || !implementers_match(task, &s)
1028 || !s
1029 .land_approval
1030 .as_ref()
1031 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1032 {
1033 ActionDecision::Refuse(format!(
1034 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1035 q.id, run, run
1036 ))
1037 } else {
1038 ActionDecision::Resume(run.clone())
1039 }
1040 }
1041 Ok(s) if s.liveness(false) == Liveness::Live => {
1042 ActionDecision::Refuse("run already has a live driver".into())
1043 }
1044 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
1045 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
1046 q.short(),
1047 ask::short_id(run),
1048 )),
1049 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1050 q.short(),
1051 ask::short_id(run),
1052 &format!("{e:#}"),
1053 )),
1054 }
1055 }
1056 }
1057}
1058
1059struct Phrases {
1073 graph_stopped: fn(&str, &str) -> String,
1075 quorum_lost: &'static str,
1076 quota_took_out: &'static str,
1078 run_ended: &'static str,
1080 waiting_for_answer: &'static str,
1082 recovered_running: &'static str,
1083 no_run_to_recover: &'static str,
1084 could_not_start: &'static str,
1085 resume_not_latest: fn(&str, &str) -> String,
1086 resume_cannot_progress: fn(&str, &str) -> String,
1087 resume_unreadable: fn(&str, &str, &str) -> String,
1088 handover_refused: fn(&crate::handover::Refused) -> String,
1091 handover_hint: &'static str,
1093 reviewers_never_answered: fn(usize, usize) -> String,
1096}
1097
1098const PHRASES_EN: Phrases = Phrases {
1099 graph_stopped: |status, detail| {
1100 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1101 },
1102 quorum_lost: "the judging panel lost its quorum",
1103 quota_took_out: "; quota took out ",
1104 run_ended: "run ended ",
1105 waiting_for_answer: " - waiting for operator answer to question ",
1106 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1107 no_run_to_recover: "task was `running` with no live daemon and no readable \
1108 run to recover; held for a human to check what happened",
1109 could_not_start: "could not start the run: ",
1110 resume_not_latest: |q, run| {
1111 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1112 },
1113 resume_cannot_progress: |q, run| {
1114 format!("question {q} asked to resume run {run}, which cannot make progress")
1115 },
1116 resume_unreadable: |q, run, e| {
1117 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1118 },
1119 handover_refused: |r| r.to_string(),
1120 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1121 reviewers_never_answered: |missing, rounds| {
1122 format!(
1123 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1124 refusing to call it clean"
1125 )
1126 },
1127};
1128
1129const PHRASES_JA: Phrases = Phrases {
1130 graph_stopped: |status, detail| {
1131 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1132 },
1133 quorum_lost: "審査パネルが定足数を失いました",
1134 quota_took_out: "。クォータで脱落: ",
1135 run_ended: "run 終了: ",
1136 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1137 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1138 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1139 何が起きたか人が確認するため保留にしました",
1140 could_not_start: "run を開始できませんでした: ",
1141 resume_not_latest: |q, run| {
1142 format!(
1143 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1144 )
1145 },
1146 resume_cannot_progress: |q, run| {
1147 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1148 },
1149 resume_unreadable: |q, run, e| {
1150 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1151 },
1152 handover_refused: |r| {
1153 use crate::handover::Refused;
1154 match r {
1155 Refused::Foreign { branch, path, why } => format!(
1156 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1157 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1158 から、やり直してください(詳細: {why})"
1159 ),
1160 Refused::Unsafe { branch, path, why } => format!(
1161 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1162 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1163 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1164 ),
1165 Refused::ReleaseFailed { branch, path, run } => format!(
1166 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1167 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1168 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1169 ),
1170 }
1171 },
1172 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1173 reviewers_never_answered: |missing, rounds| {
1174 format!(
1175 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1176 クリーンとは認めません"
1177 )
1178 },
1179};
1180
1181fn phrases(language: &str) -> &'static Phrases {
1182 if crate::lang::is_japanese(language) {
1183 &PHRASES_JA
1184 } else {
1185 &PHRASES_EN
1186 }
1187}
1188
1189fn language_of(task: &Task, fallback: &Path) -> String {
1193 crate::lang::of_repo(&repo_for(task, fallback))
1194}
1195
1196pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1200 if q.node == crate::conduct::NODE {
1201 return tasks.iter().find(|t| t.id == q.run);
1202 }
1203 tasks.iter().find(|t| t.runs.contains(&q.run))
1204}
1205
1206fn repair_choice_offers(questions: &Questions) {
1209 for q in questions.list().into_iter().filter(|q| {
1210 q.status.open()
1211 && q.node == crate::conduct::NODE
1212 && q.choices
1213 .iter()
1214 .any(|c| crate::choice_effect::describe(q, c).is_none())
1215 }) {
1216 let _ = questions.update(&q.id, |current| {
1217 if current.status.open() {
1218 crate::choice_effect::executable_conductor(current);
1219 if let Some(deputy) = &mut current.deputy {
1220 deputy.brief = crate::deputy::brief(
1221 ¤t.run,
1222 ¤t.detail,
1223 ¤t.choices,
1224 ¤t.actions,
1225 );
1226 }
1227 }
1228 Ok(())
1229 });
1230 }
1231}
1232
1233pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1236 apply_maintenance_choices(queue, questions, home).await;
1237 apply_task_choice_actions(queue, questions, home);
1238}
1239
1240async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1243 for q in questions.list() {
1244 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1245 continue;
1246 };
1247 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1250 continue;
1251 }
1252 let dir = home.join("choice-actions");
1253 if std::fs::create_dir_all(&dir).is_err() {
1254 continue;
1255 }
1256 let Ok(lock) = std::fs::OpenOptions::new()
1257 .create(true)
1258 .truncate(false)
1259 .write(true)
1260 .open(dir.join(format!("{}.lock", q.id)))
1261 else {
1262 continue;
1263 };
1264 if crate::pr_head::try_lock_file(&lock).is_err() {
1265 continue;
1266 }
1267 let fresh = !home
1268 .join("choice-actions")
1269 .join(format!("{}.json", q.id))
1270 .exists();
1271 let result = async {
1272 let receipt = dir.join(format!("{}.json", q.id));
1273 if receipt.exists() {
1274 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1275 anyhow::ensure!(
1276 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1277 "previous maintenance attempt did not confirm completion; inspect its receipt"
1278 );
1279 return Ok::<_, anyhow::Error>(());
1280 }
1281 let record = |status: &str| -> anyhow::Result<()> {
1284 let tmp = receipt.with_extension("tmp");
1285 std::fs::write(
1286 &tmp,
1287 serde_json::to_vec(&serde_json::json!({
1288 "action": action, "status": status,
1289 }))?,
1290 )?;
1291 std::fs::rename(tmp, &receipt)?;
1292 Ok(())
1293 };
1294 let applied = async {
1295 match action {
1296 ask::ChoiceAction::ResumeRun { run } => {
1297 crate::resume::validate(home, run)?;
1298 record("started")?;
1299 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1300 crate::resume::validate_unqueued(home, run)?;
1301 crate::resume::start(home, run.clone()).await?;
1302 }
1303 }
1304 ask::ChoiceAction::ReleaseTask {
1305 task,
1306 held_at,
1307 reason,
1308 } => {
1309 anyhow::ensure!(
1310 queue.resolve_id(task)? == *task,
1311 "task action requires an exact task ID"
1312 );
1313 let _claim = queue.claim(task)?;
1314 let mut task = queue.get(task)?;
1315 if task.action_applied(&q.id) {
1316 return Ok(());
1317 }
1318 if q.node == crate::conduct::NODE
1323 && q.run == task.id
1324 && task.status == TaskStatus::Blocked
1325 && task.blocked_from == Some(TaskStatus::Held)
1326 && task.blocked_by.as_slice() == [q.id.as_str()]
1327 {
1328 task.unblock(&q.id);
1329 }
1330 anyhow::ensure!(
1331 task.status == TaskStatus::Held
1332 && task.held_at == Some(*held_at)
1333 && task.hold_reason.as_deref() == reason.as_deref()
1334 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1335 "task no longer has the hold this question names"
1336 );
1337 task.release_operator("choice", Some(&q.id));
1338 task.mark_action_applied(&q.id);
1339 queue.put(&mut task)?;
1340 }
1341 ask::ChoiceAction::AbandonRun { run } => {
1342 record("started")?;
1343 crate::driver::abandon_proven_dead(home, run)?;
1344 }
1345 ask::ChoiceAction::RerunChecks {
1346 repo,
1347 pr,
1348 head,
1349 runs,
1350 } => {
1351 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1352 .await?;
1353 }
1354 _ => unreachable!(),
1355 }
1356 Ok::<_, anyhow::Error>(())
1357 }
1358 .await;
1359 match applied {
1360 Ok(()) => record("applied"),
1361 Err(e) => {
1362 record(&format!("refused or failed: {e:#}"))?;
1363 Err(e)
1364 }
1365 }
1366 }
1367 .await;
1368 if home
1373 .join("choice-actions")
1374 .join(format!("{}.json", q.id))
1375 .exists()
1376 && let Some(listed) = task_of_question(&queue.list(), &q)
1377 && !listed.action_applied(&q.id)
1378 && let Ok(_claim) = queue.claim(&listed.id)
1379 && let Ok(mut origin) = queue.get(&listed.id)
1380 {
1381 if !origin.operator_held()
1382 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1383 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1384 {
1385 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1386 origin.hold_machine(Some(format!(
1387 "Maintenance choice {} settled; inspect its target before releasing this task",
1388 q.id
1389 )));
1390 } else {
1391 origin.unblock(&q.id);
1392 }
1393 }
1394 origin.mark_action_applied(&q.id);
1395 let _ = queue.put(&mut origin);
1396 }
1397 if let Err(e) = result
1398 && fresh
1399 {
1400 notices::raise_in(
1401 home,
1402 Notice::warn(
1403 &format!("action:{}", q.id),
1404 format!("Choice action {} was not completed: {e:#}", q.id),
1405 ),
1406 );
1407 }
1408 }
1409}
1410
1411pub(crate) async fn rerun_checks_once(
1418 home: &Path,
1419 request: &str,
1420 repo: &Path,
1421 pr: &str,
1422 head: &str,
1423 runs: &[String],
1424 started: impl FnOnce() -> anyhow::Result<()>,
1425) -> anyhow::Result<()> {
1426 use crate::release_watch::{GhForge, ReleaseForge};
1427 anyhow::ensure!(
1428 !runs.is_empty()
1429 && runs.len() <= 20
1430 && runs
1431 .iter()
1432 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1433 && !pr.starts_with('-'),
1434 "invalid rerun targets"
1435 );
1436 let snap = GhForge.snapshot(repo, pr).await?;
1437 anyhow::ensure!(
1438 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1439 "pull request head or lifecycle changed"
1440 );
1441 anyhow::ensure!(
1442 runs.iter().all(|r| snap
1443 .checks
1444 .iter()
1445 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1446 "named workflow is no longer failed on this pull request"
1447 );
1448 started()?;
1449 let dir = home.join("choice-actions");
1450 std::fs::create_dir_all(&dir)?;
1451 let rerun_lock = std::fs::OpenOptions::new()
1452 .create(true)
1453 .truncate(false)
1454 .write(true)
1455 .open(dir.join("reruns.lock"))?;
1456 crate::pr_head::try_lock_file(&rerun_lock)?;
1457 let ledger = dir.join("reruns.json");
1458 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1459 serde_json::from_slice(&std::fs::read(&ledger)?)?
1460 } else {
1461 Default::default()
1462 };
1463 for run in runs {
1464 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1465 if !recorded.insert(key) {
1466 continue;
1467 }
1468 let tmp = ledger.with_extension("tmp");
1469 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1470 std::fs::rename(tmp, &ledger)?;
1471 GhForge.rerun(repo, run).await?;
1472 }
1473 Ok(())
1474}
1475
1476pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1485 let tasks = queue.list();
1486 for q in questions.list() {
1487 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1489 continue;
1490 }
1491 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1492 continue;
1493 }
1494 let Some(listed) = task_of_question(&tasks, &q) else {
1495 if q.node == land::APPROVAL_NODE
1496 && q.resolution().as_deref().is_some_and(|choice| {
1497 choice == land::REPAIR || crate::merge_decision::stops(choice)
1498 })
1499 {
1500 notices::raise_in(
1501 home,
1502 Notice::warn(
1503 &format!("repair-owner:{}", q.id),
1504 format!(
1505 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1506 q.run, q.run
1507 ),
1508 ),
1509 );
1510 }
1511 continue;
1512 };
1513 if listed.action_applied(&q.id) {
1514 continue;
1515 }
1516 let Ok(_claim) = queue.claim(&listed.id) else {
1517 continue;
1518 };
1519 let Ok(mut task) = queue.get(&listed.id) else {
1520 continue;
1521 };
1522 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1525 let blockers: Vec<_> = task
1526 .blocked_by
1527 .iter()
1528 .filter(|id| {
1529 crate::queue::dependency(queue, questions, id)
1530 == crate::queue::Dependency::Question(q.id.clone())
1531 })
1532 .cloned()
1533 .collect();
1534 if !blockers.is_empty() {
1535 for id in blockers {
1536 task.unblock(&id);
1537 }
1538 let answer = match &q.answer {
1539 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1540 None => String::new(),
1541 };
1542 task.record_answer(q.summary.clone(), answer);
1543 }
1544 }
1545 let language = language_of(&task, Path::new("."));
1546 let decision = decide_action(&task, &q, phrases(&language), |id| {
1547 RunState::load_under(id, home)
1548 });
1549 let ran = matches!(
1550 decision,
1551 ActionDecision::Resume(_)
1552 | ActionDecision::Requeue
1553 | ActionDecision::Hold
1554 | ActionDecision::Done
1555 );
1556 if ran && asker_may_still_read(questions, &q, home) {
1557 continue;
1566 }
1567 let done = matches!(decision, ActionDecision::Done);
1568 match decision {
1569 ActionDecision::Skip => continue,
1570 ActionDecision::Resume(run) => {
1571 task.release_operator("choice", Some(&q.id));
1572 task.resume_override = Some(crate::queue::OperatorResume {
1573 question_id: q.id.clone(),
1574 at: Timestamp::now(),
1575 conductor_rehold: None,
1576 forced: true,
1577 pinned_run: Some(run),
1578 });
1579 }
1580 ActionDecision::Stale => {}
1581 ActionDecision::Requeue => {
1582 task.release_operator("choice-requeue", Some(&q.id));
1583 task.requeue();
1584 }
1585 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1586 ActionDecision::Done => task.succeed(),
1587 ActionDecision::Refuse(why) => {
1588 if task.operator_held() {
1589 task.hold_reason = Some(match task.hold_reason.take() {
1592 Some(reason) => format!("{reason}\n\n{why}"),
1593 None => why,
1594 });
1595 } else {
1596 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1597 task.hold_machine(Some(why));
1598 }
1599 notices::raise(
1600 Notice::warn(
1601 &format!("action:{}", q.id),
1602 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1603 )
1604 .link(Link::Task {
1605 id: task.id.clone(),
1606 }),
1607 );
1608 }
1609 }
1610 task.mark_action_applied(&q.id);
1611 record(queue, &mut task);
1614 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1615 supersede_prior_runs(&task, home);
1616 }
1617 let _ = questions.update(&q.id, |r| {
1620 r.answer_delivered = true;
1621 Ok(())
1622 });
1623 }
1624}
1625
1626fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1630 let now = Timestamp::now();
1631 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1632 return true;
1633 }
1634 q.node != crate::conduct::NODE
1635 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1636 s.seats_active()
1637 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1638 })
1639}
1640
1641fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1652 let tasks = queue.list();
1653 let by_id: std::collections::BTreeMap<&str, &Task> =
1654 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1655 let referenced: Vec<&str> = tasks
1656 .iter()
1657 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1658 .collect();
1659
1660 for mut question in questions.list() {
1661 if !question.status.open() || question.node != crate::conduct::NODE {
1662 continue;
1663 }
1664 if referenced
1669 .iter()
1670 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1671 {
1672 continue;
1673 }
1674 let Some(task) = by_id.get(question.run.as_str()) else {
1675 continue;
1676 };
1677 question.abandon(format!(
1678 "task {} no longer waits for this answer",
1679 task.short()
1680 ));
1681 if let Err(e) = questions.put(&mut question) {
1682 tracing::warn!(
1683 "could not retire question {} for task {}: {e:#}",
1684 question.short(),
1685 task.short()
1686 );
1687 }
1688 }
1689}
1690
1691#[derive(Debug, Clone, Copy)]
1698pub struct Verdict {
1699 pub status: RunStatus,
1701 pub left_pr: bool,
1703 pub quota_hit: bool,
1705 pub parked: bool,
1707 pub no_viable_candidates: bool,
1714}
1715
1716pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1773 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1774}
1775
1776fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1778 if verdict.parked {
1784 task.park(detail);
1785 return;
1786 }
1787 match verdict.status {
1788 RunStatus::Merged | RunStatus::Ready
1789 if task.pr_recovery.as_ref().is_some_and(|s| {
1790 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1791 }) =>
1792 {
1793 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1794 }
1795 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1796 RunStatus::AlreadyInBase => task.already_landed(detail),
1797 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1798 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1799 task.stall(detail)
1800 }
1801 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1802 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1803 RunStatus::Blocked => task.fail(detail, max_attempts),
1804 RunStatus::VerifiedNoop => task.handed_off(detail),
1805 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1806 }
1807}
1808
1809pub fn supersede_prior_runs(task: &Task, home: &Path) {
1858 let now = Timestamp::now();
1859 let last_run_succeeded = task
1866 .runs
1867 .last()
1868 .and_then(|id| RunState::load_under(id, home).ok())
1869 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1870 for id in task.superseded_attempts(last_run_succeeded) {
1871 let mut state = match RunState::load_under(id, home) {
1872 Ok(s) => s,
1873 Err(e) => {
1874 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1875 continue;
1876 }
1877 };
1878 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1879 continue;
1880 }
1881 let daemon_claims = is_working_on(home, id, now);
1882 if state.liveness(daemon_claims) == Liveness::Live {
1883 continue;
1884 }
1885 state.status = RunStatus::Superseded;
1886 if let Err(e) = state.save_under(home) {
1887 tracing::warn!("could not mark run {id} superseded: {e:#}");
1888 }
1889 }
1890}
1891
1892fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1914 for task in queue.list() {
1915 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1916 continue;
1917 }
1918 supersede_prior_runs(&task, home);
1919 }
1920}
1921
1922fn settle_and_diagnose(
1929 task: &mut Task,
1930 verdict: Verdict,
1931 detail: &str,
1932 max_attempts: usize,
1933 state: &RunState,
1934) {
1935 let p = phrases(&state.config.graph.language);
1936 if let Some(d) = &state.merge_decision {
1939 use crate::merge_decision as md;
1940 if task.runs.last().is_some_and(|id| id != &state.id) {
1941 return;
1942 }
1943 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1944 task.succeed();
1945 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1946 if state.liveness(false) != Liveness::Dead {
1947 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1948 return;
1949 }
1950 let replacement = md::Replacement {
1951 question: d.question.clone(),
1952 run: state.id.clone(),
1953 pr: d.target.pr.clone(),
1954 agent: agent.to_owned(),
1955 context: format!(
1956 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1957 d.target.pr,
1958 state.id,
1959 serde_json::to_string(&state.reviews).unwrap_or_default()
1960 ),
1961 };
1962 task.requeue();
1963 task.pr_recovery = None;
1964 task.review_of = None;
1965 task.replacement = Some(replacement);
1966 } else {
1967 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1968 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1969 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1970 }
1971 task.mark_action_applied(&d.question);
1972 if let Some(pass) = state.passes.last() {
1973 task.record_pass_at(
1974 &state.id,
1975 pass.index,
1976 Some(pass.started_at),
1977 task.attempts,
1978 None,
1979 );
1980 }
1981 return;
1982 }
1983 if let Some(replacement) = &task.replacement
1984 && !state.parked
1985 && state.resource_block.is_none()
1986 && (state.viable().is_empty()
1987 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1988 {
1989 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1990 return;
1991 }
1992 let attempts_before = task.attempts;
1993 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2000 task.notice_cause(
2001 &format!("resource:{}:{}", block.kind, block.node),
2002 None,
2003 None,
2004 );
2005 task.last_error = Some(detail.to_owned());
2006 task.diagnostic = None;
2007 task.attempts = task.attempts.saturating_sub(1);
2008 if block.kind == "disk pressure" {
2009 task.hold_machine(Some(format!(
2010 "{}{}",
2011 crate::disk::RUNNING_PRESSURE,
2012 block.detail
2013 )));
2014 } else {
2015 task.hold_machine(Some(format!(
2016 "{}{}; no attempt was spent; resume to retry the affected operation",
2017 crate::cache::RESOURCE_HOLD,
2018 block.description()
2019 )));
2020 }
2021 if let Some(pass) = state.passes.last() {
2022 task.record_pass_at(
2023 &state.id,
2024 pass.index,
2025 Some(pass.started_at),
2026 attempts_before,
2027 None,
2028 );
2029 }
2030 return;
2031 }
2032 let unresolved = task.hold_notice_cause.clone();
2033 if !verdict.parked
2034 && task
2035 .pr_recovery
2036 .as_ref()
2037 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2038 && matches!(
2039 verdict.status,
2040 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2041 )
2042 {
2043 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2044 task.succeed();
2045 } else {
2046 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2047 }
2048 } else if !verdict.parked
2049 && task.pr_recovery.is_some()
2050 && matches!(
2051 verdict.status,
2052 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2053 )
2054 && !verdict.quota_hit
2055 {
2056 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2057 } else {
2058 settle_in(task, verdict, detail, max_attempts, p);
2059 }
2060 if !verdict.parked
2061 && !verdict.quota_hit
2062 && matches!(
2063 verdict.status,
2064 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2065 )
2066 {
2067 task.recovery_failed(state);
2068 }
2069 if state.prep_failure.is_some() {
2070 task.hold_notice_cause = unresolved;
2073 task.startup_notice_cause("startup-candidates");
2074 }
2075 if let Some(pass) = state.passes.last() {
2078 let parked = match &pass.end {
2079 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2080 _ => None,
2081 };
2082 task.record_pass_at(
2083 &state.id,
2084 pass.index,
2085 Some(pass.started_at),
2086 attempts_before,
2087 parked,
2088 );
2089 }
2090 if task.status == TaskStatus::Held {
2091 if state.prep_failure.is_none() {
2092 task.notice_cause(
2093 label(verdict.status),
2094 Some(&state.id),
2095 state.passes.last().map(|p| p.index),
2096 );
2097 }
2098 task.diagnostic = diagnostic(state);
2099 note_open_question(task, &state.id, p);
2100 }
2101}
2102
2103fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2118 let Some(home) = crate::run::try_home() else {
2119 return;
2120 };
2121 let open = Questions::at(home.join("questions")).open_for(run);
2122 let Some(q) = open.first() else {
2123 return;
2124 };
2125 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2126 decisions.sort_unstable();
2127 task.notice_cause(
2128 &format!("questions:{}", decisions.join(",")),
2129 Some(run),
2130 None,
2131 );
2132 let base = task.hold_reason.clone().unwrap_or_default();
2133 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2134}
2135
2136fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2147 match last_run {
2148 Some(state) => {
2149 let verdict = Verdict {
2150 status: state.status,
2151 left_pr: state.pr.is_some(),
2152 quota_hit: !state.quota.is_empty(),
2153 parked: state.parked,
2154 no_viable_candidates: state.viable().is_empty(),
2155 };
2156 let detail = format!(
2157 "{}{}",
2158 phrases(&state.config.graph.language).recovered_running,
2159 describe(&state)
2160 );
2161 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2162 }
2163 None => {
2164 let why = phrases(language).no_run_to_recover;
2167 task.last_error = Some(why.to_owned());
2168 task.notice_cause("missing-run", None, None);
2171 task.hold_machine(Some(why.to_owned()));
2172 }
2173 }
2174}
2175
2176fn reclaim_orphaned_running(
2198 queue: &Queue,
2199 max_attempts: usize,
2200 live_workers: &std::collections::BTreeSet<String>,
2201) -> Vec<String> {
2202 let mut reclaimed = Vec::new();
2203 for listed in queue.list() {
2204 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2205 continue;
2206 }
2207 let Ok(_claim) = queue.claim(&listed.id) else {
2208 continue;
2209 };
2210 let Ok(mut task) = queue.get(&listed.id) else {
2214 continue;
2215 };
2216 if task.status != TaskStatus::Running {
2217 continue;
2218 }
2219 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2220 if let Some(state) = &last_run
2230 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2231 {
2232 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2233 }
2234 let language = if last_run.is_none() {
2235 language_of(&task, Path::new("."))
2236 } else {
2237 String::new()
2238 };
2239 reclaim(&mut task, last_run, max_attempts, &language);
2240 if task.status == TaskStatus::Done {
2241 supersede_prior_runs(&task, &crate::run::home());
2242 }
2243 record(queue, &mut task);
2244 reclaimed.push(task.id.clone());
2245 }
2246 reclaimed
2247}
2248
2249fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2281 reclaim_abandoned_runs_with(
2282 home,
2283 now,
2284 crate::proc::pid_status,
2285 crate::proc::process_started_at,
2286 )
2287}
2288
2289fn reclaim_abandoned_runs_with<F, G>(
2295 home: &Path,
2296 now: Timestamp,
2297 query: F,
2298 identity: G,
2299) -> Vec<String>
2300where
2301 F: Fn(u32) -> Option<bool>,
2302 G: Fn(u32) -> Option<String>,
2303{
2304 let mut abandoned = Vec::new();
2305 for entry in std::fs::read_dir(home.join("runs"))
2306 .into_iter()
2307 .flatten()
2308 .flatten()
2309 {
2310 let id = entry.file_name().to_string_lossy().into_owned();
2311 if !crate::run::is_run_id(&id) {
2312 continue;
2313 }
2314 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2317 continue;
2318 };
2319 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2327 continue;
2328 };
2329 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2330 continue;
2331 };
2332 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2333 continue;
2334 }
2335 let daemon_claims = is_working_on(home, &id, now);
2347 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2348 != crate::run::Liveness::Dead
2349 {
2350 continue;
2351 }
2352 state.abandon("daemon");
2353 if let Err(e) = state.save_under(home) {
2354 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2355 continue;
2356 }
2357 if let Some(notice) = notices::run_ended(&state) {
2360 notices::raise_in(home, notice);
2361 }
2362 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2370 tracing::warn!("abandon questions for {id}: {e:#}");
2371 }
2372 abandoned.push(id);
2373 }
2374 abandoned
2375}
2376
2377#[derive(Debug, Clone, Serialize, Deserialize)]
2378struct TaskWorker {
2379 opts: Opts,
2380 queue: PathBuf,
2381 task: String,
2382 #[serde(default)]
2383 lane: PermitKind,
2384 #[serde(default)]
2385 standalone: bool,
2386}
2387
2388fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2389 Ok(crate::worker::active(home)?
2390 .into_iter()
2391 .filter(|job| job.kind == "task")
2392 .collect())
2393}
2394
2395fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2396 task_jobs(home)?
2397 .into_iter()
2398 .filter_map(
2399 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2400 Ok(request) if !request.standalone => Some(Ok(job)),
2401 Ok(_) => None,
2402 Err(error) => Some(Err(error.into())),
2403 },
2404 )
2405 .collect()
2406}
2407
2408#[must_use]
2410pub fn managed(home: &Path) -> bool {
2411 match crate::worker::active(home) {
2412 Err(error) => {
2413 tracing::error!("cannot determine scheduler ownership: {error:#}");
2414 true
2415 }
2416 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2417 Ok(_) => {
2418 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2419 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2420 }
2421 }
2422}
2423
2424pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2426 anyhow::ensure!(
2427 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2428 "the scheduler is stopping; wait for retirement before starting it again"
2429 );
2430 crate::worker::spawn(
2431 home,
2432 "scheduler",
2433 "scheduler",
2434 serde_json::to_value(opts.absolute()?)?,
2435 )
2436 .await
2437}
2438
2439fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2440 home.join("worker-parks").join(format!("{}.json", job.id))
2441}
2442
2443fn request_task_park(
2444 home: &Path,
2445 job: &crate::worker::Job,
2446 cause: crate::run::ParkCause,
2447) -> Result<()> {
2448 let path = park_request_path(home, job);
2449 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2450 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2451 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2452}
2453
2454pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2456 if control == crate::worker::Control::Park {
2457 for job in owned_task_jobs(home)? {
2458 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2459 }
2460 }
2461 crate::worker::request(home, "scheduler", control)
2462}
2463
2464pub async fn restart_detached(home: &Path) -> Result<()> {
2466 let active = crate::worker::active(home)?;
2467 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2468 if prior.is_none() && !managed(home) {
2469 return Ok(());
2470 }
2471 let opts: Opts = match prior {
2472 Some(job) => serde_json::from_value(job.payload)?,
2473 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2474 };
2475 request_control(home, crate::worker::Control::Stop)?;
2476 loop {
2477 let reserved = crate::worker::active(home)?
2478 .iter()
2479 .any(|job| job.kind == "scheduler");
2480 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2481 break;
2482 }
2483 tokio::time::sleep(Duration::from_millis(100)).await;
2484 }
2485 spawn_detached(home, &opts).await?;
2486 Ok(())
2487}
2488
2489pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2491 let opts: Opts = serde_json::from_value(payload)?;
2492 let _ = home;
2493 serve_until(opts, Stop::new()).await
2494}
2495
2496pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2498 let request: TaskWorker = serde_json::from_value(payload)?;
2499 let queue = Queue::at(request.queue);
2500 let key = format!("task:{}", request.task);
2501 let job = task_jobs(home)?
2502 .into_iter()
2503 .find(|job| job.key == key)
2504 .context("task worker reservation is missing")?;
2505 let park_path = park_request_path(home, &job);
2506 let _claim = loop {
2509 match queue.claim(&request.task) {
2510 Ok(claim) => break claim,
2511 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2514 return Err(e.context("claim the task for its worker"));
2515 }
2516 Err(_) => {
2517 sweep_stale_claims(&queue, STALE_CLAIM);
2518 tokio::time::sleep(Duration::from_millis(50)).await;
2519 }
2520 }
2521 };
2522 let mut task = queue.get(&request.task)?;
2523 if !task.status.runnable()
2524 || (!request.standalone
2525 && request.lane
2526 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2527 {
2528 return Ok(());
2529 }
2530 let stop = Stop::new();
2531 let watcher = {
2532 let home = home.to_path_buf();
2533 let stop = stop.clone();
2534 let park_path = park_path.clone();
2535 tokio::spawn(async move {
2536 loop {
2537 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2538 let cause = std::fs::read(&park_path)
2539 .ok()
2540 .and_then(|body| serde_json::from_slice(&body).ok())
2541 .unwrap_or(crate::run::ParkCause::Operator);
2542 stop.park(cause);
2543 return;
2544 }
2545 tokio::time::sleep(Duration::from_millis(200)).await;
2546 }
2547 })
2548 };
2549 let quota = if request.standalone {
2550 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2551 } else {
2552 let status = Arc::new(Mutex::new(Status::new()));
2553 Box::pin(attempt(
2554 &request.opts,
2555 &queue,
2556 &status,
2557 &stop,
2558 crate::graph::Pause::new(),
2559 &mut task,
2560 ))
2561 .await
2562 };
2563 watcher.abort();
2564 let _ = std::fs::remove_file(park_path);
2565 persist_cooldown(home, "a).await?;
2566 Ok(())
2567}
2568
2569pub async fn spawn_claimed(
2572 home: &Path,
2573 opts: &Opts,
2574 queue: &Queue,
2575 task: &Task,
2576 claim: crate::queue::Claim,
2577) -> Result<crate::worker::Job> {
2578 let request = TaskWorker {
2579 opts: opts.absolute()?,
2580 queue: std::path::absolute(queue.root())?,
2581 task: task.id.clone(),
2582 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2583 standalone: true,
2584 };
2585 let job = crate::worker::spawn(
2586 home,
2587 "task",
2588 &format!("task:{}", task.id),
2589 serde_json::to_value(request)?,
2590 )
2591 .await?;
2592 drop(claim);
2593 Ok(job)
2594}
2595
2596pub async fn run_detached_claimed(
2599 opts: &Opts,
2600 queue: &Queue,
2601 task: &mut Task,
2602 claim: crate::queue::Claim,
2603) -> Result<()> {
2604 let home = crate::run::home();
2605 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2606 loop {
2607 crate::worker::active(&home)?;
2610 let current = crate::worker::get(&home, &job.id)?;
2611 if matches!(
2612 current.status,
2613 crate::worker::Status::Finished | crate::worker::Status::Failed
2614 ) {
2615 *task = queue.get(&task.id)?;
2616 anyhow::ensure!(
2617 current.status != crate::worker::Status::Failed,
2618 "task worker failed: {}",
2619 current.result.as_deref().unwrap_or("unknown failure")
2620 );
2621 return Ok(());
2622 }
2623 tokio::time::sleep(Duration::from_millis(200)).await;
2624 }
2625}
2626
2627async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2628 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2629 let _lease = loop {
2630 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2631 break lease;
2632 }
2633 tokio::time::sleep(Duration::from_millis(10)).await;
2634 };
2635 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2636 let path = home.join("task-quota-cooldown.json");
2637 let temporary = path.with_extension("json.tmp");
2638 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2639 std::fs::rename(temporary, path)?;
2640 }
2641 }
2642 Ok(())
2643}
2644
2645fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2646 match std::fs::read(home.join("task-quota-cooldown.json")) {
2647 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2648 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2649 Err(error) => Err(error.into()),
2650 }
2651}
2652
2653pub async fn serve(opts: Opts) -> Result<()> {
2659 tracing::info!(
2660 "automatic update and restart requires magi web; standalone serve does not restart"
2661 );
2662 serve_until(opts, Stop::new()).await
2663}
2664
2665pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2676 let opts = opts.absolute()?;
2677 let home = crate::run::home();
2678 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2679 .context("another scheduler already owns this home")?;
2680 if let Some(reading) = read_status(&home)
2681 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2682 {
2683 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2684 }
2685 if !crate::worker::active(&home)?
2686 .iter()
2687 .any(|job| job.kind == "scheduler")
2688 {
2689 crate::worker::clear_control(&home, "scheduler")?;
2690 }
2691 std::fs::write(
2692 home.join("scheduler-options.json"),
2693 serde_json::to_vec(&opts)?,
2694 )?;
2695 let control = {
2696 let home = home.clone();
2697 let stop = stop.clone();
2698 tokio::spawn(async move {
2699 loop {
2700 match crate::worker::control(&home, "scheduler").ok().flatten() {
2701 Some(crate::worker::Control::Park) => {
2702 let _ = request_control(&home, crate::worker::Control::Park);
2703 stop.park(crate::run::ParkCause::Operator);
2704 break;
2705 }
2706 Some(crate::worker::Control::Stop) => {
2707 stop.stop();
2708 break;
2709 }
2710 None => {}
2711 }
2712 tokio::time::sleep(Duration::from_millis(100)).await;
2713 }
2714 })
2715 };
2716 let signal = {
2717 let stop = stop.clone();
2718 tokio::spawn(async move {
2719 if tokio::signal::ctrl_c().await.is_ok() {
2720 stop.stop();
2721 tracing::info!("scheduler shutdown requested; independent workers continue");
2722 }
2723 })
2724 };
2725
2726 let worktrees_root = opts
2727 .worktrees_root
2728 .clone()
2729 .unwrap_or_else(crate::run::default_worktree_root);
2730 let outcome = drive(
2731 &opts,
2732 &Queue::open(),
2733 &status_path(),
2734 &crate::run::home(),
2735 &worktrees_root,
2736 &stop,
2737 )
2738 .await;
2739
2740 signal.abort();
2741 control.abort();
2742 outcome
2743}
2744
2745async fn drive(
2758 opts: &Opts,
2759 queue: &Queue,
2760 status_file: &Path,
2761 home: &Path,
2762 worktrees_root: &Path,
2763 stop: &Stop,
2764) -> Result<()> {
2765 let recovery_cfg = prepare(&opts.repo, opts).ok();
2773 let daemon_cfg = recovery_cfg
2774 .as_ref()
2775 .map(|c| c.daemon.clone())
2776 .unwrap_or_default();
2777 let reconciled = crate::driver::reconcile(home);
2778 if !reconciled.is_empty() {
2779 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2780 }
2781 let mut initial_status = Status::new();
2782 let owned = owned_task_jobs(home)?;
2783 initial_status.current = current_work(home, Timestamp::now())
2784 .into_iter()
2785 .filter(|current| {
2786 owned
2787 .iter()
2788 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2789 })
2790 .collect();
2791 initial_status.idle = initial_status.current.is_empty();
2792 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2793 recovery_cfg.as_ref(),
2794 daemon_cfg.max_deputies,
2795 ));
2796 let status = Arc::new(Mutex::new(initial_status));
2797 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2798 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2799
2800 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2806
2807 tracing::info!(
2808 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2809 queue.root().display(),
2810 opts.poll.as_secs(),
2811 opts.max_attempts,
2812 concurrency,
2813 if daemon_cfg.pause_for_interrupts {
2814 ", interrupts enabled"
2815 } else {
2816 ""
2817 }
2818 );
2819
2820 let outcome = tokio::select! {
2821 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2822 max_concurrent: concurrency,
2823 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2824 recovery_cfg,
2825 }) => outcome,
2826 () = async {
2827 while !stop.stopped() {
2828 tokio::time::sleep(Duration::from_millis(25)).await;
2829 }
2830 } => {
2831 if stop.parking() {
2832 request_control(home, crate::worker::Control::Park)?;
2833 }
2834 Ok(())
2835 }
2836 };
2837
2838 stop.busy.store(0, Ordering::SeqCst);
2839 beat.abort();
2840 clear_status_at(status_file);
2841 outcome
2842}
2843
2844pub(crate) fn release_watch_settings_from_config(
2847 home: &Path,
2848 repo: &Path,
2849 cfg: &Config,
2850) -> (Vec<PathBuf>, u64) {
2851 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2852 .into_iter()
2853 .map(|r| r.path)
2854 .collect();
2855 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2858 .list()
2859 .into_iter()
2860 .map(|t| t.repo)
2861 .collect();
2862 let runs = home.join("runs");
2863 for id in crate::run::list_ids_in(&runs) {
2864 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2865 .ok()
2866 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2867 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2868 known.extend(repo);
2869 }
2870 for k in known {
2871 let k = k.canonicalize().unwrap_or(k);
2872 if k.join(".git").exists() && !paths.contains(&k) {
2873 paths.push(k);
2874 }
2875 }
2876 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2877 if !paths.contains(&own) {
2878 paths.push(own);
2879 }
2880 (paths, cfg.daemon.release_stall_minutes)
2881}
2882
2883const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2885
2886async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2892 loop {
2893 tokio::time::sleep(HEARTBEAT).await;
2894 let snapshot = {
2895 let mut guard = lock(&status);
2896 guard.updated_at = Timestamp::now();
2897 guard.clone()
2898 };
2899 if let Err(e) = write_status_to(&path, &snapshot) {
2900 tracing::warn!("could not refresh the daemon status file: {e:#}");
2903 }
2904 }
2905}
2906
2907#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2910enum LandResume {
2911 NotLanding,
2914 StillWaiting,
2919 Ready,
2923}
2924
2925fn land_resume_state(task: &Task) -> LandResume {
2929 let Some(run_id) = task.runs.last() else {
2930 return LandResume::NotLanding;
2931 };
2932 let Ok(state) = RunState::load(run_id) else {
2933 return LandResume::NotLanding;
2934 };
2935 if !state.parked {
2936 return LandResume::NotLanding;
2937 }
2938 if state.status == RunStatus::Landing
2939 && state.land_approval.as_ref().is_some_and(|binding| {
2940 ask::Questions::open()
2941 .get(&binding.question)
2942 .is_ok_and(|q| {
2943 q.run == state.id
2944 && q.resolution()
2945 .as_deref()
2946 .is_some_and(crate::merge_decision::stops)
2947 })
2948 })
2949 {
2950 return LandResume::Ready;
2951 }
2952 if !implementers_match(task, &state) {
2953 return LandResume::NotLanding;
2954 }
2955 let node = if state.status == RunStatus::Landing {
2959 land::APPROVAL_NODE
2960 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2961 crate::github_text::ASK_NODE
2962 } else {
2963 return LandResume::NotLanding;
2964 };
2965 let store = ask::Questions::open();
2966 let waiting = store
2967 .list()
2968 .into_iter()
2969 .filter(|q| &q.run == run_id && q.node == node)
2970 .max_by(|a, b| a.id.cmp(&b.id));
2971 let Some(mut q) = waiting else {
2972 return LandResume::Ready;
2973 };
2974 if !q.status.open() {
2975 if q.resolution().as_deref() == Some(land::REPAIR) {
2978 return if state.liveness(false) == Liveness::Live {
2979 LandResume::StillWaiting
2980 } else {
2981 LandResume::NotLanding
2982 };
2983 }
2984 return LandResume::Ready;
2985 }
2986 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2993 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2994 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
2995 q.abandon(format!(
2996 "no answer within {}s of asking",
2997 timeout.as_secs().max(1)
2998 ));
2999 if store.put(&mut q).is_ok() {
3002 return LandResume::Ready;
3003 }
3004 }
3005 LandResume::StillWaiting
3006}
3007
3008const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3016
3017const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3029
3030#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3043enum Interrupt {
3044 Idle,
3046 Parking {
3058 parked: Vec<String>,
3059 interrupt_task: String,
3060 },
3061 Running {
3069 parked: Vec<String>,
3070 interrupt_task: String,
3071 },
3072 Resuming { parked: Vec<String> },
3079}
3080
3081fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3102 match state {
3103 Interrupt::Idle => {
3104 if in_flight.len() != 1 {
3116 return Interrupt::Idle;
3117 }
3118 match runnable.iter().find(|t| t.interrupt) {
3119 Some(t) => Interrupt::Parking {
3120 parked: in_flight.to_vec(),
3121 interrupt_task: t.id.clone(),
3122 },
3123 None => Interrupt::Idle,
3124 }
3125 }
3126 Interrupt::Parking {
3127 parked,
3128 interrupt_task,
3129 } => {
3130 if in_flight.iter().any(|id| parked.contains(id)) {
3131 Interrupt::Parking {
3133 parked,
3134 interrupt_task,
3135 }
3136 } else if in_flight.contains(&interrupt_task) {
3137 Interrupt::Running {
3138 parked,
3139 interrupt_task,
3140 }
3141 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3142 Interrupt::Parking {
3146 parked,
3147 interrupt_task,
3148 }
3149 } else {
3150 Interrupt::Resuming { parked }
3155 }
3156 }
3157 Interrupt::Running {
3158 parked,
3159 interrupt_task,
3160 } => {
3161 if in_flight.contains(&interrupt_task) {
3162 Interrupt::Running {
3163 parked,
3164 interrupt_task,
3165 }
3166 } else {
3167 Interrupt::Resuming { parked }
3173 }
3174 }
3175 Interrupt::Resuming { parked } => {
3176 if in_flight.iter().any(|id| parked.contains(id)) {
3177 Interrupt::Idle
3183 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3184 Interrupt::Resuming { parked }
3185 } else {
3186 Interrupt::Idle
3189 }
3190 }
3191 }
3192}
3193
3194fn advance_interrupt_tick(
3200 enabled: bool,
3201 state: Interrupt,
3202 in_flight: &[String],
3203 runnable: &[Task],
3204) -> Interrupt {
3205 if !enabled {
3206 return Interrupt::Idle;
3207 }
3208 advance_interrupt(state, in_flight, runnable)
3209}
3210
3211fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3216 match state {
3217 Interrupt::Idle => candidates,
3218 Interrupt::Parking {
3219 parked,
3220 interrupt_task,
3221 } => {
3222 if in_flight.iter().any(|id| parked.contains(id)) {
3223 Vec::new()
3224 } else {
3225 candidates
3226 .into_iter()
3227 .filter(|t| &t.id == interrupt_task)
3228 .collect()
3229 }
3230 }
3231 Interrupt::Running { .. } => Vec::new(),
3232 Interrupt::Resuming { parked } => candidates
3240 .into_iter()
3241 .find(|t| parked.contains(&t.id))
3242 .into_iter()
3243 .collect(),
3244 }
3245}
3246
3247struct DispatchLimits {
3251 max_concurrent: usize,
3254 pause_for_interrupts: bool,
3256 recovery_cfg: Option<Config>,
3257}
3258
3259#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3268enum PermitKind {
3269 None,
3274 Urgent,
3281 #[default]
3284 Ordinary,
3285}
3286
3287fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3290 if priority {
3291 PermitKind::None
3292 } else if urgent {
3293 PermitKind::Urgent
3294 } else {
3295 PermitKind::Ordinary
3296 }
3297}
3298
3299fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3300 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3301 (0usize, 0usize),
3302 |(ordinary, urgent), request| match request.lane {
3303 PermitKind::Ordinary => (ordinary + 1, urgent),
3304 PermitKind::Urgent => (ordinary, urgent + 1),
3305 PermitKind::None => (ordinary, urgent),
3306 },
3307 );
3308 (
3309 limit.saturating_sub(ordinary),
3310 1usize.saturating_sub(urgent),
3311 )
3312}
3313
3314struct DispatchClaim {
3315 task: Task,
3316 claim: crate::queue::Claim,
3317 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3318}
3319
3320fn claim_dispatch(
3323 queue: &Queue,
3324 candidate: &Task,
3325 cooling_down: bool,
3326 ordinary: &Arc<tokio::sync::Semaphore>,
3327 urgent: &Arc<tokio::sync::Semaphore>,
3328) -> Option<DispatchClaim> {
3329 let Ok(claim) = queue.claim(&candidate.id) else {
3330 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3331 return None;
3332 };
3333 let task = match queue.get(&candidate.id) {
3334 Ok(t) if t.status.runnable() => t,
3335 Ok(_) => return None,
3336 Err(e) => {
3337 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3338 return None;
3339 }
3340 };
3341 let resume = land_resume_state(&task);
3342 if resume == LandResume::StillWaiting {
3343 return None;
3344 }
3345 let priority = resume == LandResume::Ready;
3346 if !priority && cooling_down {
3347 return None;
3348 }
3349 let permit = match permit_kind(priority, task.urgent) {
3352 PermitKind::None => None,
3353 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3354 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3355 };
3356 let stamp = Timestamp::now();
3359 let mut task = task;
3360 task.dispatched_at = Some(stamp);
3361 if let Err(e) = queue.modify(&task.id, |t| {
3362 t.dispatched_at = Some(stamp);
3363 true
3364 }) {
3365 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3366 }
3367 Some(DispatchClaim {
3368 task,
3369 claim,
3370 permit,
3371 })
3372}
3373
3374#[derive(Serialize, Deserialize)]
3393struct ServiceWorker {
3394 opts: Opts,
3395 queue: PathBuf,
3396 worktrees_root: PathBuf,
3397 operation: String,
3398}
3399
3400fn load_conductor(home: &Path) -> Result<Conductor> {
3401 match std::fs::read(home.join("conductor-state.json")) {
3402 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3403 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3404 Err(error) => Err(error.into()),
3405 }
3406}
3407
3408fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3409 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3410 .into_iter()
3411 .map(|c| c.task)
3412 .collect();
3413 let stalled = stalled_tasks(queue, home, Timestamp::now());
3414 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3415 let finished = finished_tasks(queue)
3416 .into_iter()
3417 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3418 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3419 .collect();
3420 let queued = queued_tasks(queue)
3421 .into_iter()
3422 .filter(|t| !live.contains(&t.id))
3423 .collect();
3424 (queued, stalled, finished)
3425}
3426
3427async fn start_service(
3428 home: &Path,
3429 opts: &Opts,
3430 queue: &Queue,
3431 worktrees_root: &Path,
3432 operation: &str,
3433 cfg: Option<&Config>,
3434 busy: bool,
3435) -> Result<()> {
3436 let fetch_cfg = if operation == "fetch" {
3439 match prepare(&opts.repo, opts) {
3440 Ok(cfg) => Some(cfg),
3441 Err(error) => {
3442 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3443 return Ok(());
3444 }
3445 }
3446 } else {
3447 None
3448 };
3449 let cfg = fetch_cfg.as_ref().or(cfg);
3450 let ticks = home.join("service-ticks");
3451 std::fs::create_dir_all(&ticks)?;
3452 let receipt = ticks.join(format!("daemon-{operation}"));
3453 let last = std::fs::read(&receipt)
3454 .ok()
3455 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3456 let interval = match operation {
3457 "disk-pressure" => 30,
3458 "cache" => CACHE_CHECK_INTERVAL_SECS,
3459 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3460 _ => opts.poll.as_secs().max(1),
3461 };
3462 let now = Timestamp::now();
3463 if !cache_check_due(last, now, interval) {
3464 return Ok(());
3465 }
3466 let key = format!("service:daemon-{operation}");
3467 if crate::worker::active(home)?
3468 .iter()
3469 .any(|job| job.key == key)
3470 {
3471 return Ok(());
3472 }
3473 let eligible = match operation {
3474 "conductor" => {
3475 let (queued, stalled, finished) = conductor_work(queue, home);
3476 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3477 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3478 }
3479 "maintenance" => Questions::at(home.join("questions"))
3480 .list()
3481 .iter()
3482 .any(|q| {
3483 q.chosen_action().is_some_and(|action| action.maintenance())
3484 && !home
3485 .join("choice-actions")
3486 .join(format!("{}.json", q.id))
3487 .exists()
3488 }),
3489 "housekeeping" => !busy && runnable(queue).is_empty(),
3490 "cache" => {
3491 !busy
3492 && !runnable(queue).is_empty()
3493 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3494 }
3495 "disk-pressure" => busy,
3496 "fetch" => {
3497 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3498 }
3499 _ => false,
3500 };
3501 if !eligible {
3502 return Ok(());
3503 }
3504 let request = ServiceWorker {
3505 opts: opts.absolute()?,
3506 queue: std::path::absolute(queue.root())?,
3507 worktrees_root: std::path::absolute(worktrees_root)?,
3508 operation: operation.to_owned(),
3509 };
3510 if let Err(error) =
3511 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3512 {
3513 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3514 }
3515 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3516 Ok(())
3517}
3518
3519pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3522 let request: ServiceWorker = serde_json::from_value(payload)?;
3523 let opts = &request.opts;
3524 let queue = Queue::at(request.queue);
3525 let questions = Questions::at(home.join("questions"));
3526 match request.operation.as_str() {
3527 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3528 "conductor" => {
3529 let path = home.join("conductor-state.json");
3530 let mut conductor = load_conductor(home)?;
3531 let (queued, stalled, finished) = conductor_work(&queue, home);
3532 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3533 && conductor.worth_a_look(&queue, &stalled, &finished)
3534 {
3535 let cfg = prepare(&opts.repo, opts)?;
3536 conductor
3537 .maybe_run(
3538 &cfg,
3539 &opts.repo,
3540 &queue,
3541 &questions,
3542 home,
3543 &queued,
3544 &stalled,
3545 &finished,
3546 opts.max_attempts,
3547 )
3548 .await;
3549 let temporary = path.with_extension("json.tmp");
3550 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3551 std::fs::rename(temporary, path)?;
3552 }
3553 }
3554 "fetch" => {
3555 let cfg = prepare(&opts.repo, opts)?;
3556 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3557 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3558 tracing::debug!("fetch origins: {outcome:?}");
3559 }
3560 }
3561 "cache" => {
3562 if task_jobs(home)?.is_empty() {
3563 let mut checked = None;
3564 maybe_prune_cache_between_runs(
3565 &opts.repo,
3566 opts,
3567 home,
3568 &Stop::new(),
3569 &mut checked,
3570 Timestamp::now(),
3571 )
3572 .await;
3573 }
3574 }
3575 "disk-pressure" => {
3576 if let Ok(cfg) = prepare(&opts.repo, opts) {
3577 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3578 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3579 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3580 }
3581 for current in current_work(home, Timestamp::now()) {
3582 if let Ok(state) = RunState::load_under(¤t.run, home)
3583 && let Some(reason) = disk_gate(&state.repo, &state.config)
3584 {
3585 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3586 job.key.strip_prefix("task:") == Some(current.task.as_str())
3587 }) {
3588 request_task_park(
3589 home,
3590 &job,
3591 crate::run::ParkCause::Interrupt {
3592 reason: reason.clone(),
3593 },
3594 )?;
3595 }
3596 notices::raise_in(
3597 home,
3598 Notice::warn(
3599 "disk:pressure",
3600 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3601 ),
3602 );
3603 }
3604 }
3605 }
3606 }
3607 "housekeeping" => {
3608 if task_jobs(home)?.is_empty() {
3609 if let Ok(cfg) = prepare(&opts.repo, opts) {
3610 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3611 }
3612 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3613 resweep_superseded_attempts(&queue, home);
3614 triage_held(&queue, home, opts).await;
3615 }
3616 }
3617 operation => bail!("unknown daemon service {operation}"),
3618 }
3619 Ok(())
3620}
3621
3622fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3626 let key = format!("task:{}", request.task);
3627 let payload = serde_json::to_value(request)?;
3628 if crate::worker::active(home)?
3629 .iter()
3630 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3631 {
3632 tracing::warn!(
3633 "task {} acquired a competing worker reservation: {error:#}",
3634 crate::run::short_of(&request.task)
3635 );
3636 return Ok(());
3637 }
3638 tracing::error!("task worker startup stopped dispatch: {error:#}");
3639 notices::raise_in(
3640 home,
3641 Notice::error(
3642 "loop:worker-startup",
3643 "A task worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Startup details are kept in the local execution log.",
3644 ),
3645 );
3646 Err(error).with_context(|| {
3647 format!(
3648 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3649 crate::run::short_of(&request.task)
3650 )
3651 })
3652}
3653
3654async fn poll(
3655 opts: &Opts,
3656 queue: &Queue,
3657 status: &Arc<Mutex<Status>>,
3658 home: &Path,
3659 worktrees_root: &Path,
3660 stop: &Stop,
3661 limits: DispatchLimits,
3662) -> Result<()> {
3663 let mut attempted = Vec::new();
3664 let interrupt_path = home.join("scheduler-interrupt.json");
3665 let mut interrupt = match std::fs::read(&interrupt_path) {
3666 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3667 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3668 Err(error) => return Err(error.into()),
3669 };
3670 let mut previous = std::collections::BTreeSet::<String>::new();
3671 while !stop.stopped() {
3672 let jobs: Vec<_> = crate::worker::active(home)?
3673 .into_iter()
3674 .filter(|job| job.kind == "task")
3675 .collect();
3676 let requests: Vec<_> = jobs
3677 .iter()
3678 .map(|job| {
3679 serde_json::from_value::<TaskWorker>(job.payload.clone())
3680 .with_context(|| format!("read live task worker {}", job.id))
3681 })
3682 .collect::<Result<_>>()?;
3683 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3684 let owned: std::collections::BTreeSet<_> = requests
3685 .iter()
3686 .filter(|r| !r.standalone)
3687 .map(|r| r.task.clone())
3688 .collect();
3689 let current: Vec<_> = requests
3690 .iter()
3691 .zip(&jobs)
3692 .filter(|(request, _)| !request.standalone)
3693 .map(|(request, job)| Current {
3694 worker: Some(job.id.clone()),
3695 task: request.task.clone(),
3696 run: queue
3697 .get(&request.task)
3698 .ok()
3699 .and_then(|task| task.runs.last().cloned())
3700 .unwrap_or_default(),
3701 })
3702 .collect();
3703 {
3704 let mut snapshot = lock(status);
3705 snapshot.polls += 1;
3706 snapshot.completed += previous.difference(&owned).count();
3707 snapshot.current = current;
3708 snapshot.idle = owned.is_empty();
3709 }
3710 previous = owned.clone();
3711 stop.busy.store(owned.len(), Ordering::SeqCst);
3712
3713 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3716 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3717 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3718 let mut permits = Vec::new();
3719 sweep_stale_claims(queue, STALE_CLAIM);
3720 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3721 reclaim_abandoned_runs(home, Timestamp::now());
3722 let questions = Questions::at(home.join("questions"));
3723
3724 crate::task_chat::sweep(
3727 queue,
3728 &crate::talk::Talks::at(home.join("talks")),
3729 &questions,
3730 home,
3731 );
3732 repair_choice_offers(&questions);
3733 resolve_blockers(queue, &questions);
3734 apply_task_choice_actions(queue, &questions, home);
3735 triage::apply_answers(queue, &questions);
3736 reconcile_task_questions(queue, &questions);
3737 let talks = crate::talk::Talks::at(home.join("talks"));
3738 crate::chat_notify::sweep(queue, &talks, home);
3739 crate::chat_watch::sweep(home, &talks);
3740 let mut turns = tokio::task::JoinSet::new();
3741 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3742 while turns.join_next().await.is_some() {}
3744
3745 crate::consult::start_services(
3746 home,
3747 &opts.repo,
3748 limits.recovery_cfg.as_ref(),
3749 opts.config.as_deref(),
3750 )
3751 .await?;
3752 for operation in [
3753 "conductor",
3754 "maintenance",
3755 "housekeeping",
3756 "disk-pressure",
3757 "cache",
3758 "fetch",
3759 ] {
3760 start_service(
3761 home,
3762 opts,
3763 queue,
3764 worktrees_root,
3765 operation,
3766 limits.recovery_cfg.as_ref(),
3767 !live.is_empty(),
3768 )
3769 .await?;
3770 }
3771 let latest = crate::worker::active(home)?;
3772 if latest.iter().any(|job| job.key == "service:daemon-cache")
3773 || latest
3774 .iter()
3775 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3776 {
3777 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3778 continue;
3779 }
3780 let candidates: Vec<Task> = runnable(queue)
3781 .into_iter()
3782 .filter(|task| !live.contains(&task.id))
3783 .filter(|task| !opts.once || !attempted.contains(&task.id))
3784 .collect();
3785 let in_flight: Vec<_> = owned.iter().cloned().collect();
3786 let prior_interrupt = std::mem::discriminant(&interrupt);
3787 interrupt = advance_interrupt_tick(
3788 limits.pause_for_interrupts,
3789 interrupt,
3790 &in_flight,
3791 &candidates,
3792 );
3793 if prior_interrupt != std::mem::discriminant(&interrupt) {
3796 let temporary = interrupt_path.with_extension("json.tmp");
3797 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3798 std::fs::rename(temporary, &interrupt_path)?;
3799 }
3800 if let Interrupt::Parking {
3801 parked,
3802 interrupt_task,
3803 } = &interrupt
3804 {
3805 for job in jobs.iter().filter(|job| {
3806 parked
3807 .iter()
3808 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3809 }) {
3810 if crate::worker::control(home, &job.key)?.is_none() {
3811 request_task_park(
3812 home,
3813 job,
3814 crate::run::ParkCause::Interrupt {
3815 reason: format!(
3816 "task {} asked to run first",
3817 crate::run::short_of(interrupt_task)
3818 ),
3819 },
3820 )?;
3821 }
3822 }
3823 }
3824 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3825 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3826 let mut started = false;
3827 let mut resync = false;
3828 for candidate in candidates {
3829 if stop.stopped() {
3830 break;
3831 }
3832 let Some(DispatchClaim {
3833 task,
3834 claim,
3835 permit,
3836 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3837 else {
3838 continue;
3839 };
3840 if crate::worker::active(home)?
3841 .iter()
3842 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3843 {
3844 drop(claim);
3845 resync = true;
3846 break;
3847 }
3848 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3849 let request = TaskWorker {
3850 opts: opts.absolute()?,
3851 queue: std::path::absolute(queue.root())?,
3852 task: task.id.clone(),
3853 lane,
3854 standalone: false,
3855 };
3856 if let Err(error) = crate::worker::spawn(
3859 home,
3860 "task",
3861 &format!("task:{}", task.id),
3862 serde_json::to_value(&request)?,
3863 )
3864 .await
3865 {
3866 task_worker_spawn_error(home, &request, error)?;
3867 drop(claim);
3868 resync = true;
3869 break;
3870 }
3871 drop(claim);
3872 permits.extend(permit);
3873 attempted.push(task.id);
3874 started = true;
3875 }
3876 if started || resync {
3877 continue;
3878 }
3879 if owned.is_empty() && opts.once {
3880 while crate::worker::active(home)?
3881 .iter()
3882 .any(|job| job.key == "service:daemon-housekeeping")
3883 {
3884 stop.idle(RECHECK_WHILE_BUSY).await;
3885 if stop.stopped() {
3886 return Ok(());
3887 }
3888 }
3889 break;
3890 }
3891 stop.idle(if live.is_empty() {
3892 opts.poll
3893 } else {
3894 RECHECK_WHILE_BUSY.min(opts.poll)
3895 })
3896 .await;
3897 }
3898 stop.busy.store(0, Ordering::SeqCst);
3901 Ok(())
3902}
3903
3904async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
3908 let busy = |error: &anyhow::Error| {
3909 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
3910 error.kind() == std::io::ErrorKind::WouldBlock
3911 || cfg!(windows) && error.raw_os_error() == Some(33)
3912 })
3913 };
3914 match crate::pr_head::owner_lock(home, id) {
3915 Ok(lock) => return Ok(lock),
3916 Err(error) if busy(&error) => {}
3917 Err(error) => return Err(error),
3918 }
3919 let wait = RunState::load_under(id, home)?
3920 .config
3921 .graph
3922 .timeout_resource_wait;
3923 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
3924}
3925
3926async fn public_resume_owner(
3930 home: &Path,
3931 runs: &[String],
3932 repo: &Path,
3933) -> Result<Option<RunState>> {
3934 let mut owner = None;
3935 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3936 let _lock = resume_owner_lock(home, id).await?;
3937 let state = RunState::load_under(id, home)?;
3938 if state.released()
3939 || state.retired_to.is_some()
3940 || state.pr_taken_over_by.is_some()
3941 || !crate::handover::landing_checkpoint(&state)
3942 {
3943 continue;
3944 }
3945 anyhow::ensure!(
3946 crate::git::toplevel(&state.repo).await?.canonicalize()?
3947 == crate::git::toplevel(repo).await?.canonicalize()?,
3948 "public owner repository differs; inspect run lineage"
3949 );
3950 anyhow::ensure!(
3951 owner.is_none(),
3952 "multiple retained public owners; inspect run lineage"
3953 );
3954 anyhow::ensure!(
3955 crate::handover::can_resume(&state),
3956 "public owner cannot resume"
3957 );
3958 let claimed = is_working_on_other(home, id, Timestamp::now())?;
3959 anyhow::ensure!(
3960 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3961 "public owner's driver is live or unknown; wait before retrying"
3962 );
3963 owner = Some(state);
3964 }
3965 Ok(owner)
3966}
3967
3968async fn attempt(
3974 opts: &Opts,
3975 queue: &Queue,
3976 status: &Arc<Mutex<Status>>,
3977 stop: &Stop,
3978 interrupt_pause: crate::graph::Pause,
3979 task: &mut Task,
3980) -> Vec<QuotaLoss> {
3981 let repo = repo_for(task, &opts.repo);
3982 tracing::info!(
3983 "task {} — {} (repo {})",
3984 task.short(),
3985 task.title,
3986 repo.display()
3987 );
3988
3989 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3990 Ok(owner) => owner,
3991 Err(e) => {
3992 task.hold_machine(Some(format!(
3993 "Cannot verify existing public owner: {e:#}; no attempt started."
3994 )));
3995 record(queue, task);
3996 return Vec::new();
3997 }
3998 };
3999 let resume_runs = public_owner
4000 .as_ref()
4001 .map(|s| vec![s.id.clone()])
4002 .unwrap_or_else(|| task.runs.clone());
4003
4004 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4008 && previous
4009 .merge_decision
4010 .as_ref()
4011 .is_some_and(|decision| !decision.closed)
4012 && previous.status != RunStatus::Landing
4013 && !crate::merge_decision::resumable_hold(&previous)
4014 {
4015 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4016 record(queue, task);
4017 return Vec::new();
4018 }
4019 let stop_owner = resume_runs
4020 .last()
4021 .and_then(|id| RunState::load(id).ok())
4022 .filter(|state| {
4023 crate::merge_decision::resumable_hold(state)
4024 || state.status == RunStatus::Landing
4025 && state.land_approval.as_ref().is_some_and(|binding| {
4026 Questions::open().get(&binding.question).is_ok_and(|q| {
4027 q.run == state.id
4028 && q.resolution()
4029 .as_deref()
4030 .is_some_and(crate::merge_decision::stops)
4031 })
4032 })
4033 });
4034 if let Some(owner) = stop_owner {
4035 let mut runner = match Runner::resume(&owner.id) {
4036 Ok(runner) => runner,
4037 Err(e) => {
4038 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4039 record(queue, task);
4040 return Vec::new();
4041 }
4042 };
4043 runner.on_pause(stop.pause());
4044 runner.watch_interrupt(interrupt_pause);
4045 let quota_before = runner.state.quota.clone();
4046 task.start(owner.id.clone());
4047 record(queue, task);
4048 lock(status).current.push(Current {
4049 worker: None,
4050 task: task.id.clone(),
4051 run: owner.id,
4052 });
4053 runner.state.next_pass_task_entry = true;
4054 let result = runner.execute().await;
4055 return finish_attempt(
4056 opts.max_attempts,
4057 queue,
4058 task,
4059 &runner.state,
4060 "a_before,
4061 result,
4062 );
4063 }
4064
4065 if let Some(owner) = &public_owner
4066 && (task.fresh_start || !may_resume_run(owner, task))
4067 {
4068 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4069 record(queue, task);
4070 return Vec::new();
4071 }
4072
4073 let unfinished = (!task.fresh_start)
4074 .then(|| unfinished_run(&resume_runs, task.short()))
4075 .flatten()
4076 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4077 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4078 if !task.fresh_start
4079 && task.pr_recovery.is_none()
4080 && task.review_branch.is_none()
4081 && task.resume_override.is_none()
4082 && previous
4083 .as_ref()
4084 .is_some_and(|run| run.retired_to.is_some())
4085 && unfinished.is_none()
4086 {
4087 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4088 record(queue, task);
4089 return Vec::new();
4090 }
4091
4092 let saved_recovery = previous.as_ref().filter(|run| {
4093 task.replacement.is_none()
4094 && task.pr_recovery.as_ref().is_some_and(|selected| {
4095 selected.check_task(&task.id, &task.instruction).is_ok()
4096 && selected_starter(selected, Some(run), unfinished.as_deref())
4097 == Starter::Resume(run.id.clone())
4098 })
4099 });
4100
4101 let explicit = match &task.overrides {
4105 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4106 Some(_) => opts.config.as_deref(),
4107 None => opts.config.as_deref(),
4108 };
4109 let (mut config, layers) = if let Some(run) = saved_recovery {
4110 let mut saved = run.clone();
4113 if let Err(e) = saved.configure_recovery_resume() {
4114 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4115 record(queue, task);
4116 return Vec::new();
4117 }
4118 (saved.config, Vec::new())
4119 } else {
4120 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4121 Ok(loaded) => loaded,
4122 Err(e) => {
4123 let reason = format!("[config] {e:#}");
4124 record_config_failure(task, &e);
4125 task.last_error = Some(reason.clone());
4126 task.hold_machine(Some(reason));
4127 record(queue, task);
4128 return Vec::new();
4129 }
4130 };
4131 if let Err(e) = apply_options(&mut config, opts, task) {
4132 task.hold_manual(Some(format!(
4133 "[config] {e:#}; correct the merge override before releasing the task"
4134 )));
4135 record(queue, task);
4136 return Vec::new();
4137 }
4138 if let Some(selected) = &task.pr_recovery
4139 && let Err(e) = selected.configure(&mut config)
4140 {
4141 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4142 record(queue, task);
4143 return Vec::new();
4144 }
4145 (config, layers)
4146 };
4147 let config_evidence = crate::pr_recovery::ConfigEvidence {
4148 layers,
4149 overrides: task.overrides.clone(),
4150 daemon_merge: task
4151 .overrides
4152 .as_ref()
4153 .is_none_or(|o| o.inherit_daemon_options)
4154 .then(|| opts.merge.clone())
4155 .flatten(),
4156 };
4157 apply_task_implementers(&mut config, task);
4158 if let Some(replacement) = &task.replacement {
4159 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4160 old.liveness(false) == Liveness::Dead
4161 && old.merge_decision.as_ref().is_some_and(|d| {
4162 d.closed
4163 && d.question == replacement.question
4164 && d.replacement_run.as_ref().is_none_or(|run| {
4165 task.runs.last() == Some(run)
4166 && RunState::load(run).is_ok_and(|current| {
4167 current.replaces.as_ref().is_some_and(|pin| {
4168 pin.question == replacement.question
4169 && pin.agent == replacement.agent
4170 })
4171 })
4172 })
4173 })
4174 });
4175 if !safe {
4176 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4177 record(queue, task);
4178 return Vec::new();
4179 }
4180 match config.agent(&replacement.agent) {
4181 Ok(agent) if crate::agent::installed(agent) => {}
4182 _ => {
4183 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4184 record(queue, task);
4185 return Vec::new();
4186 }
4187 }
4188 config.graph.implementers = 1;
4189 config.graph.land_approval = true;
4190 config.merge.mode = MergeMode::Pr;
4191 config.graph.selected_implementer = Some(replacement.agent.clone());
4192 }
4193 let p = phrases(&config.graph.language);
4194 let start_failed = p.could_not_start;
4195 if let Ok(base) =
4200 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4201 .await
4202 && let Ok(head) =
4203 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4204 {
4205 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4206 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4207 let source = crate::git::rev_parse(&repo, branch)
4208 .await
4209 .unwrap_or_default();
4210 inputs.push_str(&format!(";review:{branch}:{source}"));
4211 }
4212 if let Some(reason) = task.recovery_exhausted(
4213 &head,
4214 &inputs,
4215 task.attempt_limit
4216 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4217 ) {
4218 task.hold_unrequested_retry(Some(reason));
4219 record(queue, task);
4220 return Vec::new();
4221 }
4222 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4223 }
4224
4225 if let Some(reason) = disk_gate(&repo, &config) {
4233 task.notice_cause("disk-space", None, None);
4234 task.last_error = Some(reason.clone());
4235 task.hold_machine(Some(reason.clone()));
4236 record(queue, task);
4237 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4238 notices::raise(
4241 Notice::warn(
4242 &format!("disk:{}", repo.display()),
4243 "A task was held for want of free disk space; free some, then release it from the queue.",
4244 )
4245 .link(Link::Task {
4246 id: task.id.clone(),
4247 }),
4248 );
4249 return Vec::new();
4250 }
4251
4252 let review_branch = take_review_branch(task, previous.as_ref());
4277 let branch_exists = match &review_branch {
4278 Some(branch) => crate::git::branch_exists(&repo, branch)
4279 .await
4280 .unwrap_or(false),
4281 None => false,
4282 };
4283 let proposed = review_retry_starter(
4287 task,
4288 previous.as_ref(),
4289 review_branch.as_deref(),
4290 branch_exists,
4291 unfinished.as_deref(),
4292 );
4293 let retry_selected = task.replacement.is_some()
4294 && task.resumption.is_some()
4295 && !task.fresh_start
4296 && previous.as_ref().is_some_and(|run| {
4297 run.replaces.is_some()
4298 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4299 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4300 });
4301 let stop_pin = task
4302 .resume_override
4303 .as_ref()
4304 .filter(|pin| {
4305 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4306 q.node == land::APPROVAL_NODE
4307 && q.resolution()
4308 .as_deref()
4309 .is_some_and(crate::merge_decision::stops)
4310 })
4311 })
4312 .and_then(|pin| pin.pinned_run.clone());
4313 let starter = if let Some(owner) = &public_owner {
4314 Starter::Resume(owner.id.clone())
4315 } else if let Some(run) = stop_pin {
4316 Starter::Resume(run)
4319 } else if retry_selected {
4320 Starter::Resume(previous.as_ref().unwrap().id.clone())
4321 } else if let Some(selected) = &task.pr_recovery {
4322 if selected.check_task(&task.id, &task.instruction).is_ok() {
4323 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4324 } else {
4325 Starter::Review(selected.branch.clone())
4326 }
4327 } else {
4328 starter_with_implementers(task, previous.as_ref(), proposed)
4329 };
4330 if let Some(pin) = &task.resume_override
4333 && let Some(run) = &pin.pinned_run
4334 && starter != Starter::Resume(run.clone())
4335 {
4336 let run = run.clone();
4337 task.notice_cause("resume-owner", Some(&run), None);
4338 task.hold_machine(Some(format!(
4339 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4340 task.id
4341 )));
4342 record(queue, task);
4343 return Vec::new();
4344 }
4345 if task.replacement.is_some()
4346 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4347 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4348 {
4349 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4350 record(queue, task);
4351 return Vec::new();
4352 }
4353 if let Some(reason) = retained_attempt_hold(
4354 task,
4355 &starter,
4356 config.disk.retained_attempt_limit,
4357 RunState::load,
4358 ) {
4359 if let Starter::Review(branch) = &starter {
4362 task.review_branch = Some(branch.clone());
4363 }
4364 task.hold_unrequested_retry(Some(reason));
4365 record(queue, task);
4366 return Vec::new();
4367 }
4368 let attachments = match task_attachments(queue, task) {
4373 Ok(a) => a,
4374 Err(e) => {
4375 task.attempts += 1;
4376 task.fail_startup_target(
4377 "startup-attachments",
4378 e.name.clone(),
4379 format!("{start_failed}{e:#}"),
4380 opts.max_attempts,
4381 );
4382 record(queue, task);
4383 return Vec::new();
4384 }
4385 };
4386 let startup_pause = stop.pause();
4387 let startup_interrupt = interrupt_pause.clone();
4388 let wait = crate::repo_lock::WaitPolicy::new(
4389 Duration::from_secs(config.graph.timeout_resource_wait),
4390 move || startup_pause.parked() || startup_interrupt.parked(),
4391 );
4392 let started = wait.scope(Box::pin(async { match &starter {
4395 Starter::Review(_) if task.pr_recovery.is_some() => {
4396 let selected = task.pr_recovery.clone().expect("selected recovery");
4397 match selected.check_task(&task.id, &task.instruction) {
4398 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4399 .await
4400 .map(|mut r| {
4401 r.state.attachments = attachments.clone();
4402 r
4403 }),
4404 Err(e) => Err(e),
4405 }
4406 }
4407 Starter::Review(branch) => {
4408 tracing::info!(
4409 "task {} reopens `{branch}` as a review-only pass",
4410 task.short()
4411 );
4412 let takeover = crate::handover::Takeover {
4415 earlier: task.earlier_attempts().to_vec(),
4416 home: crate::run::home(),
4417 choice: take_divergence_answer(branch, &config.merge.remote, task),
4418 task_instruction: Some(instruction_for(task)),
4419 };
4420 Runner::review_taking_over(
4421 &repo,
4422 branch,
4423 config,
4424 Some(takeover),
4425 crate::run::Origin::queue(&task.id),
4426 )
4427 .await
4428 }
4429 Starter::Resume(id) => {
4430 tracing::info!("resuming run {id} rather than competing again");
4431 Runner::resume(id).and_then(|mut r| {
4432 if retry_selected {
4433 for candidate in &mut r.state.candidates {
4436 candidate.failed = None;
4437 candidate.empty = false;
4438 candidate.commits = 0;
4439 }
4440 r.state.status = RunStatus::Implementing;
4441 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4442 }
4443 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4444 r.state.sources = task.sources.clone();
4445 }
4446 if r.state.prep_failure.is_some() {
4447 let approval = task
4448 .resumption
4449 .as_ref()
4450 .context("preparation retry requires an explicit task release")?;
4451 r.retry_preparation(approval.at)?;
4452 }
4453 if let Some(instruction) =
4454 prepare_instruction(&starter, Some(&r.state.instruction), task)
4455 {
4456 r.state.instruction = instruction;
4457 }
4458 r.state.attachments = attachments.clone();
4459 if task.replacement.is_some() {
4460 r.state.config.merge.mode = MergeMode::Pr;
4461 r.state.config.graph.land_approval = true;
4462 }
4463 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4467 .overrides
4468 .as_ref()
4469 .and_then(|o| o.merge.as_deref())
4470 .and_then(|m| merge_mode(m).ok())
4471 {
4472 r.state.config.merge.mode = mode;
4473 }
4474 Ok(r)
4475 })
4476 }
4477 Starter::Start => {
4478 if let Some(branch) = &review_branch {
4479 tracing::warn!(
4480 "conductor chose review for task {} but branch `{branch}` no longer \
4481 exists; requeuing as a fresh competition instead",
4482 task.short()
4483 );
4484 }
4485 let instruction = prepare_instruction(&starter, None, task)
4486 .unwrap_or_else(|| task.instruction.clone());
4487 Runner::start_naming(
4488 &repo,
4489 instruction,
4490 &task.title,
4491 config,
4492 crate::run::Origin::queue(&task.id),
4493 )
4494 .await
4495 .map(|mut r| {
4496 r.state.attachments = attachments.clone();
4497 r.state.sources = task.sources.clone();
4498 r
4499 })
4500 }
4501 }})).await;
4502 let mut runner = match started {
4503 Ok(r) => r,
4504 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4505 task.last_error = Some(e.to_string());
4506 if stop.parking() || interrupt_pause.parked() {
4507 let attempts = task.attempts;
4508 task.park(e.to_string());
4509 task.attempts = attempts;
4511 } else {
4512 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4513 }
4514 record(queue, task);
4515 return Vec::new();
4516 }
4517 Err(e) if task.pr_recovery.is_some() => {
4521 task.startup_notice_cause("startup-public-head");
4522 task.attempts += 1;
4523 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4524 task.link_run(&partial.run);
4525 task.record_pass(&partial.run, 0, task.attempts, None);
4526 }
4527 task.last_error = Some(format!("{start_failed}{e:#}"));
4528 task.hold_unrequested_retry(Some(format!(
4529 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4530 )));
4531 record(queue, task);
4532 return Vec::new();
4533 }
4534 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4535 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4536 Some(r) => (p.handover_refused)(r),
4537 None => format!("{e:#}"),
4538 };
4539 task.startup_notice_cause("startup-handover");
4540 let reason = format!("{start_failed}{detail}");
4541 task.last_error = Some(reason.clone());
4542 let branch = match &starter {
4545 Starter::Review(branch) => Some(branch.clone()),
4546 _ => None,
4547 };
4548 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4552 record(queue, task);
4553 tracing::warn!(
4554 "holding {} for a branch it cannot take over: {e:#}",
4555 task.short()
4556 );
4557 return Vec::new();
4558 }
4559 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4564 let d = e
4565 .downcast_ref::<crate::reconcile::Diverged>()
4566 .expect("checked by the guard");
4567 let mut q = ask::Question::new(
4568 task.id.clone(),
4569 crate::reconcile::NODE.to_owned(),
4570 crate::reconcile::SEAT.to_owned(),
4571 d.summary(),
4572 d.detail(),
4573 d.choices(),
4574 );
4575 match Questions::open().put(&mut q) {
4576 Ok(()) => {
4577 task.last_error = Some(format!("{e:#}"));
4578 task.review_branch = Some(d.branch.clone());
4581 task.review_instruction = Some(task.instruction.clone());
4582 task.block(vec![q.id.clone()], Some(d.summary()));
4583 }
4584 Err(put) => {
4585 tracing::warn!("could not file the divergence question: {put:#}");
4586 task.attempts += 1;
4587 task.fail_startup(
4588 "startup-question-store",
4589 format!("{start_failed}{e:#}"),
4590 opts.max_attempts,
4591 );
4592 }
4593 }
4594 record(queue, task);
4595 return Vec::new();
4596 }
4597 Err(e) => {
4598 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4604 && let Starter::Review(branch) = &starter
4605 {
4606 task.review_branch = Some(branch.clone());
4607 task.review_instruction = Some(task.instruction.clone());
4608 task.last_error = Some(format!("{start_failed}{e:#}"));
4609 task.status = crate::queue::TaskStatus::Failed;
4610 record(queue, task);
4611 return Vec::new();
4612 }
4613 task.attempts += 1;
4614 task.last_error = Some(format!("{start_failed}{e:#}"));
4615 task.startup_notice_cause("startup-source");
4616 task.hold_unrequested_retry(Some(format!(
4617 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4618 )));
4619 record(queue, task);
4620 return Vec::new();
4621 }
4622 };
4623 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4624 runner.state.recovery_config = Some(config_evidence);
4625 }
4626 if let Some(replacement) = &task.replacement {
4627 if matches!(starter, Starter::Start) {
4628 runner.state.instruction.push_str(&format!(
4629 "\n\n# Replacement review context\n{}",
4630 replacement.context
4631 ));
4632 }
4633 runner.state.replaces = Some(replacement.clone());
4634 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4635 if let Ok(mut old) = RunState::load(&replacement.run)
4636 && let Some(d) = &mut old.merge_decision
4637 {
4638 d.replacement_run = Some(runner.state.id.clone());
4639 old.event(
4640 "replacement",
4641 format!(
4642 "replacement run {} owns the next attempt; old PR remains closed",
4643 runner.state.id
4644 ),
4645 );
4646 if let Err(e) = old.save() {
4647 task.hold_manual(Some(format!(
4648 "Could not link replacement history: {e}; no agents dispatched"
4649 )));
4650 record(queue, task);
4651 return Vec::new();
4652 }
4653 }
4654 }
4655 runner.state.task_instruction = Some(task.instruction.clone());
4656 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4659 if runner.state.origin_chat.is_none() {
4660 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4661 }
4662 runner.on_pause(stop.pause());
4664 runner.watch_interrupt(interrupt_pause);
4668
4669 let run = runner.state.id.clone();
4672 task.start(run.clone());
4673 record(queue, task);
4674 lock(status).current.push(Current {
4675 worker: None,
4676 task: task.id.clone(),
4677 run,
4678 });
4679
4680 let quota_before = runner.state.quota.clone();
4683 runner.state.next_pass_task_entry = true;
4684 let result = runner.execute().await;
4685 finish_attempt(
4686 opts.max_attempts,
4687 queue,
4688 task,
4689 &runner.state,
4690 "a_before,
4691 result,
4692 )
4693}
4694
4695pub fn finish_attempt(
4700 max_attempts: usize,
4701 queue: &Queue,
4702 task: &mut Task,
4703 state: &RunState,
4704 quota_before: &[QuotaLoss],
4705 result: Result<()>,
4706) -> Vec<QuotaLoss> {
4707 let detail = match result {
4708 Ok(()) => describe(state),
4709 Err(e) => format!("{e:#}"),
4710 };
4711 let fresh = losses_this_attempt(quota_before, &state.quota);
4712 let verdict = Verdict {
4713 status: state.status,
4714 left_pr: state.pr.is_some(),
4717 quota_hit: !fresh.is_empty(),
4723 parked: state.parked,
4727 no_viable_candidates: state.viable().is_empty(),
4730 };
4731 if let Some(replacement) = &state.replaces
4732 && let Some(pr) = &state.pr
4733 && let Ok(mut old) = RunState::load(&replacement.run)
4734 && let Some(d) = &mut old.merge_decision
4735 {
4736 d.replacement_pr = Some(pr.url.clone());
4737 let _ = old.save();
4738 }
4739 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4740 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4741 task.hold_unrequested_retry(Some(format!(
4742 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4743 task.short()
4744 )));
4745 }
4746 if task.status == TaskStatus::Done {
4747 supersede_prior_runs(task, &crate::run::home());
4748 }
4749 record(queue, task);
4750 tracing::info!(
4751 "task {} is {} after run {} ({})",
4752 task.short(),
4753 task.status.as_str(),
4754 state.short(),
4755 label(state.status)
4756 );
4757 fresh
4758}
4759
4760pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4764 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4767 && !awaiting_resume_with(task, RunState::load)
4768 && withheld_text_wait(task).is_none()
4772 && !(task.fresh_start && task.replacement.is_some())
4773 {
4774 let why = task.last_error.clone().map_or_else(
4775 || "the run did not finish".to_owned(),
4776 |e| format!("the run did not finish: {e}"),
4777 );
4778 task.hold_unrequested_retry(Some(format!(
4779 "{why}. It was started by hand, so it is not retried \
4780 automatically; `magi task release` retries it."
4781 )));
4782 record(queue, task);
4783 }
4784}
4785
4786#[must_use]
4790pub fn loop_would_resume(run: &str) -> bool {
4791 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4792}
4793
4794#[must_use]
4804pub fn foreign_loop(
4805 reading: Option<&Reading>,
4806 now: Timestamp,
4807 own_pid: u32,
4808) -> Option<Option<u32>> {
4809 let reading = reading.filter(|r| r.running(now))?;
4810 match reading.pid {
4811 Some(pid) if pid == own_pid => None,
4812 pid => Some(pid),
4813 }
4814}
4815
4816fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4817 reading.worker_protocol != Some(1)
4818 && (foreign_loop(Some(reading), now, own_pid).is_some()
4819 || reading
4820 .pid
4821 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4822}
4823
4824pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4835 run_claimed_until(opts, queue, task, &Stop::new()).await;
4836}
4837
4838async fn run_claimed_until(
4839 opts: &Opts,
4840 queue: &Queue,
4841 task: &mut Task,
4842 stop: &Stop,
4843) -> Vec<QuotaLoss> {
4844 let status = Arc::new(Mutex::new(Status::new()));
4845 let mut quota = Box::pin(attempt(
4848 opts,
4849 queue,
4850 &status,
4851 stop,
4852 crate::graph::Pause::new(),
4853 task,
4854 ))
4855 .await;
4856 let mut resumes = 0;
4865 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
4866 let Some(timeout) = withheld_text_wait(task) else {
4867 break;
4868 };
4869 eprintln!(
4870 "waiting for the owner's answer on a withheld pull request text \
4871 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4872 );
4873 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4876 let ready = loop {
4877 match land_resume_state(task) {
4878 LandResume::StillWaiting
4879 if !stop.parking() && std::time::Instant::now() < limit =>
4880 {
4881 tokio::time::sleep(WITHHELD_POLL).await;
4882 }
4883 LandResume::Ready => break true,
4884 _ => break false,
4885 }
4886 };
4887 if !ready || stop.parking() {
4888 break;
4889 }
4890 resumes += 1;
4891 quota.extend(
4892 Box::pin(attempt(
4893 opts,
4894 queue,
4895 &status,
4896 stop,
4897 crate::graph::Pause::new(),
4898 task,
4899 ))
4900 .await,
4901 );
4902 }
4903 hold_if_runnable(queue, task);
4904 quota
4905}
4906
4907const MAX_WITHHELD_RESUMES: u32 = 3;
4910
4911const WITHHELD_POLL: Duration = Duration::from_secs(15);
4913
4914fn withheld_text_wait(task: &Task) -> Option<u64> {
4917 let id = task.runs.last()?;
4918 let s = RunState::load(id).ok()?;
4919 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4920 .then_some(s.config.graph.answer_timeout)
4921}
4922
4923fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4933 after
4934 .iter()
4935 .filter(|q| !before.contains(q))
4936 .cloned()
4937 .collect()
4938}
4939
4940fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4943 if quota.is_empty() {
4944 return None;
4945 }
4946 let with_hint = quota.iter().find(|q| q.reset.is_some());
4947 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4948 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4949 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4950 Some(
4951 now.checked_add(jiff::SignedDuration::from_secs(secs))
4952 .unwrap_or(Timestamp::MAX),
4953 )
4954}
4955
4956fn implementers_match(task: &Task, state: &RunState) -> bool {
4958 task.replacement.as_ref().map_or_else(
4959 || {
4960 task.implementers()
4961 .is_none_or(|n| n == state.config.graph.implementers)
4962 },
4963 |replacement| {
4964 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4965 && state.config.graph.implementers == 1
4966 },
4967 )
4968}
4969
4970fn starter_with_implementers(
4972 task: &Task,
4973 previous: Option<&RunState>,
4974 proposed: Starter,
4975) -> Starter {
4976 if previous.is_some_and(|s| !implementers_match(task, s)) {
4977 task.review_of
4978 .as_ref()
4979 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4980 } else {
4981 proposed
4982 }
4983}
4984
4985fn apply_task_implementers(config: &mut Config, task: &Task) {
4987 if let Some(n) = task.implementers() {
4988 config.graph.implementers = n;
4989 }
4990}
4991
4992#[cfg(test)]
4997fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
4998 let explicit = match &task.overrides {
4999 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5000 _ => opts.config.as_deref(),
5001 };
5002 let (mut config, _) = Config::discover(repo, explicit)?;
5003 apply_options(&mut config, opts, task)?;
5004 Ok(config)
5005}
5006
5007fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5008 if task
5009 .overrides
5010 .as_ref()
5011 .is_none_or(|o| o.inherit_daemon_options)
5012 && let Some(mode) = opts.merge.as_deref()
5013 {
5014 config.merge.mode = merge_mode(mode)?;
5015 }
5016 if let Some(o) = &task.overrides {
5017 o.apply(config);
5018 }
5019 Ok(())
5020}
5021
5022fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5024 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5025 if let Some(mode) = &opts.merge {
5026 config.merge.mode = merge_mode(mode)?;
5027 }
5028 Ok(config)
5029}
5030
5031async fn maybe_prune_cache_between_runs(
5036 repo: &Path,
5037 opts: &Opts,
5038 home: &Path,
5039 stop: &Stop,
5040 last_checked: &mut Option<Timestamp>,
5041 now: Timestamp,
5042) {
5043 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5044 return;
5045 }
5046 *last_checked = Some(now);
5047 let cfg = match prepare(repo, opts) {
5048 Ok(cfg) => cfg,
5049 Err(e) => {
5050 tracing::warn!("cache check: no config: {e:#}");
5051 return;
5052 }
5053 };
5054 match clean::prune_cache_if_over_limit(&cfg, home) {
5055 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5056 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5057 pruned.files,
5058 pruned.freed
5059 ),
5060 Ok(_) => {}
5061 Err(e) => {
5062 tracing::warn!("housekeep: prune cache: {e:#}");
5063 notices::raise_in(
5064 home,
5065 Notice::warn(
5066 "housekeep:cache",
5067 "Pruning the shared build cache failed; disk usage may keep growing.",
5068 ),
5069 );
5070 }
5071 }
5072}
5073
5074fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5078 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5079}
5080
5081async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5100 let cfg = match prepare(repo, opts) {
5101 Ok(cfg) => cfg,
5102 Err(e) => {
5103 tracing::warn!("housekeep: no config: {e:#}");
5104 return;
5105 }
5106 };
5107 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5116 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5117 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5122 let mut extra = Vec::new();
5123 if out.unreadable > 0 {
5124 extra.push(format!("{} unreadable", out.unreadable));
5125 }
5126 if out.orphaned_worktrees > 0 {
5127 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5128 }
5129 let detail = if extra.is_empty() {
5130 String::new()
5131 } else {
5132 format!(" ({})", extra.join(", "))
5133 };
5134 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5135 }
5136 if out.external_merges_recorded > 0 {
5137 tracing::info!(
5138 "housekeep: recorded {} run(s) as merged externally",
5139 out.external_merges_recorded
5140 );
5141 }
5142 if out.stale_pr_states_repaired > 0 {
5143 tracing::info!(
5144 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5145 out.stale_pr_states_repaired
5146 );
5147 }
5148 if out.cache_files > 0 {
5149 tracing::info!(
5150 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5151 out.cache_files,
5152 out.cache_freed
5153 );
5154 }
5155 if out.questions_abandoned > 0 {
5156 tracing::info!(
5157 "housekeep: abandoned {} question(s) left open by a finished run",
5158 out.questions_abandoned
5159 );
5160 }
5161}
5162
5163async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5172 let questions = Questions::at(home.join("questions"));
5173 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5174 if report.is_empty() {
5175 return;
5176 }
5177 if !report.quarantined.is_empty() {
5178 tracing::info!(
5179 "triage: held {} blocked task(s) whose blocked-on task or \
5180 question no longer exists: {}",
5181 report.quarantined.len(),
5182 report.quarantined.join(", ")
5183 );
5184 }
5185 if !report.resumed.is_empty() {
5186 tracing::info!(
5187 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5188 report.resumed.len(),
5189 report.resumed.join(", ")
5190 );
5191 }
5192 if !report.asked.is_empty() {
5193 tracing::info!(
5194 "triage: asked about {} held task(s): {}",
5195 report.asked.len(),
5196 report.asked.join(", ")
5197 );
5198 }
5199 if !report.answered.is_empty() {
5200 tracing::info!(
5201 "triage: applied {} operator answer(s): {}",
5202 report.answered.len(),
5203 report.answered.join(", ")
5204 );
5205 }
5206}
5207
5208pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5215 disk_gate_with(repo, config, crate::disk::free_bytes)
5216}
5217
5218fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5222 repo: &Path,
5223 config: &Config,
5224 free_bytes: F,
5225) -> Option<String> {
5226 let min = config.disk.min_free_bytes;
5227 if min == 0 {
5228 return None;
5229 }
5230 let mut paths = vec![repo.to_owned()];
5231 if let Some(cache) = config.cache_dir() {
5232 paths.push(cache);
5233 }
5234 if let Some(root) = &config.graph.worktree_root {
5235 paths.push(root.clone());
5236 }
5237 for path in paths {
5238 let probe = if path == repo {
5239 path.as_path()
5240 } else {
5241 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5242 };
5243 let reason = match free_bytes(probe) {
5244 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5245 Err(e) => Some(crate::disk::unmeasured_in(
5246 probe,
5247 &e,
5248 &config.graph.language,
5249 )),
5250 };
5251 if reason.is_some() {
5252 return reason;
5253 }
5254 }
5255 None
5256}
5257
5258const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5265
5266const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5270
5271fn quota_wait(
5280 reset_at: Option<Timestamp>,
5281 now: Timestamp,
5282 fallback: Duration,
5283 cap: Duration,
5284) -> Duration {
5285 match reset_at {
5286 Some(at) if at > now => {
5287 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5288 Duration::from_secs(secs).min(cap)
5289 }
5290 _ => fallback,
5291 }
5292}
5293
5294fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5308 parse_reset_hint_zoned(text, now)
5309 .or_else(|| parse_reset_hint_dated(text))
5310 .or_else(|| parse_reset_hint_relative(text, recorded))
5311}
5312
5313fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5317 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5318 let mut rest = rest.trim();
5319 if rest.is_empty() {
5320 return None;
5321 }
5322 let mut total: i64 = 0;
5323 let mut matched = false;
5324 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5325 if let Some((digits, tail)) = rest.split_once(unit)
5326 && !digits.is_empty()
5327 && digits.bytes().all(|b| b.is_ascii_digit())
5328 {
5329 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5330 rest = tail;
5331 matched = true;
5332 }
5333 }
5334 if !rest.is_empty() || !matched {
5335 return None;
5336 }
5337 recorded
5338 .checked_add(jiff::SignedDuration::from_secs(total))
5339 .ok()
5340}
5341
5342fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5346 let clock = clock.trim().to_lowercase();
5347 let (digits, pm) = clock
5348 .strip_suffix("am")
5349 .map(|d| (d, false))
5350 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5351 let (h, m) = digits.trim().split_once(':')?;
5352 let mut hour: i8 = h.trim().parse().ok()?;
5353 let minute: i8 = m.trim().parse().ok()?;
5354 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5355 return None;
5356 }
5357 if pm && hour != 12 {
5358 hour += 12;
5359 } else if !pm && hour == 12 {
5360 hour = 0;
5361 }
5362 Some((hour, minute))
5363}
5364
5365fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5370 let open = text.find('(')?;
5371 let close = text.rfind(')')?;
5372 if close <= open {
5373 return None;
5374 }
5375 let zone = text[open + 1..close].trim();
5376 let (hour, minute) = parse_12h_clock(&text[..open])?;
5377 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5378 let candidate = now
5379 .to_zoned(tz)
5380 .with()
5381 .hour(hour)
5382 .minute(minute)
5383 .second(0)
5384 .millisecond(0)
5385 .microsecond(0)
5386 .nanosecond(0)
5387 .build()
5388 .ok()?;
5389 let mut at = candidate.timestamp();
5390 if at <= now {
5391 at += jiff::SignedDuration::from_hours(24);
5392 }
5393 Some(at)
5394}
5395
5396fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5405 let words: Vec<&str> = text.split_whitespace().collect();
5406 if words.len() < 5 {
5407 return None;
5408 }
5409 (0..=words.len() - 5)
5410 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5411}
5412
5413fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5419 if trailing.is_some_and(|next| next.starts_with('(')) {
5420 return None;
5421 }
5422 let month = month_number(window[0])?;
5423 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5424 let day_digits = ["st", "nd", "rd", "th"]
5425 .iter()
5426 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5427 let day: i8 = day_digits.parse().ok()?;
5428 let year_token = window[2];
5429 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5430 return None;
5431 }
5432 let year: i16 = year_token.parse().ok()?;
5433 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5437 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5438 let date = jiff::civil::Date::new(year, month, day).ok()?;
5439 let candidate = date
5440 .at(hour, minute, 0, 0)
5441 .to_zoned(jiff::tz::TimeZone::UTC)
5442 .ok()?;
5443 Some(candidate.timestamp())
5444}
5445
5446fn month_number(name: &str) -> Option<i8> {
5449 const NAMES: [&str; 12] = [
5450 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5451 ];
5452 let lower = name.to_lowercase();
5453 NAMES
5454 .iter()
5455 .position(|n| *n == lower.as_str())
5456 .map(|i| i as i8 + 1)
5457}
5458
5459#[cfg(test)]
5461fn exhausted_review_budget(state: &RunState) -> bool {
5462 !crate::handover::can_resume(state)
5465 && state.status == RunStatus::Blocked
5466 && !state.released()
5467 && state.retired_to.is_none()
5468}
5469
5470fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5507 unfinished_run_with(runs, short, RunState::load)
5508}
5509
5510pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5513where
5514 F: FnMut(&str) -> Result<RunState>,
5515{
5516 let mut id = runs.last()?.clone();
5517 let mut visited = std::collections::BTreeSet::new();
5518 let mut repo = None;
5519 loop {
5520 if !runs.contains(&id) || !visited.insert(id.clone()) {
5521 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5522 return None;
5523 }
5524 let state = match load(&id) {
5525 Ok(state) if state.id == id => state,
5526 Ok(_) => return None,
5527 Err(e) => {
5528 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5529 return None;
5530 }
5531 };
5532 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5533 return None;
5534 }
5535 repo = Some(state.repo.clone());
5536 if let Some(authority) = state.retired_to {
5537 id = authority;
5538 continue;
5539 }
5540 return (crate::handover::can_resume(&state)
5541 && state.liveness(false) != crate::run::Liveness::Live)
5542 .then_some(id);
5543 }
5544}
5545
5546fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5554where
5555 F: FnOnce(&str) -> Result<RunState>,
5556{
5557 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5559 || task.fresh_start
5560 || task.review_branch.is_some()
5561 {
5562 return false;
5563 }
5564 let Some(id) = task.runs.last() else {
5565 return false;
5566 };
5567 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5568}
5569
5570#[derive(Debug, Clone, PartialEq, Eq)]
5573enum Starter {
5574 Review(String),
5577 Resume(String),
5579 Start,
5581}
5582
5583fn retained_attempt_hold(
5587 task: &Task,
5588 starter: &Starter,
5589 limit: usize,
5590 mut load: impl FnMut(&str) -> Result<RunState>,
5591) -> Option<String> {
5592 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5593 return None;
5594 }
5595 let mut retained = 0;
5596 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5597 let state = match load(id) {
5598 Ok(state) if state.id == *id => state,
5599 _ => {
5600 return Some(format!(
5601 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5602 ));
5603 }
5604 };
5605 if state.retired_to.is_some() || state.released() {
5606 continue;
5607 }
5608 retained += 1;
5611 }
5612 (retained >= limit).then(|| format!(
5613 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5614 ))
5615}
5616
5617fn selected_starter(
5619 selected: &crate::pr_recovery::Selection,
5620 previous: Option<&RunState>,
5621 unfinished: Option<&str>,
5622) -> Starter {
5623 if let Some(id) = unfinished
5624 && previous.is_some_and(|r| {
5625 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5626 })
5627 {
5628 Starter::Resume(id.to_owned())
5629 } else {
5630 Starter::Review(selected.branch.clone())
5631 }
5632}
5633
5634fn take_divergence_answer(
5638 branch: &str,
5639 remote: &str,
5640 task: &mut Task,
5641) -> Option<crate::reconcile::Choice> {
5642 let summary = crate::reconcile::summary_for(branch, remote);
5643 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5644 (a.question == summary)
5645 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5646 .flatten()
5647 .map(|c| (i, c))
5648 })?;
5649 task.answers.remove(idx);
5650 Some(choice)
5651}
5652
5653fn choose_starter(
5665 review_branch: Option<&str>,
5666 branch_exists: bool,
5667 unfinished: Option<&str>,
5668) -> Starter {
5669 match review_branch {
5670 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5671 Some(_) => Starter::Start,
5672 None => match unfinished {
5673 Some(id) => Starter::Resume(id.to_owned()),
5674 None => Starter::Start,
5675 },
5676 }
5677}
5678
5679fn review_retry_starter(
5683 task: &Task,
5684 previous: Option<&RunState>,
5685 branch: Option<&str>,
5686 exists: bool,
5687 unfinished: Option<&str>,
5688) -> Starter {
5689 if let Some(run) = previous
5690 && unfinished == Some(run.id.as_str())
5691 && run.reviewed_commits.is_some()
5692 && branch.is_some_and(|branch| {
5693 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
5694 })
5695 && may_resume_run(run, task)
5696 {
5697 return Starter::Resume(run.id.clone());
5698 }
5699 match (branch, unfinished, &task.review_of) {
5700 (None, None, Some(branch)) => Starter::Review(branch.clone()),
5701 _ => choose_starter(branch, exists, unfinished),
5702 }
5703}
5704
5705fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5707 let branch = task.review_branch.take();
5708 let instruction = task.review_instruction.take();
5709 branch.filter(|branch| match &instruction {
5710 Some(authored) => *authored == task.instruction,
5711 None => previous.map_or_else(
5714 || task.review_of.as_ref() == Some(branch),
5715 |run| compatible_run_instruction(run, task),
5716 ),
5717 })
5718}
5719
5720fn may_resume_run(run: &RunState, task: &Task) -> bool {
5723 task.resume_override
5724 .as_ref()
5725 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5726 || compatible_run_instruction(run, task)
5727}
5728
5729pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5731 if let Some(authored) = &run.task_instruction {
5732 return compatible_instruction(authored, task);
5733 }
5734 if compatible_instruction(&run.instruction, task) {
5735 return true;
5736 }
5737 if run.reviewed_commits.is_some() {
5740 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5741 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5742 }
5743 if let Some(branch) = &task.review_of {
5744 let canonical = format!(
5745 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5746 );
5747 let generated = format!(
5748 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5749 );
5750 return task.instruction == canonical && run.instruction.starts_with(&generated);
5751 }
5752 }
5753 false
5754}
5755
5756pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5758 strip_answers_block(old, task) == task.instruction
5759}
5760
5761fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5764 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5765 return fallback.to_path_buf();
5766 }
5767 task.repo.clone()
5768}
5769
5770const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5774
5775fn answers_block(task: &Task, count: usize) -> String {
5777 let mut s = ANSWERS_HEADER.to_owned();
5778 for a in &task.answers[..count] {
5779 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5780 }
5781 s
5782}
5783
5784fn resumption_block(r: &crate::queue::Resumption) -> String {
5787 format!(
5788 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5789 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5790 r.generation, r.at, r.instruction, r.answers_through,
5791 )
5792}
5793
5794fn append_answers(base: &str, task: &Task) -> String {
5796 let mut s = base.to_owned();
5797 if !task.answers.is_empty() {
5798 s.push_str(&answers_block(task, task.answers.len()));
5799 }
5800 if let Some(r) = &task.resumption {
5801 s.push_str(&resumption_block(r));
5802 }
5803 s
5804}
5805
5806fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5810 let mut instruction = instruction;
5811 for approval in task
5812 .resumption
5813 .iter()
5814 .chain(task.resumption_history.iter().rev())
5815 {
5816 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5817 instruction = base;
5818 break;
5819 }
5820 }
5821 for count in (1..=task.answers.len()).rev() {
5822 let block = answers_block(task, count);
5823 if let Some(base) = instruction.strip_suffix(&block) {
5824 return base;
5825 }
5826 }
5827 instruction
5828}
5829
5830fn instruction_for(task: &Task) -> String {
5838 append_answers(&task.instruction, task)
5839}
5840
5841fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5853 append_answers(strip_answers_block(old_instruction, task), task)
5854}
5855
5856fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5858 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5859 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5860 if let Some(cause) = task.hold_notice_cause.as_mut() {
5861 cause.target = failure.map(|f| f.target.clone());
5862 }
5863}
5864
5865#[derive(Debug, thiserror::Error)]
5866#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5867struct MissingAttachment {
5868 name: String,
5869 path: PathBuf,
5870}
5871
5872fn task_attachments(
5874 queue: &Queue,
5875 task: &Task,
5876) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5877 let paths = queue.attachment_paths(task);
5878 for (name, path) in task.attachments.iter().zip(&paths) {
5879 if !path.is_file() {
5880 return Err(MissingAttachment {
5881 name: name.clone(),
5882 path: path.clone(),
5883 });
5884 }
5885 }
5886 Ok(paths)
5887}
5888
5889fn prepare_instruction(
5900 starter: &Starter,
5901 old_instruction: Option<&str>,
5902 task: &Task,
5903) -> Option<String> {
5904 match starter {
5905 Starter::Start => Some(instruction_for(task)),
5906 Starter::Resume(_) => Some(resumed_instruction(
5907 old_instruction.expect("a resumed run always has a prior instruction"),
5908 task,
5909 )),
5910 Starter::Review(_) => None,
5911 }
5912}
5913
5914fn record(queue: &Queue, task: &mut Task) {
5918 if let Err(e) = queue.put(task) {
5919 tracing::error!("could not record task {}: {e:#}", task.short());
5920 let notice = Notice::error(
5921 "loop:record",
5922 "The loop could not save a task's state; check the disk.",
5923 )
5924 .link(Link::Task {
5925 id: task.id.clone(),
5926 });
5927 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5929 Some(home) => notices::raise_in(home, notice),
5930 None => notices::raise(notice),
5931 }
5932 }
5933}
5934
5935fn runnable(queue: &Queue) -> Vec<Task> {
5941 let mut tasks: Vec<Task> = queue
5942 .list()
5943 .into_iter()
5944 .filter(|t| t.status.runnable())
5945 .collect();
5946 tasks.sort_unstable_by(crate::queue::dispatch_order);
5947 tasks
5948}
5949
5950fn describe(state: &RunState) -> String {
5964 let p = phrases(&state.config.graph.language);
5965 let mut detail = if state.status == RunStatus::Stalled {
5966 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5967 seats.sort_unstable();
5968 seats.dedup();
5969 if seats.is_empty() {
5970 p.quorum_lost.to_owned()
5971 } else {
5972 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5973 }
5974 } else {
5975 format!("{}{}", p.run_ended, state.status.display_label())
5976 };
5977 if let Some(last) = state.events.last() {
5978 let unanswered = state
5982 .reviews
5983 .last()
5984 .filter(|r| {
5985 state.status == RunStatus::Blocked
5986 && last.node == "review"
5987 && r.incomplete()
5988 && r.blocking == 0
5989 && r.round == state.config.graph.review_rounds
5990 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
5991 })
5992 .map(|r| (r.expected - r.answered, r.round));
5993 match unanswered {
5994 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
5995 detail.push_str(&format!(
5996 " ({}: {})",
5997 last.node,
5998 (p.reviewers_never_answered)(missing, rounds)
5999 ));
6000 }
6001 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6002 }
6003 }
6004 detail.push_str(&format!(" [run {}]", state.id));
6005 detail
6006}
6007
6008const DIAGNOSTIC_MAX: usize = 4_000;
6014
6015const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6020
6021fn diagnostic(state: &RunState) -> Option<String> {
6035 let mut parts: Vec<String> = Vec::new();
6036
6037 for o in state.gate.iter().filter(|o| !o.ok()) {
6039 parts.push(format!(
6040 "gate `{}` failed ({}):\n{}",
6041 o.command,
6042 notices::exit_status(o.code),
6043 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6044 ));
6045 }
6046
6047 if let Some(last) = state
6050 .events
6051 .iter()
6052 .rev()
6053 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6054 {
6055 parts.push(last.message.clone());
6056 }
6057
6058 if state.viable().is_empty() {
6065 for c in &state.candidates {
6066 if let Some(evidence) = &c.verified_noop {
6067 parts.push(format!(
6068 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6069 c.label
6070 ));
6071 } else if !c.summary.trim().is_empty() {
6072 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6073 } else if let Some(why) = &c.failed {
6074 parts.push(format!("candidate {}: {why}", c.label));
6075 }
6076 }
6077 }
6078
6079 if parts.is_empty() {
6080 return None;
6081 }
6082 Some(crate::run::tail(
6087 &parts.join("\n\n"),
6088 DIAGNOSTIC_MAX.saturating_sub(100),
6089 ))
6090}
6091
6092fn label(status: RunStatus) -> &'static str {
6100 status.as_str()
6101}
6102
6103pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6105 match mode {
6106 "none" => Ok(MergeMode::None),
6107 "local" => Ok(MergeMode::Local),
6108 "pr" => Ok(MergeMode::Pr),
6109 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6110 }
6111}
6112
6113fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6118 mutex
6119 .lock()
6120 .unwrap_or_else(std::sync::PoisonError::into_inner)
6121}
6122
6123#[cfg(test)]
6124mod tests {
6125 use super::*;
6126 use crate::queue::{Source, TaskStatus};
6127 use crate::run::{Candidate, CommandOutcome};
6128 use pretty_assertions::assert_eq;
6129
6130 fn task() -> Task {
6131 Task::new(
6132 "add retries".to_owned(),
6133 "add retries".to_owned(),
6134 PathBuf::from("/repo"),
6135 Source::Human,
6136 )
6137 }
6138
6139 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6140 let job = crate::worker::Job {
6141 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6142 sequence: 1,
6143 kind: "task".into(),
6144 key: format!("task:{}", request.task),
6145 payload: serde_json::to_value(request).unwrap(),
6146 generation: "a".repeat(64),
6147 version: None,
6148 status: crate::worker::Status::Starting,
6149 result: None,
6150 pid: None,
6151 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6152 finished_at: None,
6153 };
6154 let dir = home.join("workers/jobs").join(&job.id);
6155 std::fs::create_dir_all(&dir).unwrap();
6156 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6157 job
6158 }
6159
6160 #[test]
6161 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6162 let home = tempfile::tempdir().unwrap();
6163 let request = TaskWorker {
6164 opts: Opts::default(),
6165 queue: home.path().join("queue"),
6166 task: "startup".into(),
6167 lane: PermitKind::Ordinary,
6168 standalone: false,
6169 };
6170 let job = starting_task_reservation(home.path(), &request);
6171 task_worker_spawn_error(
6172 home.path(),
6173 &request,
6174 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6175 )
6176 .expect_err("our own reservation must not turn a failed spawn into a resync");
6177 let active = crate::worker::active(home.path()).unwrap();
6178 assert_eq!(active.len(), 1);
6179 assert_eq!(active[0].id, job.id);
6180 assert_eq!(active[0].status, crate::worker::Status::Starting);
6181 assert!(
6182 crate::worker::control(home.path(), &job.key)
6183 .unwrap()
6184 .is_none()
6185 );
6186 let notices = notices::Notices::at(home.path().join("notifications")).list();
6187 assert_eq!(notices.len(), 1);
6188 assert_eq!(notices[0].key, "loop:worker-startup");
6189 assert!(!notices[0].message.contains("private-error-detail"));
6190 }
6191
6192 #[test]
6193 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6194 let home = tempfile::tempdir().unwrap();
6195 let mut request = TaskWorker {
6196 opts: Opts::default(),
6197 queue: home.path().join("queue"),
6198 task: "handoff".into(),
6199 lane: PermitKind::Ordinary,
6200 standalone: true,
6201 };
6202 let job = starting_task_reservation(home.path(), &request);
6203 request.standalone = false;
6204 task_worker_spawn_error(
6205 home.path(),
6206 &request,
6207 anyhow::anyhow!("operation key is already reserved for a different request"),
6208 )
6209 .expect("a different standalone request is the handoff race, not our startup failure");
6210 let active = crate::worker::active(home.path()).unwrap();
6211 assert_eq!(active.len(), 1);
6212 assert_eq!(active[0].id, job.id);
6213 assert_eq!(active[0].payload, job.payload);
6214 assert!(
6215 notices::Notices::at(home.path().join("notifications"))
6216 .list()
6217 .is_empty()
6218 );
6219 }
6220
6221 #[test]
6222 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6223 let home = tempfile::tempdir().unwrap();
6224 let queue = Queue::at(home.path().join("queue"));
6225 let mut task = task();
6226 task.runs = vec!["previous".into()];
6227 queue.put(&mut task).unwrap();
6228 let request = TaskWorker {
6229 opts: Opts::default(),
6230 queue: home.path().join("queue"),
6231 task: task.id.clone(),
6232 lane: PermitKind::Ordinary,
6233 standalone: false,
6234 };
6235 let job = starting_task_reservation(home.path(), &request);
6236 let claims = current_work(home.path(), Timestamp::now());
6237 assert_eq!(claims.len(), 1);
6238 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6239 assert_eq!(claims[0].run, "previous");
6240 assert!(other_claims_run(&claims, "previous", None));
6241 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6244 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6245 }
6246
6247 #[test]
6248 fn admission_excludes_only_the_callers_exact_reservation() {
6249 let own = Current {
6250 worker: Some("own".into()),
6251 task: "same-task".into(),
6252 run: "previous".into(),
6253 };
6254 assert!(!other_claims_run(
6255 std::slice::from_ref(&own),
6256 "previous",
6257 Some("own")
6258 ));
6259 assert!(other_claims_run(
6260 std::slice::from_ref(&own),
6261 "previous",
6262 None
6263 ));
6264 for worker in [None, Some("other".into())] {
6265 let other = Current {
6266 worker,
6267 ..own.clone()
6268 };
6269 assert!(other_claims_run(
6270 &[own.clone(), other],
6271 "previous",
6272 Some("own")
6273 ));
6274 }
6275 let legacy: Current =
6276 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6277 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6278 }
6279
6280 #[test]
6281 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6282 let mut task = task();
6283 let mut run = run_state(RunStatus::Reviewing);
6284 run.task_instruction = Some(task.instruction.clone());
6285 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6286 let branch = "magi/test/B";
6287 run.candidates.push(crate::run::Candidate {
6288 branch: branch.into(),
6289 ..candidate('B', "preserved work", false, None)
6290 });
6291 task.runs = vec![run.id.clone()];
6292 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6293 assert_eq!(
6294 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6295 Starter::Resume(run.id.clone())
6296 );
6297 assert!(matches!(
6298 review_retry_starter(
6299 &task,
6300 Some(&run),
6301 Some("different"),
6302 true,
6303 unfinished.as_deref()
6304 ),
6305 Starter::Review(_)
6306 ));
6307 assert_eq!(
6308 review_retry_starter(
6309 &task,
6310 Some(&run),
6311 Some(branch),
6312 false,
6313 unfinished.as_deref()
6314 ),
6315 Starter::Start
6316 );
6317 task.instruction.push_str(" changed criteria");
6318 assert!(matches!(
6319 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6320 Starter::Review(_)
6321 ));
6322 assert!(matches!(
6323 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6324 Starter::Review(_)
6325 ));
6326 }
6327
6328 #[test]
6329 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6330 let dir = tempfile::tempdir().unwrap();
6331 let queue = Queue::at(dir.path().join("queue"));
6332 let mut task = task();
6333 task.status = TaskStatus::Running;
6334 task.attempts = 1;
6335 queue.put(&mut task).unwrap();
6336 let live = std::collections::BTreeSet::from([task.id.clone()]);
6337 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6338 drop(queue.claim(&task.id).unwrap());
6339 let saved = queue.get(&task.id).unwrap();
6340 assert_eq!(saved.status, TaskStatus::Running);
6341 assert_eq!(saved.attempts, 1);
6342 assert!(saved.last_error.is_none());
6343 }
6344
6345 #[test]
6346 fn standalone_workers_do_not_consume_scheduler_lanes() {
6347 let request = |task: &str, lane, standalone| TaskWorker {
6348 opts: Opts::default(),
6349 queue: PathBuf::from("/queue"),
6350 task: task.to_owned(),
6351 lane,
6352 standalone,
6353 };
6354 let mut jobs = vec![
6355 request("ordinary", PermitKind::Ordinary, false),
6356 request("standalone-urgent", PermitKind::Urgent, true),
6357 request("standalone-ordinary", PermitKind::Ordinary, true),
6358 request("decision", PermitKind::None, false),
6359 ];
6360 assert_eq!(available_slots(&jobs, 2), (1, 1));
6361 jobs.push(request("urgent", PermitKind::Urgent, false));
6362 assert_eq!(available_slots(&jobs, 2), (1, 0));
6363 assert_eq!(available_slots(&jobs, 0), (0, 0));
6364 }
6365
6366 #[test]
6367 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6368 let now = Timestamp::now();
6369 let mut reading = Reading {
6370 pid: Some(std::process::id()),
6371 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6372 ..Reading::default()
6373 };
6374 assert!(!reading.running(now));
6375 assert!(legacy_scheduler_running(&reading, now, 0));
6376 reading.worker_protocol = Some(1);
6377 assert!(!legacy_scheduler_running(&reading, now, 0));
6378 }
6379
6380 #[test]
6381 fn selected_public_head_overrides_historical_resume_and_pin() {
6382 let mut t = task();
6383 let mut old = run_state(RunStatus::Blocked);
6384 let selection = crate::pr_recovery::Selection {
6385 url: "url".into(),
6386 branch: format!("magi/task-{}", t.id),
6387 head: "a".repeat(40),
6388 source_head: None,
6389 source_run: None,
6390 source_proof: None,
6391 disposition: Some(crate::pr_recovery::Disposition::Review),
6392 instruction: t.instruction.clone(),
6393 approved_at: Timestamp::now(),
6394 };
6395 t.resume_override = Some(crate::queue::OperatorResume {
6396 pinned_run: Some(old.id.clone()),
6397 question_id: "choice".into(),
6398 at: Timestamp::now(),
6399 conductor_rehold: None,
6400 forced: false,
6401 });
6402 assert_eq!(
6403 selected_starter(&selection, Some(&old), Some(&old.id)),
6404 Starter::Review(selection.branch.clone())
6405 );
6406 old.pr_recovery = Some(selection.clone());
6407 assert_eq!(
6408 selected_starter(&selection, Some(&old), Some(&old.id)),
6409 Starter::Review(selection.branch.clone())
6410 );
6411 old.reviewed_commits = Some(vec!["implementation".into()]);
6412 assert_eq!(
6413 selected_starter(&selection, Some(&old), Some(&old.id)),
6414 Starter::Resume(old.id.clone())
6415 );
6416 let mut changed = selection.clone();
6417 changed.head = "b".repeat(40);
6418 assert_eq!(
6419 selected_starter(&changed, Some(&old), Some(&old.id)),
6420 Starter::Review(changed.branch)
6421 );
6422 }
6423
6424 #[test]
6425 fn candidate_transplant_failure_is_one_actionable_hold() {
6426 let dir = tempfile::tempdir().unwrap();
6427 let q = Queue::at(dir.path().join("queue"));
6428 let mut t = task();
6429 let mut state = run_state(RunStatus::Blocked);
6430 state.event("prep", "cannot apply referenced commit: conflict");
6431 state.prep_failure = Some("conflict".into());
6432 t.start(state.id.clone());
6433 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6434 assert_eq!(t.attempts, 1);
6435 assert_eq!(t.status, TaskStatus::Held);
6436 assert!(t.operator_held());
6437 assert!(!t.status.runnable());
6438 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6439 let first = t.chat_notice_key().unwrap();
6440 t.chat_notices.push(first.clone());
6441 t.release_operator("test", None);
6442 state.id = "another-unprepared-run".into();
6443 t.start(state.id.clone());
6444 finish_attempt(
6445 3,
6446 &q,
6447 &mut t,
6448 &state,
6449 &[],
6450 Err(anyhow::anyhow!("still conflicts")),
6451 );
6452 assert_eq!(
6453 t.chat_notice_key().as_ref(),
6454 Some(&first),
6455 "no work happened"
6456 );
6457 t.release_operator("test", None);
6458 state.prep_failure = None;
6459 state.status = RunStatus::Failed;
6460 state.begin_pass();
6461 t.start(state.id.clone());
6462 finish_attempt(
6463 1,
6464 &q,
6465 &mut t,
6466 &state,
6467 &[],
6468 Err(anyhow::anyhow!("new gate failure")),
6469 );
6470 let failure = t.chat_notice_key().unwrap();
6471 assert_ne!(failure, first, "approved recovery reached a real failure");
6472 t.release_operator("test", None);
6473 state.begin_pass();
6474 t.start(state.id.clone());
6475 finish_attempt(
6476 1,
6477 &q,
6478 &mut t,
6479 &state,
6480 &[],
6481 Err(anyhow::anyhow!("new failure on resume")),
6482 );
6483 assert_ne!(
6484 t.chat_notice_key().as_ref(),
6485 Some(&failure),
6486 "a new settled pass"
6487 );
6488 }
6489
6490 #[test]
6491 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6492 for source in ["cli", "web"] {
6493 let mut t = task();
6494 let mut run = run_state(RunStatus::Blocked);
6495 run.instruction = "Run A original criteria".into();
6496 run.task_instruction = Some(run.instruction.clone());
6497 t.start(run.id.clone());
6498 t.release_operator("choice", Some("resume-question"));
6499 t.resume_override = Some(crate::queue::OperatorResume {
6500 question_id: "resume-question".into(),
6501 at: Timestamp::now(),
6502 conductor_rehold: None,
6503 forced: true,
6504 pinned_run: Some(run.id.clone()),
6505 });
6506 t.hold_machine(Some("not enough free space to start a run".into()));
6507 let mut unchanged = t.clone();
6508 unchanged
6509 .edit("title changed".into(), unchanged.instruction.clone())
6510 .unwrap();
6511 unchanged.release_operator(source, None);
6512 assert_eq!(
6513 unchanged
6514 .resume_override
6515 .as_ref()
6516 .unwrap()
6517 .pinned_run
6518 .as_deref(),
6519 Some(run.id.as_str())
6520 );
6521 assert!(!compatible_run_instruction(&run, &unchanged));
6522 assert!(may_resume_run(&run, &unchanged));
6523
6524 t.edit(
6525 "approved recovery".into(),
6526 "Reuse measured PR head B; fix Clippy; review".into(),
6527 )
6528 .unwrap();
6529 assert!(t.resume_override.is_none());
6530 t.release_operator(source, None);
6531 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6532 assert!(reloaded.resume_override.is_none());
6533 assert!(!may_resume_run(&run, &reloaded));
6534 let unfinished =
6535 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6536 .filter(|_| may_resume_run(&run, &reloaded));
6537 assert_eq!(
6538 choose_starter(None, false, unfinished.as_deref()),
6539 Starter::Start
6540 );
6541 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6542 assert_eq!(
6543 reloaded.resumption_history[0].question_id.as_deref(),
6544 Some("resume-question")
6545 );
6546
6547 unchanged.instruction = "Reuse measured PR head B".into();
6550 unchanged.hold_machine(Some("disk hold".into()));
6551 unchanged.release_operator(source, None);
6552 assert!(unchanged.resume_override.is_none());
6553 assert!(!may_resume_run(&run, &unchanged));
6554 }
6555 }
6556
6557 #[test]
6558 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6559 let mut t = task();
6560 let mut run = run_state(RunStatus::Blocked);
6561 t.review_of = Some("approved-head".into());
6562 t.instruction =
6563 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6564 .into();
6565 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6566 run.reviewed_commits = Some(vec!["Commit log".into()]);
6567 t.start(run.id.clone());
6568 t.hold_machine(Some("interrupted review".into()));
6569 t.release_operator("cli", None);
6570 assert!(
6571 compatible_run_instruction(&run, &t),
6572 "legacy explicit review remains resumable"
6573 );
6574 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6575 .filter(|_| compatible_run_instruction(&run, &t));
6576 assert_eq!(
6577 choose_starter(None, false, unfinished.as_deref()),
6578 Starter::Resume(run.id.clone())
6579 );
6580 run.task_instruction = Some(t.instruction.clone());
6581 assert!(compatible_run_instruction(&run, &t));
6582 let reloaded: RunState =
6583 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6584 assert!(compatible_run_instruction(&reloaded, &t));
6585 t.edit(
6586 "new approved head".into(),
6587 "Reuse measured PR head B".into(),
6588 )
6589 .unwrap();
6590 assert!(!compatible_run_instruction(&run, &t));
6591
6592 t.review_of = None;
6594 t.instruction = "Fix Clippy on approved head".into();
6595 run.task_instruction = None;
6596 run.instruction = format!(
6597 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6598 t.instruction
6599 );
6600 assert!(compatible_run_instruction(&run, &t));
6601 t.instruction.push('\n');
6602 assert!(
6603 compatible_run_instruction(&run, &t),
6604 "legacy generated review trimmed the original criteria"
6605 );
6606 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6607 let mut unchanged = t.clone();
6608 assert_eq!(
6609 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6610 Some("approved-head")
6611 );
6612 t.edit("new plan".into(), "Reuse measured PR head B".into())
6613 .unwrap();
6614 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6615 assert!(!compatible_run_instruction(&run, &t));
6616 t.review_branch = Some("approved-head".into());
6617 t.review_instruction = None;
6618 assert!(
6619 take_review_branch(&mut t, Some(&run)).is_none(),
6620 "legacy retained candidate cannot replace a revised approved head"
6621 );
6622 }
6623
6624 #[test]
6625 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6626 crate::run::pin_test_home();
6627 let dir = tempfile::tempdir().unwrap();
6628 let queue = Queue::at(dir.path().join("queue"));
6629 for resume in [true, false] {
6630 let state = run_state(RunStatus::Blocked);
6631 let mut t = task();
6632 t.start(state.id.clone());
6633 let action = if resume {
6634 ask::ChoiceAction::Resume {
6635 run: state.id.clone(),
6636 }
6637 } else {
6638 ask::ChoiceAction::Requeue
6639 };
6640 let mut q = action_question(&state.id, action);
6641 q.status = ask::QuestionStatus::Open;
6642 q.answer = None;
6643 q.answered_at = None;
6644 t.fail("run stopped with a question", 3);
6645 hold_if_runnable(&queue, &mut t);
6646 assert!(t.operator_decided_at.is_none());
6647 assert!(t.operator_held());
6648 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6649 let expected = if resume {
6650 ActionDecision::Resume(state.id.clone())
6651 } else {
6652 ActionDecision::Requeue
6653 };
6654 assert_eq!(
6655 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6656 expected
6657 );
6658 t.hold_manual(Some("later explicit hold".into()));
6659 assert_eq!(
6660 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6661 ActionDecision::Stale
6662 );
6663 }
6664 }
6665
6666 #[test]
6667 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6668 let mut t = task();
6669 t.answers.push(crate::queue::AnsweredQuestion {
6670 question: "Inspect the remote?".into(),
6671 answer: "Inspect then wait".into(),
6672 });
6673 let old = instruction_for(&t);
6674 assert!(compatible_instruction(&old, &t));
6675 t.instruction = "Reuse approved PR head and review".into();
6676 t.release_operator("cli", None);
6677 assert!(!compatible_instruction(&old, &t));
6678 let approved = instruction_for(&t);
6679 assert!(approved.contains("waiting premises are superseded"));
6680 assert!(compatible_instruction(&approved, &t));
6681 assert_eq!(resumed_instruction(&approved, &t), approved);
6682 t.release_operator("web", None);
6683 assert!(compatible_instruction(&approved, &t));
6684 assert_eq!(t.resumption_history.len(), 1);
6685 t.start("run".into());
6686 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6687 }
6688
6689 #[test]
6693 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
6694 let mut state = RunState::new(
6695 PathBuf::from("/repo"),
6696 "main".to_owned(),
6697 "abc1234".to_owned(),
6698 "task".to_owned(),
6699 Config::default(),
6700 );
6701 state.status = RunStatus::Blocked;
6702 state.resource_block = Some(crate::run::ResourceBlock {
6703 cause: crate::run::ResourceBlockCause::Other,
6704 node: "graft".to_owned(),
6705 kind: "shared build cache".to_owned(),
6706 detail: "timed out".to_owned(),
6707 at: jiff::Timestamp::now(),
6708 });
6709 let mut t = task();
6710 t.start("20260902-000000-aaaa".to_owned());
6711 assert_eq!(t.attempts, 1);
6712 let verdict = Verdict {
6713 status: RunStatus::Blocked,
6714 left_pr: false,
6715 quota_hit: false,
6716 parked: false,
6717 no_viable_candidates: false,
6718 };
6719 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6720 assert_eq!(t.status, TaskStatus::Held);
6721 assert_eq!(t.attempts, 0, "the attempt is refunded");
6722 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6723 assert!(
6724 !t.operator_held(),
6725 "a machine hold, released without a human"
6726 );
6727
6728 let mut selected = task();
6729 selected.start("20260902-000000-cccc".into());
6730 selected.replacement = Some(crate::merge_decision::Replacement {
6731 question: "fixture".into(),
6732 run: "earlier".into(),
6733 pr: "https://example.invalid/pr/1".into(),
6734 agent: "selected".into(),
6735 context: "fixture".into(),
6736 });
6737 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6738 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6739 assert_eq!(selected.attempts, 0);
6740 assert!(
6741 !selected.operator_held(),
6742 "a selected agent also retains a machine resource hold"
6743 );
6744 assert!(
6745 selected
6746 .hold_reason
6747 .as_deref()
6748 .unwrap()
6749 .starts_with("[resource] ")
6750 );
6751
6752 let mut p = task();
6754 p.start("20260902-000000-bbbb".to_owned());
6755 settle_and_diagnose(
6756 &mut p,
6757 Verdict {
6758 parked: true,
6759 ..verdict
6760 },
6761 "parked",
6762 2,
6763 &state,
6764 );
6765 assert_eq!(p.status, TaskStatus::Parked);
6766 assert_eq!(
6767 p.attempts, 0,
6768 "parking refunds once rather than also settling the resource hold"
6769 );
6770 }
6771
6772 #[test]
6773 fn record_failure_notice_links_the_task() {
6774 let dir = tempfile::tempdir().unwrap();
6775 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6778 let queue = Queue::at(dir.path().join("queue"));
6779 let mut t = interrupt_task("rec-fail-1");
6780 record(&queue, &mut t);
6781
6782 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6783 assert_eq!(found.len(), 1);
6784 assert_eq!(found[0].key, "loop:record");
6785 assert_eq!(
6786 found[0].link,
6787 Some(Link::Task {
6788 id: "rec-fail-1".to_owned()
6789 })
6790 );
6791 }
6792
6793 fn interrupt_task(id: &str) -> Task {
6796 let mut t = task();
6797 t.id = id.to_owned();
6798 t.interrupt = true;
6799 t
6800 }
6801
6802 fn task_with_id(id: &str) -> Task {
6804 let mut t = task();
6805 t.id = id.to_owned();
6806 t
6807 }
6808
6809 fn urgent_task(id: &str) -> Task {
6811 let mut t = task();
6812 t.id = id.to_owned();
6813 t.urgent = true;
6814 t
6815 }
6816
6817 #[test]
6818 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6819 let dir = tempfile::tempdir().unwrap();
6820 let queue = Queue::at(dir.path().join("queue"));
6821 let mut parked = task_with_id("parked");
6822 parked.start("run".into());
6823 parked.park("upgrade");
6824 parked.record_pass_at(
6825 "run",
6826 0,
6827 Some(Timestamp::now()),
6828 1,
6829 Some(crate::run::ParkCause::Upgrade {
6830 from: "old".into(),
6831 to: None,
6832 }),
6833 );
6834 let mut fresh = task_with_id("fresh");
6835 fresh.priority = 100;
6836 let mut urgent_task = urgent_task("urgent");
6837 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6838 queue.put(t).unwrap();
6839 }
6840 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6841 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6842 let candidates = runnable(&queue);
6843 assert_eq!(candidates[0].id, parked.id);
6844 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6845 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6846 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6847 drop(first);
6848 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6849 let next = runnable(&queue)
6850 .into_iter()
6851 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6852 .unwrap();
6853 assert_eq!(next.task.id, fresh.id);
6854 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6855 drop(claimed_elsewhere);
6856 drop(next);
6857 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6858 }
6859
6860 #[test]
6861 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6862 let dir = tempfile::tempdir().unwrap();
6863 let queue = Queue::at(dir.path().join("queue"));
6864 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6865 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6866 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6867 let mut first = task();
6868 queue.put(&mut first).unwrap();
6869 let stale_plain = queue.get(&first.id).unwrap();
6870 first.set_urgent(true).unwrap();
6871 queue.put(&mut first).unwrap();
6872 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6873 let mut second = urgent_task("second");
6874 queue.put(&mut second).unwrap();
6875 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6876 drop(queue.claim(&second.id).unwrap());
6877 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6878 drop(dispatched);
6879 let stale_urgent = queue.get(&first.id).unwrap();
6880 first.set_urgent(false).unwrap();
6881 queue.put(&mut first).unwrap();
6882 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6883 assert_eq!(urgent.available_permits(), 1);
6884 drop(queue.claim(&first.id).unwrap());
6885 drop(occupied);
6886 }
6887
6888 #[test]
6889 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6890 let dir = tempfile::tempdir().unwrap();
6891 let queue = Queue::at(dir.path().join("queue"));
6892 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6893 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6894 let mut t = task();
6895 queue.put(&mut t).unwrap();
6896 for enabled in [true, false] {
6897 let stale = queue.get(&t.id).unwrap();
6898 t.set_urgent(enabled).unwrap();
6899 queue.put(&mut t).unwrap();
6900 assert_ne!(stale.urgent, enabled);
6901 let DispatchClaim {
6902 task: fresh,
6903 claim,
6904 permit,
6905 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6906 assert_eq!(fresh.urgent, enabled);
6907 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6908 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6909 assert!(queue.claim(&t.id).is_err());
6910 assert_eq!(fresh.attempts, stale.attempts);
6911 drop((claim, permit));
6912 assert_eq!(ordinary.available_permits(), 1);
6913 assert_eq!(urgent.available_permits(), 1);
6914 }
6915 t.set_urgent(true).unwrap();
6916 queue.put(&mut t).unwrap();
6917 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6918 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6919 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6920 drop(queue.claim(&t.id).unwrap());
6921 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6922 drop(occupied);
6923 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6924 drop(queue.claim(&t.id).unwrap());
6925 let held = queue.claim(&t.id).unwrap();
6926 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6927 drop(held);
6928 t.status = TaskStatus::Held;
6929 queue.put(&mut t).unwrap();
6930 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6931 drop(queue.claim(&t.id).unwrap());
6932 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6933 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6934 drop(queue.claim(&t.id).unwrap());
6935 assert_eq!(urgent.available_permits(), 1);
6936 }
6937
6938 #[test]
6943 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6944 assert_eq!(permit_kind(true, false), PermitKind::None);
6945 assert_eq!(permit_kind(true, true), PermitKind::None);
6946 }
6947
6948 #[test]
6953 fn permit_kind_separates_urgent_from_ordinary() {
6954 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6955 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6956 }
6957
6958 #[test]
6965 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6966 let cfg = Config::default();
6967 let repo = Path::new("/any/repo/path");
6968
6969 let reason =
6970 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
6971 assert!(reason.contains("1024"), "{reason}");
6972 assert!(
6973 reason.contains(&cfg.disk.min_free_bytes.to_string()),
6974 "{reason}"
6975 );
6976
6977 assert_eq!(
6978 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
6979 None,
6980 "exactly at the floor is open"
6981 );
6982 assert_eq!(
6983 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
6984 None,
6985 "comfortably above the floor is open"
6986 );
6987 }
6988
6989 #[test]
6990 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
6991 let mut cfg = Config::default();
6992 cfg.disk.min_free_bytes = 0;
6993 let repo = Path::new("/any/repo/path");
6994 assert_eq!(
6995 disk_gate_with(repo, &cfg, |_| Ok(0)),
6996 None,
6997 "a zero floor never measures at all"
6998 );
6999 }
7000
7001 #[test]
7002 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7003 let cfg = Config::default();
7004 let repo = Path::new("/any/repo/path");
7005 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7006 .expect("a measurement failure must close the gate, not open it");
7007 assert!(reason.contains("could not measure"), "{reason}");
7008 }
7009
7010 #[test]
7011 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7012 let ordinary = task();
7013 let next = advance_interrupt(
7014 Interrupt::Idle,
7015 std::slice::from_ref(&ordinary.id),
7016 std::slice::from_ref(&ordinary),
7017 );
7018 assert_eq!(next, Interrupt::Idle);
7019 }
7020
7021 #[test]
7022 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7023 let marked = interrupt_task("marked");
7026 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7027 assert_eq!(next, Interrupt::Idle);
7028 }
7029
7030 #[test]
7031 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7032 let marked = interrupt_task("marked");
7033 let next = advance_interrupt(
7034 Interrupt::Idle,
7035 &["running".to_owned()],
7036 std::slice::from_ref(&marked),
7037 );
7038 assert_eq!(
7039 next,
7040 Interrupt::Parking {
7041 parked: vec!["running".to_owned()],
7042 interrupt_task: "marked".to_owned(),
7043 }
7044 );
7045 }
7046
7047 #[test]
7056 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7057 let marked = interrupt_task("marked");
7058
7059 let two = advance_interrupt(
7060 Interrupt::Idle,
7061 &["a".to_owned(), "b".to_owned()],
7062 std::slice::from_ref(&marked),
7063 );
7064 assert_eq!(two, Interrupt::Idle);
7065
7066 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7067 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7068 }
7069
7070 #[test]
7071 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7072 let state = Interrupt::Parking {
7073 parked: vec!["running".to_owned()],
7074 interrupt_task: "marked".to_owned(),
7075 };
7076 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7078 assert_eq!(still_going, state);
7079
7080 let stopped_but_not_yet_dispatched =
7084 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7085 assert_eq!(stopped_but_not_yet_dispatched, state);
7086
7087 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7089 assert_eq!(
7090 dispatched,
7091 Interrupt::Running {
7092 parked: vec!["running".to_owned()],
7093 interrupt_task: "marked".to_owned(),
7094 }
7095 );
7096 }
7097
7098 #[test]
7099 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7100 let state = Interrupt::Running {
7101 parked: vec!["running".to_owned()],
7102 interrupt_task: "marked".to_owned(),
7103 };
7104 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7105 assert_eq!(still_running, state);
7106
7107 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7114 assert_eq!(
7115 ended,
7116 Interrupt::Resuming {
7117 parked: vec!["running".to_owned()]
7118 }
7119 );
7120 }
7121
7122 #[test]
7123 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7124 let state = Interrupt::Resuming {
7125 parked: vec!["running".to_owned()],
7126 };
7127 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7128 assert_eq!(still_waiting, state);
7129
7130 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7131 assert_eq!(dispatched, Interrupt::Idle);
7132 }
7133
7134 #[test]
7140 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7141 {
7142 let state = Interrupt::Parking {
7143 parked: vec!["running".to_owned()],
7144 interrupt_task: "marked".to_owned(),
7145 };
7146 let next = advance_interrupt(state, &[], &[]);
7149 assert_eq!(
7150 next,
7151 Interrupt::Resuming {
7152 parked: vec!["running".to_owned()]
7153 },
7154 "abandoning the interrupt must not abandon the resume it owes"
7155 );
7156 }
7157
7158 #[test]
7161 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7162 let state = Interrupt::Resuming {
7163 parked: vec!["running".to_owned()],
7164 };
7165 let next = advance_interrupt(state, &[], &[]);
7166 assert_eq!(
7167 next,
7168 Interrupt::Idle,
7169 "nothing is left to wait for; the loop must not stay wedged"
7170 );
7171 }
7172
7173 #[test]
7174 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7175 let marked = interrupt_task("marked");
7176 let next = advance_interrupt_tick(
7177 false,
7178 Interrupt::Idle,
7179 &["running".to_owned()],
7180 std::slice::from_ref(&marked),
7181 );
7182 assert_eq!(
7183 next,
7184 Interrupt::Idle,
7185 "an unmarked, unconfigured daemon must behave exactly as before"
7186 );
7187 }
7188
7189 #[test]
7190 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7191 let state = Interrupt::Parking {
7192 parked: vec!["running".to_owned()],
7193 interrupt_task: "marked".to_owned(),
7194 };
7195 let candidates = vec![interrupt_task("marked"), task()];
7196 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7197 assert!(
7198 allowed.is_empty(),
7199 "nothing may dispatch - not even the interrupt task itself - \
7200 until the parked run has actually stopped"
7201 );
7202 }
7203
7204 #[test]
7218 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7219 for state in [
7220 Interrupt::Parking {
7221 parked: vec!["running".to_owned()],
7222 interrupt_task: "marked".to_owned(),
7223 },
7224 Interrupt::Running {
7225 parked: vec!["running".to_owned()],
7226 interrupt_task: "marked".to_owned(),
7227 },
7228 Interrupt::Resuming {
7229 parked: vec!["running".to_owned()],
7230 },
7231 ] {
7232 let candidates = vec![
7233 interrupt_task("marked"),
7234 urgent_task("hot"),
7235 task_with_id("ordinary"),
7236 ];
7237 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7238 assert!(
7239 !allowed.iter().any(|t| t.id == "hot"),
7240 "an urgent candidate must wait out the same gate as anything \
7241 else while the run it would run alongside has not actually \
7242 left flight, for state {state:?}: {allowed:?}"
7243 );
7244 }
7245 }
7246
7247 #[test]
7253 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7254 let state = Interrupt::Resuming {
7255 parked: vec!["hot".to_owned()],
7256 };
7257 let candidates = vec![urgent_task("hot"), task()];
7258 let allowed = interrupt_gate(&state, &[], candidates);
7259 assert_eq!(
7260 allowed.iter().filter(|t| t.id == "hot").count(),
7261 1,
7262 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7263 );
7264 }
7265
7266 #[test]
7267 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7268 let state = Interrupt::Parking {
7269 parked: vec!["running".to_owned()],
7270 interrupt_task: "marked".to_owned(),
7271 };
7272 let other = task();
7273 let candidates = vec![interrupt_task("marked"), other.clone()];
7274 let allowed = interrupt_gate(&state, &[], candidates);
7275 assert_eq!(allowed.len(), 1);
7276 assert_eq!(allowed[0].id, "marked");
7277 }
7278
7279 #[test]
7280 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7281 let state = Interrupt::Running {
7282 parked: vec!["running".to_owned()],
7283 interrupt_task: "marked".to_owned(),
7284 };
7285 let candidates = vec![task(), task()];
7286 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7287 assert!(allowed.is_empty());
7288 }
7289
7290 #[test]
7297 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7298 let state = Interrupt::Resuming {
7299 parked: vec!["a".to_owned(), "c".to_owned()],
7300 };
7301 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7302 let allowed = interrupt_gate(&state, &[], candidates);
7303 assert_eq!(
7304 allowed.len(),
7305 1,
7306 "at most one candidate may be offered while resuming: {allowed:?}"
7307 );
7308 assert_eq!(allowed[0].id, "a");
7309 }
7310
7311 #[test]
7312 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7313 let state = Interrupt::Resuming {
7314 parked: vec!["a".to_owned()],
7315 };
7316 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7317 assert!(allowed.is_empty());
7318 }
7319
7320 #[test]
7326 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7327 let running = task(); let marked = interrupt_task("marked");
7329
7330 let mut state = Interrupt::Idle;
7331 let in_flight = vec![running.id.clone()];
7333 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7334 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7335 assert!(gated.is_empty(), "still waiting on `running` to park");
7336
7337 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7339 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7340 assert_eq!(
7341 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7342 vec!["marked"],
7343 "only the interrupt task may be offered to the dispatcher now"
7344 );
7345
7346 state = advance_interrupt_tick(
7348 true,
7349 state,
7350 &["marked".to_owned()],
7351 std::slice::from_ref(&running),
7352 );
7353 let gated = interrupt_gate(
7354 &state,
7355 &["marked".to_owned()],
7356 vec![marked.clone(), running.clone()],
7357 );
7358 assert!(
7359 gated.is_empty(),
7360 "the parked run must not be offered back while the interrupt \
7361 task is still running"
7362 );
7363
7364 let other = task_with_id("other");
7368 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7369 assert_eq!(
7370 state,
7371 Interrupt::Resuming {
7372 parked: vec![running.id.clone()]
7373 }
7374 );
7375 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7376 assert_eq!(
7377 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7378 vec![running.id.as_str()],
7379 "exactly the parked run resumes - not the unrelated task, even \
7380 though it was offered first"
7381 );
7382
7383 state = advance_interrupt_tick(
7387 true,
7388 state,
7389 std::slice::from_ref(&running.id),
7390 std::slice::from_ref(&other),
7391 );
7392 assert_eq!(state, Interrupt::Idle);
7393 let gated = interrupt_gate(
7394 &state,
7395 std::slice::from_ref(&running.id),
7396 vec![other.clone()],
7397 );
7398 assert_eq!(
7399 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7400 vec![other.id.as_str()],
7401 "ordinary dispatch is unrestricted again"
7402 );
7403 }
7404
7405 #[test]
7406 fn every_run_status_settles_the_task_it_came_from() {
7407 let table = [
7409 (RunStatus::Merged, TaskStatus::Done, 1),
7410 (RunStatus::Ready, TaskStatus::Done, 1),
7411 (RunStatus::Stalled, TaskStatus::Failed, 0),
7412 (RunStatus::Blocked, TaskStatus::Failed, 1),
7413 (RunStatus::Failed, TaskStatus::Failed, 1),
7414 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7415 (RunStatus::Prep, TaskStatus::Failed, 1),
7416 (RunStatus::Implementing, TaskStatus::Failed, 1),
7417 (RunStatus::Judging, TaskStatus::Failed, 1),
7418 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7419 (RunStatus::Voting, TaskStatus::Failed, 1),
7420 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7421 (RunStatus::Gating, TaskStatus::Failed, 1),
7422 ];
7423 for (run, want, attempts) in table {
7424 let mut t = task();
7425 t.start("20260902-000000-aaaa".to_owned());
7426 settle(
7427 &mut t,
7428 Verdict {
7429 status: run,
7430 left_pr: false,
7431 parked: false,
7432 quota_hit: matches!(run, RunStatus::Stalled),
7433 no_viable_candidates: false,
7434 },
7435 "why",
7436 2,
7437 );
7438 assert_eq!(t.status, want, "task status after {}", label(run));
7439 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7440 }
7441 }
7442
7443 #[test]
7444 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7445 let mut stalled = task();
7446 stalled.start("20260902-000000-aaaa".to_owned());
7447 settle(
7448 &mut stalled,
7449 Verdict {
7450 status: RunStatus::Stalled,
7451 left_pr: false,
7452 parked: false,
7453 quota_hit: true,
7454 no_viable_candidates: false,
7455 },
7456 "quota",
7457 1,
7458 );
7459 assert_eq!(stalled.attempts, 0);
7460 assert!(
7461 stalled.status.runnable(),
7462 "a machine problem must leave the task in line"
7463 );
7464
7465 let mut blocked = task();
7466 blocked.start("20260902-000000-aaaa".to_owned());
7467 settle(
7468 &mut blocked,
7469 Verdict {
7470 status: RunStatus::Blocked,
7471 left_pr: false,
7472 parked: false,
7473 quota_hit: false,
7474 no_viable_candidates: false,
7475 },
7476 "findings open",
7477 1,
7478 );
7479 assert_eq!(blocked.attempts, 1);
7480 assert_eq!(
7481 blocked.status,
7482 TaskStatus::Held,
7483 "the last attempt hands the task to a human"
7484 );
7485 }
7486
7487 #[test]
7488 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7489 let mut delivered = task();
7492 delivered.start("20260903-080619-01c2".to_owned());
7493 settle(
7494 &mut delivered,
7495 Verdict {
7496 status: RunStatus::Blocked,
7497 left_pr: true,
7498 parked: false,
7499 quota_hit: false,
7500 no_viable_candidates: false,
7501 },
7502 "no check status",
7503 4,
7504 );
7505 assert_eq!(
7506 delivered.status,
7507 TaskStatus::Held,
7508 "a pull request waiting on CI or a person is not a retryable failure"
7509 );
7510 assert!(
7511 !delivered.status.runnable(),
7512 "the loop must not pick this task up again"
7513 );
7514 assert_eq!(
7515 delivered.last_error.as_deref(),
7516 Some("no check status"),
7517 "the operator needs to be told what the gate was waiting for"
7518 );
7519
7520 let mut empty_handed = task();
7523 empty_handed.start("20260903-080619-01c2".to_owned());
7524 settle(
7525 &mut empty_handed,
7526 Verdict {
7527 status: RunStatus::Blocked,
7528 left_pr: false,
7529 parked: false,
7530 quota_hit: false,
7531 no_viable_candidates: false,
7532 },
7533 "findings open",
7534 4,
7535 );
7536 assert_eq!(empty_handed.status, TaskStatus::Failed);
7537 assert!(empty_handed.status.runnable());
7538 }
7539
7540 #[test]
7541 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7542 let mut noop = task();
7548 noop.start("20260912-131304-391f".to_owned());
7549 settle(
7550 &mut noop,
7551 Verdict {
7552 status: RunStatus::VerifiedNoop,
7553 left_pr: false,
7554 parked: false,
7555 quota_hit: false,
7556 no_viable_candidates: true,
7557 },
7558 "candidate A: already fixed by b32cfc4, on main",
7559 4,
7560 );
7561 assert_eq!(
7562 noop.status,
7563 TaskStatus::Held,
7564 "an unverified claim is a request for a human, not a failure"
7565 );
7566 assert!(
7567 !noop.status.runnable(),
7568 "the loop must not requeue this on the same unverified claim"
7569 );
7570 assert_eq!(noop.attempts, 1);
7575 }
7576
7577 #[test]
7578 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7579 let mut parked = task();
7584 parked.start("20260903-183634-2d98".to_owned());
7585 settle(
7586 &mut parked,
7587 Verdict {
7588 status: RunStatus::Implementing,
7589 left_pr: false,
7590 quota_hit: false,
7591 parked: true,
7592 no_viable_candidates: false,
7593 },
7594 "parked after `implementing`",
7595 2,
7596 );
7597 assert_eq!(parked.attempts, 0, "a park is refunded");
7598 assert!(
7599 parked.status.runnable(),
7600 "and the task stays in line so the next loop resumes its run"
7601 );
7602 assert_eq!(parked.status, TaskStatus::Parked);
7603 assert_eq!(
7604 parked.park_reason.as_deref(),
7605 Some("parked after `implementing`"),
7606 "the card says where it stopped"
7607 );
7608 assert_eq!(parked.last_error, None, "nothing failed");
7609
7610 let mut broken = task();
7614 broken.start("20260903-183634-2d98".to_owned());
7615 settle(
7616 &mut broken,
7617 Verdict {
7618 status: RunStatus::Implementing,
7619 left_pr: false,
7620 quota_hit: false,
7621 parked: false,
7622 no_viable_candidates: false,
7623 },
7624 "returned mid-flight",
7625 2,
7626 );
7627 assert_eq!(broken.attempts, 1);
7628 }
7629
7630 #[test]
7631 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7632 let mut flaky = task();
7637 flaky.start("20260903-123023-e633".to_owned());
7638 settle(
7639 &mut flaky,
7640 Verdict {
7641 status: RunStatus::Stalled,
7642 left_pr: false,
7643 parked: false,
7644 quota_hit: false,
7645 no_viable_candidates: false,
7646 },
7647 "verdict rests on 1 of 3 judges",
7648 2,
7649 );
7650 assert_eq!(
7651 flaky.attempts, 1,
7652 "flakiness spends an attempt, so `max_attempts` still bounds it"
7653 );
7654 assert!(flaky.status.runnable(), "and it is still worth retrying");
7655
7656 let mut limited = task();
7658 limited.start("20260903-123023-e633".to_owned());
7659 settle(
7660 &mut limited,
7661 Verdict {
7662 status: RunStatus::Stalled,
7663 left_pr: false,
7664 parked: false,
7665 quota_hit: true,
7666 no_viable_candidates: false,
7667 },
7668 "judge-2, judge-3 out of quota",
7669 2,
7670 );
7671 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7672 assert!(limited.status.runnable());
7673
7674 let mut worn = task();
7677 for _ in 0..2 {
7678 worn.release();
7679 }
7680 worn.start("20260903-123023-e633".to_owned());
7681 worn.attempts = 2;
7682 settle(
7683 &mut worn,
7684 Verdict {
7685 status: RunStatus::Stalled,
7686 left_pr: false,
7687 parked: false,
7688 quota_hit: false,
7689 no_viable_candidates: false,
7690 },
7691 "no quorum again",
7692 2,
7693 );
7694 assert_eq!(worn.status, TaskStatus::Held);
7695 assert!(!worn.status.runnable());
7696 }
7697
7698 #[test]
7699 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
7700 let mut wiped_out = task();
7707 wiped_out.start("20260907-025000-a1b2".to_owned());
7708 settle(
7709 &mut wiped_out,
7710 Verdict {
7711 status: RunStatus::Failed,
7712 left_pr: false,
7713 parked: false,
7714 quota_hit: true,
7715 no_viable_candidates: true,
7716 },
7717 "no candidate produced a change; nothing to judge",
7718 2,
7719 );
7720 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7721 assert!(
7722 wiped_out.status.runnable(),
7723 "a machine problem must leave the task in line"
7724 );
7725
7726 let mut partial_progress = task();
7732 partial_progress.start("20260907-025500-c3d4".to_owned());
7733 settle(
7734 &mut partial_progress,
7735 Verdict {
7736 status: RunStatus::Failed,
7737 left_pr: false,
7738 parked: false,
7739 quota_hit: true,
7740 no_viable_candidates: false,
7741 },
7742 "gate failed on the winning candidate",
7743 2,
7744 );
7745 assert_eq!(
7746 partial_progress.attempts, 1,
7747 "a candidate that actually produced a change spends the attempt \
7748 even though some other seat hit its quota"
7749 );
7750 assert!(partial_progress.status.runnable());
7751 }
7752
7753 #[test]
7754 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7755 let mut t = task();
7761 t.start("20260907-025000-a1b2".to_owned());
7762 let mut state = run_state(RunStatus::Failed);
7763 state.quota.push(QuotaLoss {
7764 seat: "cand-a".to_owned(),
7765 node: "implement".to_owned(),
7766 at: Timestamp::now(),
7767 reset: None,
7768 });
7769 assert!(
7770 state.viable().is_empty(),
7771 "no candidate was added, so nothing is viable"
7772 );
7773 reclaim(&mut t, Some(state), 2, "en");
7774 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7775 assert!(t.status.runnable());
7776 }
7777
7778 #[test]
7779 fn a_held_task_is_never_offered_to_the_loop() {
7780 let dir = tempfile::tempdir().unwrap();
7781 let queue = Queue::at(dir.path().to_path_buf());
7782 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7783 let mut t = task();
7784 t.id = format!("2026090{n}-000000-000{n}");
7785 t.priority = priority;
7786 queue.put(&mut t).unwrap();
7787 }
7788 let mut held = task();
7789 held.id = "20260909-000000-9999".to_owned();
7790 held.priority = 99;
7791 held.hold_machine(None);
7792 queue.put(&mut held).unwrap();
7793
7794 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7795 assert_eq!(order.len(), 3);
7796 assert!(!order.contains(&held.id));
7797 assert_eq!(
7798 order.first().cloned(),
7799 queue.next_runnable().map(|t| t.id),
7800 "the loop's first candidate is exactly what the queue offers"
7801 );
7802 assert_eq!(
7803 order,
7804 vec![
7805 "20260902-000000-0002".to_owned(),
7806 "20260903-000000-0003".to_owned(),
7807 "20260901-000000-0001".to_owned(),
7808 ],
7809 "priority first, then oldest, so nothing starves"
7810 );
7811 }
7812
7813 #[test]
7814 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7815 let dir = tempfile::tempdir().unwrap();
7816 let queue = Queue::at(dir.path().to_path_buf());
7817 let mut old = task();
7818 old.id = "20260101-000000-old0".to_owned();
7819 queue.put(&mut old).unwrap();
7820 let mut fresh = task();
7821 fresh.id = "20260101-000000-new0".to_owned();
7822 queue.put(&mut fresh).unwrap();
7823
7824 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7828 std::thread::sleep(Duration::from_millis(60));
7829 let live = queue.claim(&fresh.id).unwrap();
7830
7831 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7832 assert_eq!(swept, vec![old.id.clone()]);
7833 assert!(
7834 queue.claim(&old.id).is_ok(),
7835 "an unparseable lock older than the threshold is swept"
7836 );
7837 assert!(
7838 queue.claim(&fresh.id).is_err(),
7839 "a live pid protects its lock regardless of age"
7840 );
7841 drop(live);
7842 }
7843
7844 #[test]
7845 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7846 let dir = tempfile::tempdir().unwrap();
7849 let queue = Queue::at(dir.path().to_path_buf());
7850 let mut t = task();
7851 t.id = "20260101-000000-live".to_owned();
7852 queue.put(&mut t).unwrap();
7853
7854 let claim = queue.claim(&t.id).unwrap();
7855 std::thread::sleep(Duration::from_millis(60));
7856
7857 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7858 assert!(
7859 swept.is_empty(),
7860 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7861 );
7862 assert!(
7863 queue.claim(&t.id).is_err(),
7864 "the lock still protects its task"
7865 );
7866 drop(claim);
7867 }
7868
7869 fn injected_dead_pid() -> u32 {
7872 std::process::id().checked_add(1).unwrap_or(1)
7873 }
7874
7875 #[test]
7876 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7877 let dir = tempfile::tempdir().unwrap();
7878 let queue = Queue::at(dir.path().to_path_buf());
7879 let mut t = task();
7880 t.id = "20260101-000000-dead".to_owned();
7881 queue.put(&mut t).unwrap();
7882 let dead_pid = injected_dead_pid();
7883
7884 std::fs::write(
7889 dir.path().join(format!("{}.lock", t.id)),
7890 dead_pid.to_string(),
7891 )
7892 .unwrap();
7893
7894 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7895 pid != dead_pid
7896 });
7897 assert_eq!(
7898 swept,
7899 vec![t.id.clone()],
7900 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7901 );
7902 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7903 }
7904
7905 #[test]
7906 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7907 let dir = tempfile::tempdir().unwrap();
7908 let queue = Queue::at(dir.path().to_path_buf());
7909 let mut t = task();
7910 t.id = "20260101-000000-late".to_owned();
7911 queue.put(&mut t).unwrap();
7912 let dead_pid = injected_dead_pid();
7913
7914 assert!(
7917 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7918 "nothing has claimed the task yet"
7919 );
7920
7921 std::fs::write(
7924 dir.path().join(format!("{}.lock", t.id)),
7925 dead_pid.to_string(),
7926 )
7927 .unwrap();
7928
7929 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7933 pid != dead_pid
7934 });
7935 assert_eq!(swept, vec![t.id.clone()]);
7936 }
7937
7938 #[test]
7939 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7940 crate::run::pin_test_home();
7945 let dir = tempfile::tempdir().unwrap();
7946 let queue = Queue::at(dir.path().to_path_buf());
7947 let mut t = task();
7948 t.id = "20260101-000000-crsh".to_owned();
7949 t.status = TaskStatus::Running;
7950 t.attempts = 1;
7951 t.runs.push("20260904-000000-4043".to_owned());
7955 queue.put(&mut t).unwrap();
7956 let dead_pid = injected_dead_pid();
7957
7958 std::fs::write(
7961 dir.path().join(format!("{}.lock", t.id)),
7962 dead_pid.to_string(),
7963 )
7964 .unwrap();
7965
7966 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
7972 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
7973
7974 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7975 pid != dead_pid
7976 });
7977 assert_eq!(swept, vec![t.id.clone()]);
7978
7979 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
7980 assert_eq!(reclaimed, vec![t.id.clone()]);
7981 let after = queue.get(&t.id).unwrap();
7982 assert_eq!(
7983 after.status,
7984 TaskStatus::Held,
7985 "no run.json to recover from, so a human is asked"
7986 );
7987 assert_eq!(
7988 after.runs,
7989 vec!["20260904-000000-4043".to_owned()],
7990 "the crashed run's id is kept as evidence, not discarded"
7991 );
7992 }
7993
7994 #[test]
7995 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
7996 let dir = tempfile::tempdir().unwrap();
7997 let queue = Queue::at(dir.path().to_path_buf());
7998 let mut t = task();
7999 t.id = "20260101-000000-unknown".to_owned();
8000 queue.put(&mut t).unwrap();
8001 let dead_pid = injected_dead_pid();
8002 std::fs::write(
8003 dir.path().join(format!("{}.lock", t.id)),
8004 dead_pid.to_string(),
8005 )
8006 .unwrap();
8007
8008 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8009 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8010 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8011 }
8012
8013 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8014 let mut s = run_state(status);
8015 s.config.graph.language = language.to_owned();
8016 s
8017 }
8018
8019 fn unstarted_verdict(status: RunStatus) -> Verdict {
8020 Verdict {
8021 status,
8022 left_pr: false,
8023 quota_hit: false,
8024 parked: false,
8025 no_viable_candidates: false,
8026 }
8027 }
8028
8029 #[test]
8030 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8031 let mut t = task();
8032 t.attempts = 1;
8033 settle_in(
8034 &mut t,
8035 unstarted_verdict(RunStatus::AlreadyInBase),
8036 "already in main",
8037 1,
8038 phrases("en"),
8039 );
8040 assert_eq!(t.status, TaskStatus::Done);
8041 assert_eq!(t.attempts, 0);
8042 }
8043
8044 #[test]
8045 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8046 let reason = |language: &str| {
8047 let mut t = task();
8048 settle_in(
8049 &mut t,
8050 unstarted_verdict(RunStatus::Judging),
8051 "boom",
8052 1,
8053 phrases(language),
8054 );
8055 t.last_error.or(t.hold_reason).unwrap_or_default()
8056 };
8057 assert!(
8058 reason("en").starts_with("the graph stopped at `"),
8059 "{}",
8060 reason("en")
8061 );
8062 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8063 assert!(reason("日本語").contains("boom"));
8064 assert_eq!(reason("fr"), reason("en"));
8065 }
8066
8067 #[test]
8068 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8069 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8070 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8071 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8072 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8073 assert!(ja.contains("[run "), "{ja}");
8074 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8075 assert!(ended.starts_with("run 終了: "), "{ended}");
8076 let mut de = run_state_in(RunStatus::Failed, "de");
8077 let mut en = run_state_in(RunStatus::Failed, "en");
8078 de.id = "same".to_owned();
8079 en.id = "same".to_owned();
8080 assert_eq!(describe(&de), describe(&en));
8081 }
8082
8083 #[test]
8084 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8085 use crate::handover::Refused;
8086 let cases = [
8087 Refused::Foreign {
8088 branch: "b".into(),
8089 path: "/w/x".into(),
8090 why: "made by hand".into(),
8091 },
8092 Refused::Unsafe {
8093 branch: "b".into(),
8094 path: "/w/x".into(),
8095 why: "its worktree has uncommitted changes (a.rs)".into(),
8096 },
8097 Refused::ReleaseFailed {
8098 branch: "b".into(),
8099 path: "/w/x".into(),
8100 run: "ab12".into(),
8101 },
8102 ];
8103 for r in &cases {
8104 let en = (phrases("en").handover_refused)(r);
8105 assert_eq!(en, r.to_string());
8106 let ja = (phrases("ja").handover_refused)(r);
8107 assert!(
8108 !ja.contains("is checked out") && !ja.contains("try again"),
8109 "{ja}"
8110 );
8111 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8112 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8113 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8114 }
8115 }
8116 assert!(phrases("en").handover_hint.contains("release the task"));
8117 assert!(phrases("ja").handover_hint.contains("解放"));
8118 }
8119
8120 #[test]
8121 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8122 let build = |lang: &str| {
8123 let mut s = run_state_in(RunStatus::Blocked, lang);
8124 s.id = "same".to_owned();
8125 s.config.graph.review_rounds = 3;
8126 let mut r = review_round(3);
8127 r.expected = 3;
8128 r.answered = 1;
8129 s.reviews.push(r);
8130 s.event(
8131 "review",
8132 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8133 );
8134 s
8135 };
8136 let en = describe(&build("en"));
8137 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8138 let ja = describe(&build("ja"));
8139 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8140 assert!(!ja.contains("never answered"), "{ja}");
8141 assert!(ja.contains("[run "), "{ja}");
8142
8143 let mut failed = build("ja");
8146 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8147 command: "cargo test".to_owned(),
8148 code: Some(1),
8149 output_tail: String::new(),
8150 duration_ms: 0,
8151 resource_blocked: false,
8152 });
8153 failed.event("review", "stopped; e2e failed: cargo test");
8154 let ja = describe(&failed);
8155 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8156 assert!(!ja.contains("席のレビュアー"), "{ja}");
8157 }
8158
8159 #[test]
8160 fn refusals_and_recovery_prose_follow_the_language() {
8161 let t = held_task_with("r1");
8162 let q = action_question(
8163 "r1",
8164 ask::ChoiceAction::Resume {
8165 run: "r1".to_owned(),
8166 },
8167 );
8168 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
8169 ActionDecision::Refuse(s) => s,
8170 other => panic!("{other:?}"),
8171 };
8172 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8173 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8174 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8175
8176 let mut held = task();
8177 reclaim(&mut held, None, 2, "ja");
8178 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8179 let mut held = task();
8180 reclaim(&mut held, None, 2, "xx");
8181 assert!(held.hold_reason.unwrap().contains("held for a human"));
8182 }
8183
8184 fn run_state(status: RunStatus) -> RunState {
8185 let mut state = RunState::new(
8186 PathBuf::from("/repo"),
8187 "main".to_owned(),
8188 "abc1234def".to_owned(),
8189 "add retries".to_owned(),
8190 Config::default(),
8191 );
8192 state.status = status;
8193 state
8194 }
8195
8196 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8197 Candidate {
8198 index: 0,
8199 label,
8200 agent: "claude".to_owned(),
8201 branch: format!("magi/x/{label}"),
8202 worktree: PathBuf::from("/repo"),
8203 summary: summary.to_owned(),
8204 stat: String::new(),
8205 files: 0,
8206 commits: usize::from(!empty),
8207 empty,
8208 failed: failed.map(str::to_owned),
8209 verified_noop: None,
8210 duration_ms: 0,
8211 folded: false,
8212 }
8213 }
8214
8215 #[test]
8216 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8217 let mut state = run_state(RunStatus::Blocked);
8218 state.gate = vec![
8219 CommandOutcome {
8220 command: "cargo make check".to_owned(),
8221 code: Some(0),
8222 output_tail: "ok".to_owned(),
8223 duration_ms: 0,
8224 resource_blocked: false,
8225 },
8226 CommandOutcome {
8227 command: "cargo test".to_owned(),
8228 code: Some(101),
8229 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8230 duration_ms: 0,
8231 resource_blocked: false,
8232 },
8233 ];
8234 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8235 assert!(d.contains("cargo test"), "{d}");
8236 assert!(
8237 !d.contains("cargo make check"),
8238 "a passing check is not a diagnostic: {d}"
8239 );
8240 assert!(d.contains("assertion failed"), "{d}");
8241 }
8242
8243 #[test]
8244 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8245 let mut state = run_state(RunStatus::Blocked);
8246 state.event(
8247 "land",
8248 "stopped: the fixer produced no commit while 2 check(s) were failing \
8249 (build, lint); stopping instead of looping on an unchanged tree",
8250 );
8251 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8252 assert!(d.contains("build"), "{d}");
8253 assert!(d.contains("lint"), "{d}");
8254 assert!(d.contains("fixer produced no commit"), "{d}");
8255 }
8256
8257 #[test]
8258 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8259 let state = run_state(RunStatus::VerifiedNoop);
8264 let d = describe(&state);
8265 assert!(
8266 d.contains("agent-verified no-op"),
8267 "expected the display label, not the wire spelling: {d}"
8268 );
8269 assert!(!d.contains("verified_noop"), "{d}");
8270 }
8271
8272 #[test]
8273 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8274 let mut state = run_state(RunStatus::Failed);
8280 state.candidates = vec![candidate(
8281 'A',
8282 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8283 true,
8284 None,
8285 )];
8286 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8287 assert!(d.contains("candidate A"), "{d}");
8288 assert!(d.contains("tagged v1.2.3"), "{d}");
8289 }
8290
8291 #[test]
8292 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8293 let mut state = run_state(RunStatus::Failed);
8294 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8295 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8296 assert!(d.contains("candidate A"), "{d}");
8297 assert!(d.contains("agent timed out"), "{d}");
8298 }
8299
8300 #[test]
8301 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8302 let mut state = run_state(RunStatus::Failed);
8305 state.candidates = vec![candidate('A', "did the work", false, None)];
8306 assert!(diagnostic(&state).is_none());
8307 }
8308
8309 #[test]
8310 fn diagnostic_is_bounded_however_much_a_run_printed() {
8311 let mut state = run_state(RunStatus::Blocked);
8312 state.gate = vec![
8313 CommandOutcome {
8314 command: "cargo test".to_owned(),
8315 code: Some(101),
8316 output_tail: "x".repeat(50_000),
8317 duration_ms: 0,
8318 resource_blocked: false,
8319 },
8320 CommandOutcome {
8321 command: "cargo clippy".to_owned(),
8322 code: Some(1),
8323 output_tail: "y".repeat(50_000),
8324 duration_ms: 0,
8325 resource_blocked: false,
8326 },
8327 ];
8328 state.candidates = vec![
8329 candidate('A', &"z".repeat(50_000), true, None),
8330 candidate('B', &"w".repeat(50_000), true, None),
8331 ];
8332 let d = diagnostic(&state).expect("plenty here to diagnose");
8333 assert!(
8334 d.len() <= DIAGNOSTIC_MAX,
8335 "diagnostic grew to {} bytes, unbounded",
8336 d.len()
8337 );
8338 }
8339
8340 #[test]
8341 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8342 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8343 let mut t = task();
8344 let mut state = run_state(status);
8345 t.pr_recovery = Some(crate::pr_recovery::Selection {
8346 url: "url".into(),
8347 branch: format!("magi/task-{}", t.id),
8348 head: "a".repeat(40),
8349 source_head: None,
8350 source_run: None,
8351 source_proof: None,
8352 disposition: Some(crate::pr_recovery::Disposition::Land),
8353 instruction: t.instruction.clone(),
8354 approved_at: Timestamp::now(),
8355 });
8356 state.pr_recovery = t.pr_recovery.clone();
8357 t.start(state.id.clone());
8358 settle_and_diagnose(
8359 &mut t,
8360 Verdict {
8361 status,
8362 left_pr: false,
8363 quota_hit: false,
8364 parked: false,
8365 no_viable_candidates: false,
8366 },
8367 "delivery stopped",
8368 5,
8369 &state,
8370 );
8371 assert_eq!(t.status, TaskStatus::Held);
8372 let reason = t.hold_reason.as_deref().unwrap();
8373 if status == RunStatus::Ready {
8374 assert!(reason.contains("Selected PR delivery is incomplete"));
8375 assert!(t.recovery_budget.is_none());
8376 } else {
8377 assert!(reason.contains("Selected recovery stopped"));
8378 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8379 }
8380 assert!(t.pr_recovery.is_some());
8381 }
8382 }
8383
8384 #[test]
8385 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8386 let mut state = run_state(RunStatus::Blocked);
8387 state.gate = vec![CommandOutcome {
8388 command: "cargo test".to_owned(),
8389 code: Some(101),
8390 output_tail: "assertion failed".to_owned(),
8391 duration_ms: 0,
8392 resource_blocked: false,
8393 }];
8394 let verdict = Verdict {
8395 status: RunStatus::Blocked,
8396 left_pr: false,
8397 quota_hit: false,
8398 parked: false,
8399 no_viable_candidates: false,
8400 };
8401
8402 let mut t = task();
8405 t.start("run-1".to_owned());
8406 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8407 assert_eq!(t.status, TaskStatus::Failed);
8408 assert!(t.diagnostic.is_none());
8409
8410 t.start("run-2".to_owned());
8413 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8414 assert_eq!(t.status, TaskStatus::Held);
8415 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8416 assert!(d.contains("cargo test"), "{d}");
8417 }
8418
8419 #[test]
8420 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8421 crate::run::pin_test_home();
8426 let home = crate::run::home();
8427 let state = run_state(RunStatus::VerifiedNoop);
8428 let mut q = ask::Question::new(
8429 state.id.clone(),
8430 "implement".to_owned(),
8431 "impl-A".to_owned(),
8432 "is this really a no-op?".to_owned(),
8433 String::new(),
8434 Vec::new(),
8435 );
8436 Questions::at(home.join("questions")).put(&mut q).unwrap();
8437
8438 let verdict = Verdict {
8439 status: RunStatus::VerifiedNoop,
8440 left_pr: false,
8441 quota_hit: false,
8442 parked: false,
8443 no_viable_candidates: false,
8444 };
8445 let mut t = task();
8446 t.start(state.id.clone());
8447 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8448
8449 assert_eq!(t.status, TaskStatus::Held);
8450 let reason = t.hold_reason.expect("a held task must record why");
8451 assert!(
8452 reason.starts_with("run ended agent-verified no-op"),
8453 "the original settle reason must survive unchanged: {reason}"
8454 );
8455 assert!(
8456 reason.contains(q.short()),
8457 "the open question's id must be named so the notice is actionable: {reason}"
8458 );
8459 }
8460
8461 #[test]
8462 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8463 crate::run::pin_test_home();
8464 let state = run_state(RunStatus::VerifiedNoop);
8465
8466 let verdict = Verdict {
8467 status: RunStatus::VerifiedNoop,
8468 left_pr: false,
8469 quota_hit: false,
8470 parked: false,
8471 no_viable_candidates: false,
8472 };
8473 let mut t = task();
8474 t.start(state.id.clone());
8475 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8476
8477 assert_eq!(t.status, TaskStatus::Held);
8478 assert_eq!(
8479 t.hold_reason.as_deref(),
8480 Some("run ended agent-verified no-op"),
8481 "nothing to append when the question was already answered or never asked"
8482 );
8483 }
8484
8485 #[test]
8486 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8487 crate::run::pin_test_home();
8488 let mut first = run_state(RunStatus::Blocked);
8489 first.id = "20260101-000000-sup1".to_owned();
8490 first.save().unwrap();
8491 let mut second = run_state(RunStatus::Merged);
8492 second.id = "20260101-000000-sup2".to_owned();
8493 second.save().unwrap();
8494
8495 let mut t = task();
8496 t.runs = vec![first.id.clone(), second.id.clone()];
8497 t.status = TaskStatus::Done;
8498
8499 supersede_prior_runs(&t, &crate::run::home());
8500
8501 assert_eq!(
8502 RunState::load(&first.id).unwrap().status,
8503 RunStatus::Superseded,
8504 "the first attempt's Blocked no longer needs anyone's attention"
8505 );
8506 assert_eq!(
8507 RunState::load(&second.id).unwrap().status,
8508 RunStatus::Merged,
8509 "the run that actually succeeded is left exactly as it was"
8510 );
8511 }
8512
8513 #[test]
8514 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8515 crate::run::pin_test_home();
8521 let mut first = run_state(RunStatus::Blocked);
8522 first.id = "20260101-000000-sup9".to_owned();
8523 first.driver_pid = Some(std::process::id());
8526 first.driver_started_at = Some(
8527 crate::proc::process_started_at(std::process::id())
8528 .expect("this test process's own start time must be queryable"),
8529 );
8530 first.save().unwrap();
8531 let mut second = run_state(RunStatus::Merged);
8532 second.id = "20260101-000000-supa".to_owned();
8533 second.save().unwrap();
8534
8535 let mut t = task();
8536 t.runs = vec![first.id.clone(), second.id.clone()];
8537 t.status = TaskStatus::Done;
8538
8539 supersede_prior_runs(&t, &crate::run::home());
8540
8541 assert_eq!(
8542 RunState::load(&first.id).unwrap().status,
8543 RunStatus::Blocked,
8544 "a live driver_pid means something is still actually working this run, \
8545 even though no daemon claims it - rewriting under it would just be \
8546 undone the next time that process saves"
8547 );
8548 }
8549
8550 #[test]
8551 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8552 let dir = tempfile::tempdir().unwrap();
8558 let home = dir.path().to_path_buf();
8559 let queue = Queue::at(dir.path().join("queue"));
8560
8561 let mut first = run_state(RunStatus::Blocked);
8562 first.id = "20260101-000000-supd".to_owned();
8563 first.driver_pid = Some(std::process::id());
8564 first.driver_started_at = Some(
8565 crate::proc::process_started_at(std::process::id())
8566 .expect("this test process's own start time must be queryable"),
8567 );
8568 first.save_under(&home).unwrap();
8569 let mut second = run_state(RunStatus::Merged);
8570 second.id = "20260101-000000-supe".to_owned();
8571 second.save_under(&home).unwrap();
8572
8573 let mut t = task();
8574 t.runs = vec![first.id.clone(), second.id.clone()];
8575 t.status = TaskStatus::Done;
8576 queue.put(&mut t).unwrap();
8577
8578 resweep_superseded_attempts(&queue, &home);
8579 assert_eq!(
8580 RunState::load_under(&first.id, &home).unwrap().status,
8581 RunStatus::Blocked,
8582 "still live on the first pass, so still untouched"
8583 );
8584
8585 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8591 stale.driver_started_at = Some("1".to_owned());
8592 stale.save_under(&home).unwrap();
8593
8594 resweep_superseded_attempts(&queue, &home);
8595 assert_eq!(
8596 RunState::load_under(&first.id, &home).unwrap().status,
8597 RunStatus::Superseded,
8598 "the second pass catches up what the first one correctly skipped"
8599 );
8600 }
8601
8602 #[test]
8603 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8604 crate::run::pin_test_home();
8605 let mut first = run_state(RunStatus::Blocked);
8606 first.id = "20260101-000000-sup3".to_owned();
8607 first.save().unwrap();
8608 let mut second = run_state(RunStatus::Blocked);
8609 second.id = "20260101-000000-sup4".to_owned();
8610 second.save().unwrap();
8611
8612 let mut t = task();
8613 t.runs = vec![first.id.clone(), second.id.clone()];
8614 t.status = TaskStatus::Failed;
8618
8619 supersede_prior_runs(&t, &crate::run::home());
8620
8621 assert_eq!(
8622 RunState::load(&first.id).unwrap().status,
8623 RunStatus::Blocked
8624 );
8625 assert_eq!(
8626 RunState::load(&second.id).unwrap().status,
8627 RunStatus::Blocked
8628 );
8629 }
8630
8631 #[test]
8632 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
8633 crate::run::pin_test_home();
8637 let mut first = run_state(RunStatus::Blocked);
8638 first.id = "20260101-000000-sup5".to_owned();
8639 first.save().unwrap();
8640
8641 let mut t = task();
8642 t.runs = vec![first.id.clone()];
8643 t.status = TaskStatus::Done;
8644
8645 supersede_prior_runs(&t, &crate::run::home());
8646
8647 assert_eq!(
8648 RunState::load(&first.id).unwrap().status,
8649 RunStatus::Blocked,
8650 "a single-attempt task has no earlier run to supersede"
8651 );
8652 }
8653
8654 #[test]
8655 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
8656 crate::run::pin_test_home();
8663 let mut first = run_state(RunStatus::Blocked);
8664 first.id = "20260101-000000-supb".to_owned();
8665 first.save().unwrap();
8666 let mut second = run_state(RunStatus::Failed);
8667 second.id = "20260101-000000-supc".to_owned();
8668 second.save().unwrap();
8669
8670 let mut t = task();
8671 t.runs = vec![first.id.clone(), second.id.clone()];
8672 t.status = TaskStatus::Done;
8673
8674 supersede_prior_runs(&t, &crate::run::home());
8675
8676 assert_eq!(
8677 RunState::load(&first.id).unwrap().status,
8678 RunStatus::Blocked,
8679 "the task's last attempt never landed, so there is nothing here \
8680 actually superseding it"
8681 );
8682 }
8683
8684 #[test]
8685 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
8686 crate::run::pin_test_home();
8690 let mut failed = run_state(RunStatus::Failed);
8691 failed.id = "20260101-000000-sup6".to_owned();
8692 failed.save().unwrap();
8693 let mut noop = run_state(RunStatus::VerifiedNoop);
8694 noop.id = "20260101-000000-sup7".to_owned();
8695 noop.save().unwrap();
8696 let mut winner = run_state(RunStatus::Ready);
8697 winner.id = "20260101-000000-sup8".to_owned();
8698 winner.save().unwrap();
8699
8700 let mut t = task();
8701 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8702 t.status = TaskStatus::Done;
8703
8704 supersede_prior_runs(&t, &crate::run::home());
8705
8706 assert_eq!(
8707 RunState::load(&failed.id).unwrap().status,
8708 RunStatus::Failed
8709 );
8710 assert_eq!(
8711 RunState::load(&noop.id).unwrap().status,
8712 RunStatus::VerifiedNoop
8713 );
8714 }
8715
8716 fn approval_question(run: &str) -> ask::Question {
8717 ask::Question::new(
8718 run.to_owned(),
8719 land::APPROVAL_NODE.to_owned(),
8720 "land".to_owned(),
8721 "merge?".to_owned(),
8722 String::new(),
8723 vec!["merge".to_owned(), "hold".to_owned()],
8724 )
8725 }
8726
8727 #[test]
8728 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8729 crate::run::pin_test_home();
8730 let mut state = run_state(RunStatus::Landing);
8731 state.id = "20260101-000000-fre1".to_owned();
8732 state.parked = true;
8733 state.save().unwrap();
8734 ask::Questions::open()
8735 .put(&mut approval_question(&state.id))
8736 .unwrap();
8737
8738 let mut t = task();
8739 t.runs.push(state.id.clone());
8740 assert_eq!(
8741 land_resume_state(&t),
8742 LandResume::StillWaiting,
8743 "nobody has answered and the timeout has not passed"
8744 );
8745 }
8746
8747 #[test]
8748 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8749 crate::run::pin_test_home();
8750 let mut state = run_state(RunStatus::Gating);
8751 state.id = "20260101-000000-gt01".to_owned();
8752 state.parked = true;
8753 state.config.graph.answer_timeout = 60;
8754 state.github_text = Some(crate::run::GithubTextAsk {
8755 rewrite: None,
8756 fingerprint: "f".into(),
8757 title: true,
8758 body: false,
8759 categories: vec!["title-language".into()],
8760 question: None,
8761 asks: 1,
8762 resolved: false,
8763 chosen_title: None,
8764 choice: None,
8765 });
8766 state.save().unwrap();
8767 let store = ask::Questions::open();
8768 let mut q = ask::Question::new(
8769 state.id.clone(),
8770 crate::github_text::ASK_NODE.to_owned(),
8771 crate::github_text::ASK_SEAT.to_owned(),
8772 "s".into(),
8773 String::new(),
8774 vec!["use fallback".into()],
8775 );
8776 store.put(&mut q).unwrap();
8777 let mut t = task();
8778 t.runs.push(state.id.clone());
8779 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8780 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8781 store.put(&mut q).unwrap();
8782 assert_eq!(land_resume_state(&t), LandResume::Ready);
8783 assert!(!store.get(&q.id).unwrap().status.open());
8784 state.github_text.as_mut().unwrap().resolved = true;
8786 state.save().unwrap();
8787 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8788 }
8789
8790 #[test]
8791 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8792 crate::run::pin_test_home();
8797 let mut state = run_state(RunStatus::Landing);
8798 state.id = "20260101-000000-exp1".to_owned();
8799 state.parked = true;
8800 state.config.graph.answer_timeout = 60;
8801 state.save().unwrap();
8802
8803 let store = ask::Questions::open();
8804 let mut q = approval_question(&state.id);
8805 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8806 store.put(&mut q).unwrap();
8807
8808 let mut t = task();
8809 t.runs.push(state.id.clone());
8810 assert_eq!(
8811 land_resume_state(&t),
8812 LandResume::Ready,
8813 "an expired question must not be waited on forever"
8814 );
8815
8816 let after = store.get(&q.id).unwrap();
8817 assert!(
8818 !after.status.open(),
8819 "the question is abandoned, not silently ignored"
8820 );
8821 assert!(
8822 after.resolution().is_none(),
8823 "an abandoned question is not read as a decision"
8824 );
8825 }
8826
8827 #[test]
8828 fn reclaim_settles_a_running_task_against_its_last_run() {
8829 let mut t = task();
8830 t.start("20260904-000000-4043".to_owned());
8831 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8832 assert_eq!(
8833 t.status,
8834 TaskStatus::Done,
8835 "a run that actually finished must not stay `running` forever"
8836 );
8837 }
8838
8839 #[test]
8840 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8841 let mut t = task();
8842 t.start("20260904-000000-4043".to_owned());
8843 t.last_error = Some("earlier trouble".to_owned());
8844 let mut state = run_state(RunStatus::Judging);
8845 state.parked = true;
8846 reclaim(&mut t, Some(state), 1, "en");
8847 assert_eq!(t.status, TaskStatus::Parked);
8848 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8849 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8850 assert!(t.park_reason.is_some());
8851 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8852 assert!(!t.fresh_start);
8853 assert!(t.status.runnable());
8854 }
8855
8856 #[test]
8857 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8858 let mut t = task();
8859 t.start("20260904-000000-4043".to_owned());
8860 t.last_error = Some("earlier trouble".to_owned());
8861 settle(
8862 &mut t,
8863 Verdict {
8864 status: RunStatus::Judging,
8865 left_pr: false,
8866 quota_hit: false,
8867 parked: true,
8868 no_viable_candidates: false,
8869 },
8870 "parked after `judging`",
8871 1,
8872 );
8873 assert_eq!(t.status, TaskStatus::Parked);
8874 assert_eq!(t.attempts, 0);
8875 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8876 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8877 assert_eq!(t.schema, crate::queue::SCHEMA);
8878 assert!(
8879 crate::notices::task_held(&t).is_none(),
8880 "a park is not news"
8881 );
8882 t.start("20260904-000000-4043".to_owned());
8883 assert_eq!(t.park_reason, None);
8884 }
8885
8886 #[test]
8887 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8888 let mut t = task();
8892 t.start("20260904-000000-4043".to_owned());
8893 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8894 assert_eq!(t.status, TaskStatus::Failed);
8895 assert!(t.status.runnable());
8896 }
8897
8898 #[test]
8899 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8900 let mut t = task();
8901 t.start("20260904-000000-4043".to_owned());
8902 assert_eq!(t.attempts, 1);
8903 let mut run = run_state(RunStatus::Implementing);
8904 run.id = "20260904-000000-4043".to_owned();
8905 run.begin_pass();
8906 let cause = crate::run::ParkCause::Upgrade {
8907 from: "0.121.0".to_owned(),
8908 to: Some("0.121.1".to_owned()),
8909 };
8910 run.end_pass_parked(cause.clone());
8911 run.parked = true;
8912 let verdict = Verdict {
8913 status: run.status,
8914 left_pr: false,
8915 quota_hit: false,
8916 parked: true,
8917 no_viable_candidates: false,
8918 };
8919 let mut orphan = t.clone();
8922 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8923 assert_eq!(orphan.status, TaskStatus::Parked);
8924 assert_eq!(orphan.attempts, 0);
8925 assert_eq!(
8926 orphan.upgrade_resume_key(),
8927 Some((run.passes[0].started_at, orphan.id.as_str()))
8928 );
8929 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8930 assert_eq!(t.attempts, 0);
8931 assert_eq!(t.passes.len(), 1);
8932 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8933 assert_eq!(t.passes[0].parked, Some(cause));
8934 assert_eq!(t.passes[0].pass_index, 0);
8935 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8936 assert!(t.upgrade_resume_key().is_some());
8937
8938 t.start("20260904-000000-4043".to_owned());
8941 run.begin_pass();
8942 run.status = RunStatus::Failed;
8943 let verdict = Verdict {
8944 status: RunStatus::Failed,
8945 left_pr: false,
8946 quota_hit: false,
8947 parked: false,
8948 no_viable_candidates: false,
8949 };
8950 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8951 assert_eq!(t.passes.len(), 2);
8952 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8953 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8954 assert_eq!(t.passes[1].pass_index, 1);
8955 assert_eq!(t.attempts, 1);
8956 }
8957
8958 #[test]
8959 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8960 let mut t = task();
8961 t.start("20260904-000000-4043".to_owned());
8962 reclaim(&mut t, None, 2, "en");
8963 assert_eq!(t.status, TaskStatus::Held);
8964 assert!(
8965 t.last_error
8966 .as_deref()
8967 .is_some_and(|e| e.contains("running")),
8968 "the operator needs to know why this task was held"
8969 );
8970 }
8971
8972 #[test]
8973 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
8974 let dir = tempfile::tempdir().unwrap();
8975 let queue = Queue::at(dir.path().to_path_buf());
8976
8977 let mut orphaned = task();
8979 orphaned.id = "20260904-000000-orph".to_owned();
8980 orphaned.status = TaskStatus::Running;
8981 orphaned.attempts = 1;
8982 queue.put(&mut orphaned).unwrap();
8983
8984 let mut alive = task();
8985 alive.id = "20260904-000000-live".to_owned();
8986 alive.status = TaskStatus::Running;
8987 alive.attempts = 1;
8988 queue.put(&mut alive).unwrap();
8989 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
8990
8991 let mut queued = task();
8992 queued.id = "20260904-000000-wait".to_owned();
8993 queue.put(&mut queued).unwrap();
8994
8995 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8996 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
8997
8998 assert_eq!(
8999 queue.get(&orphaned.id).unwrap().status,
9000 TaskStatus::Held,
9001 "nothing was driving it and there was no run to recover"
9002 );
9003 assert_eq!(
9004 queue.get(&alive.id).unwrap().status,
9005 TaskStatus::Running,
9006 "a live claim must protect the task it belongs to"
9007 );
9008 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9009 }
9010
9011 fn read_run_under(home: &Path, id: &str) -> RunState {
9017 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9018 serde_json::from_str(&body).unwrap()
9019 }
9020
9021 #[test]
9022 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9023 let dir = tempfile::tempdir().unwrap();
9024 let home = dir.path().to_path_buf();
9025 let now = Timestamp::now();
9026 let overrun_seat = || crate::run::ActiveSeat {
9027 node: "implement".to_owned(),
9028 started_at: now - jiff::SignedDuration::new(21_000, 0),
9029 timeout_secs: 3_600,
9030 attempt: 0,
9031 task: None,
9032 command: None,
9033 index: None,
9034 total: None,
9035 };
9036
9037 let mut dead = run_state(RunStatus::Implementing);
9038 dead.id = "20260101-000000-dead".to_owned();
9039 dead.active.insert("impl-A".to_owned(), overrun_seat());
9040 dead.driver_pid = Some(4242);
9043 dead.save_under(&home).unwrap();
9044
9045 let mut alive = run_state(RunStatus::Implementing);
9048 alive.id = "20260101-000000-aliv".to_owned();
9049 alive.active.insert("impl-A".to_owned(), overrun_seat());
9050 alive.save_under(&home).unwrap();
9051 let mut status = Status::new();
9052 status.current = vec![Current {
9053 worker: None,
9054 task: "20260101-000000-task".to_owned(),
9055 run: alive.id.clone(),
9056 }];
9057 write_status_to(&home.join("daemon.json"), &status).unwrap();
9058
9059 let questions = Questions::at(home.join("questions"));
9063 let mut q = ask::Question::new(
9064 dead.id.clone(),
9065 "implement".to_owned(),
9066 "impl-A".to_owned(),
9067 "Which storage backend?".to_owned(),
9068 String::new(),
9069 vec!["SQLite".to_owned(), "Redis".to_owned()],
9070 );
9071 questions.put(&mut q).unwrap();
9072
9073 let abandoned = reclaim_abandoned_runs_with(
9074 &home,
9075 now,
9076 |pid| if pid == 4242 { Some(false) } else { None },
9077 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9078 );
9079 assert_eq!(abandoned, vec![dead.id.clone()]);
9080
9081 let reloaded = read_run_under(&home, &dead.id);
9082 assert_eq!(reloaded.status, RunStatus::Failed);
9083 assert!(reloaded.active.is_empty());
9084 assert!(
9085 !questions.get(&q.id).unwrap().status.open(),
9086 "the failed run's own open question must be settled in the same pass"
9087 );
9088
9089 let still_alive = read_run_under(&home, &alive.id);
9090 assert_eq!(
9091 still_alive.status,
9092 RunStatus::Implementing,
9093 "a live daemon's claim protects it"
9094 );
9095 assert!(!still_alive.active.is_empty());
9096 }
9097
9098 #[test]
9108 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9109 let dir = tempfile::tempdir().unwrap();
9110 let home = dir.path().to_path_buf();
9111 let now = Timestamp::now();
9112
9113 let mut manual = run_state(RunStatus::Reviewing);
9114 manual.id = "20260101-000000-manl".to_owned();
9115 manual.active.insert(
9116 "review-1".to_owned(),
9117 crate::run::ActiveSeat {
9118 node: "review".to_owned(),
9119 started_at: now - jiff::SignedDuration::new(21_000, 0),
9120 timeout_secs: 3_600,
9121 attempt: 0,
9122 task: None,
9123 command: None,
9124 index: None,
9125 total: None,
9126 },
9127 );
9128 manual.driver_pid = Some(4242);
9132 manual.driver_started_at = Some("1790000000".to_owned());
9133 manual.save_under(&home).unwrap();
9134
9135 let abandoned = reclaim_abandoned_runs_with(
9136 &home,
9137 now,
9138 |pid| if pid == 4242 { Some(true) } else { None },
9139 |pid| {
9140 if pid == 4242 {
9141 Some("1790000000".to_owned())
9142 } else {
9143 None
9144 }
9145 },
9146 );
9147 assert!(
9148 abandoned.is_empty(),
9149 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9150 );
9151
9152 let reloaded = read_run_under(&home, &manual.id);
9153 assert_eq!(reloaded.status, RunStatus::Reviewing);
9154 assert!(!reloaded.active.is_empty());
9155 }
9156
9157 #[test]
9158 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9159 let dir = tempfile::tempdir().unwrap();
9160 let queue = Queue::at(dir.path().to_path_buf());
9161 let mut only = task();
9162 queue.put(&mut only).unwrap();
9163
9164 let _elsewhere = queue.claim(&only.id).unwrap();
9165 let candidates = runnable(&queue);
9166 assert_eq!(candidates.len(), 1, "the task is still runnable");
9167 assert!(
9168 queue.claim(&candidates[0].id).is_err(),
9169 "the loop cannot take a claim somebody else holds"
9170 );
9171
9172 let after = queue.get(&only.id).unwrap();
9173 assert_eq!(after.status, TaskStatus::Queued);
9174 assert_eq!(
9175 after.attempts, 0,
9176 "losing the race is not an attempt at the task"
9177 );
9178 assert_eq!(after.last_error, None);
9179 }
9180
9181 #[test]
9182 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9183 let dir = tempfile::tempdir().unwrap();
9184 let path = dir.path().join("daemon.json");
9185
9186 let mut status = Status::new();
9187 status.idle = false;
9188 status.completed = 7;
9189 status.current = vec![Current {
9190 worker: None,
9191 task: "20260902-000000-t111".to_owned(),
9192 run: "20260902-000001-r111".to_owned(),
9193 }];
9194 write_status_to(&path, &status).unwrap();
9195 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9196 assert_eq!(first.schema, SCHEMA);
9197 assert_eq!(first.pid, std::process::id());
9198 assert!(!first.idle);
9199 assert_eq!(first.completed, 7);
9200 assert_eq!(first.current, status.current);
9201 assert!(
9202 !path.with_extension("json.tmp").exists(),
9203 "the temp file is renamed, not left behind"
9204 );
9205
9206 std::thread::sleep(Duration::from_millis(5));
9207 status.updated_at = Timestamp::now();
9208 status.polls = 3;
9209 write_status_to(&path, &status).unwrap();
9210 let second: Status =
9211 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9212 assert!(
9213 second.updated_at > first.updated_at,
9214 "a reader can only detect staleness if the heartbeat moves"
9215 );
9216 assert_eq!(
9217 second.started_at, first.started_at,
9218 "the start time is not a heartbeat"
9219 );
9220 assert_eq!(second.polls, 3);
9221 }
9222
9223 #[test]
9224 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9225 let dir = tempfile::tempdir().unwrap();
9226
9227 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9228
9229 let mut status = Status::new();
9230 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9231 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9232 let stale = read_status(dir.path()).unwrap();
9233 assert!(
9234 !stale.running(Timestamp::now()),
9235 "a minute without a heartbeat is a dead daemon, not a busy one"
9236 );
9237 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9238
9239 status.updated_at = Timestamp::now();
9240 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9241 let fresh = read_status(dir.path()).unwrap();
9242 assert!(fresh.running(Timestamp::now()));
9243 }
9244
9245 #[test]
9246 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9247 let dir = tempfile::tempdir().unwrap();
9248 let now = Timestamp::now();
9249 let mine = "20260903-080619-01c2";
9250
9251 assert!(
9252 !is_working_on(dir.path(), mine, now),
9253 "no status file means nobody is working on anything"
9254 );
9255
9256 let mut status = Status::new();
9257 status.current = vec![Current {
9258 worker: None,
9259 task: "20260903-080340-0167".to_owned(),
9260 run: mine.to_owned(),
9261 }];
9262 status.updated_at = now;
9263 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9264 assert!(is_working_on(dir.path(), mine, now));
9265 assert!(
9266 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9267 "a daemon busy with one run is not working on another"
9268 );
9269
9270 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9273 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9274 assert!(
9275 !is_working_on(dir.path(), mine, now),
9276 "a stale heartbeat is a dead daemon, so its run is a leftover"
9277 );
9278 }
9279
9280 #[test]
9281 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9282 let dir = tempfile::tempdir().unwrap();
9283 let now = Timestamp::now();
9284
9285 assert!(
9286 !is_working_on_short(dir.path(), "01c2", now),
9287 "no status file means nobody is working on anything"
9288 );
9289
9290 let mut status = Status::new();
9291 status.current = vec![Current {
9292 worker: None,
9293 task: "20260903-080340-0167".to_owned(),
9294 run: "20260903-080619-01c2".to_owned(),
9295 }];
9296 status.updated_at = now;
9297 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9298 assert!(
9299 is_working_on_short(dir.path(), "01c2", now),
9300 "the run's short id is the last block of its full id"
9301 );
9302 assert!(
9303 !is_working_on_short(dir.path(), "3cbf", now),
9304 "a daemon busy with one worktree bay is not working on another"
9305 );
9306 }
9307
9308 #[test]
9309 fn a_newer_status_file_still_yields_a_reading() {
9310 let dir = tempfile::tempdir().unwrap();
9311 std::fs::write(
9314 dir.path().join("daemon.json"),
9315 serde_json::json!({
9316 "schema": 2,
9317 "updated_at": Timestamp::now().to_string(),
9318 "idle": true,
9319 "surprise": { "nested": [1, 2, 3] },
9320 })
9321 .to_string(),
9322 )
9323 .unwrap();
9324
9325 let reading = read_status(dir.path()).expect("a forward-compatible read");
9326 assert!(reading.running(Timestamp::now()));
9327 assert!(reading.idle);
9328 assert!(reading.current.is_empty());
9329 }
9330
9331 #[test]
9332 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9333 let dir = tempfile::tempdir().unwrap();
9339 std::fs::write(
9340 dir.path().join("daemon.json"),
9341 serde_json::json!({
9342 "schema": 1,
9343 "pid": 4242,
9344 "updated_at": Timestamp::now().to_string(),
9345 "idle": false,
9346 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9347 "completed": 3,
9348 "polls": 9,
9349 })
9350 .to_string(),
9351 )
9352 .unwrap();
9353
9354 let reading = read_status(dir.path()).expect("an older shape must still parse");
9355 assert!(reading.running(Timestamp::now()));
9356 assert_eq!(
9357 reading.current,
9358 vec![Current {
9359 worker: None,
9360 task: "20260902-140501-aaaa".to_owned(),
9361 run: "20260902-140502-bbbb".to_owned(),
9362 }]
9363 );
9364 }
9365
9366 #[test]
9367 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9368 let dir = tempfile::tempdir().unwrap();
9369 std::fs::write(
9370 dir.path().join("daemon.json"),
9371 serde_json::json!({
9372 "schema": 1,
9373 "updated_at": Timestamp::now().to_string(),
9374 "idle": true,
9375 "current": null,
9376 })
9377 .to_string(),
9378 )
9379 .unwrap();
9380 let with_null = read_status(dir.path()).expect("null must still parse");
9381 assert!(with_null.current.is_empty());
9382
9383 std::fs::write(
9384 dir.path().join("daemon.json"),
9385 serde_json::json!({
9386 "schema": 1,
9387 "updated_at": Timestamp::now().to_string(),
9388 "idle": true,
9389 })
9390 .to_string(),
9391 )
9392 .unwrap();
9393 let absent = read_status(dir.path()).expect("a missing field must still parse");
9394 assert!(absent.current.is_empty());
9395 }
9396
9397 #[test]
9398 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9399 let fallback = Path::new("/default");
9400 let mut blank = task();
9401 blank.repo = PathBuf::new();
9402 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9403 let mut dot = task();
9404 dot.repo = PathBuf::from(".");
9405 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9406 assert_eq!(
9407 repo_for(&task(), fallback),
9408 PathBuf::from("/repo"),
9409 "a task that names a repository keeps it"
9410 );
9411 }
9412
9413 #[test]
9414 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9415 let dir = tempfile::tempdir().unwrap();
9416 let path = dir.path().join("explicit.toml");
9417 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9418 let opts = Opts {
9419 config: Some(path),
9420 merge: Some("none".into()),
9421 ..Default::default()
9422 };
9423 let mut t = task();
9424 t.set_implementers(
9425 2,
9426 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9427 .unwrap(),
9428 )
9429 .unwrap();
9430 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9431 assert_eq!(cfg.graph.implementers, 2);
9432 assert_eq!(cfg.graph.judges, 1);
9433 assert_eq!(cfg.merge.mode, MergeMode::None);
9434 t.set_implementers(1, &cfg).unwrap();
9435 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9436 apply_task_implementers(&mut cfg, &t);
9437 assert_eq!(cfg.graph.implementers, 1);
9438 assert_eq!(cfg.graph.judges, 1);
9439 assert_eq!(cfg.merge.mode, MergeMode::None);
9440 }
9441
9442 #[test]
9443 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9444 let dir = tempfile::tempdir().unwrap();
9445 let repo = dir.path().join("repo");
9446 std::fs::create_dir(&repo).unwrap();
9447 std::fs::write(
9448 repo.join("magi.toml"),
9449 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9450 )
9451 .unwrap();
9452 let daemon_config = dir.path().join("daemon.toml");
9453 std::fs::write(
9454 &daemon_config,
9455 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9456 )
9457 .unwrap();
9458 let opts = Opts {
9459 config: Some(daemon_config),
9460 merge: Some("pr".into()),
9461 ..Default::default()
9462 };
9463 let q = Queue::at(dir.path().join("queue"));
9464 let validation: Config =
9465 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9466 .unwrap();
9467 for record in ["{}", r#"{"candidates":2}"#] {
9469 let overrides = serde_json::from_str(record).unwrap();
9470 let filing = crate::direct::Filing {
9471 instruction: "work".into(),
9472 title: "direct".into(),
9473 repo: repo.clone(),
9474 source: crate::queue::Source::Human,
9475 solo: false,
9476 overrides,
9477 review_of: None,
9478 sources: Default::default(),
9479 };
9480 let filed =
9481 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9482 let crate::direct::Filed::Standalone {
9483 mut task,
9484 claim: _claim,
9485 } = filed
9486 else {
9487 panic!("local filing")
9488 };
9489 let before = prepare_for(&repo, &opts, &task).unwrap();
9490 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9491 assert_eq!(
9492 before.merge.mode,
9493 MergeMode::None,
9494 "daemon --merge must not leak"
9495 );
9496 for n in [2, 1, 2] {
9497 task.set_implementers(n, &validation).unwrap();
9498 q.put(&mut task).unwrap();
9499 task = q.get(&task.id).unwrap();
9500 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9501 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9502 apply_task_implementers(&mut cfg, &task);
9503 assert_eq!(cfg.graph.implementers, usize::from(n));
9504 assert_eq!(cfg.graph.judges, 2);
9505 assert_eq!(cfg.merge.mode, MergeMode::None);
9506 }
9507 }
9508 }
9509
9510 #[test]
9511 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9512 let cfg: Config =
9513 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9514 .unwrap();
9515 let mut t = task();
9516 t.review_of = Some("review-target".into());
9517 t.hold_manual(Some("retry".into()));
9518 let mut previous = RunState::new(
9519 PathBuf::from("/repo"),
9520 "main".into(),
9521 "abc1234".into(),
9522 "review".into(),
9523 cfg.clone(),
9524 );
9525 previous.config.graph.implementers = 2;
9526 for count in [1, 3] {
9527 t.set_implementers(count, &cfg).unwrap();
9528 t.release();
9529 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9530 for proposed in [
9531 Starter::Resume(previous.id.clone()),
9532 Starter::Review("recovery-target".into()),
9533 ] {
9534 assert_eq!(
9535 starter_with_implementers(&t, Some(&previous), proposed),
9536 Starter::Review("review-target".into())
9537 );
9538 }
9539 let mut configured = cfg.clone();
9540 apply_task_implementers(&mut configured, &t);
9541 assert_eq!(configured.graph.implementers, usize::from(count));
9542 assert_eq!(previous.config.graph.implementers, 2);
9543 }
9544 }
9545
9546 #[test]
9547 fn task_implementers_dispatch_and_resume_agree() {
9548 let mut t = task();
9549 let cfg: crate::config::Config =
9550 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9551 .unwrap();
9552 let mut run = RunState::new(
9553 PathBuf::from("/repo"),
9554 "main".into(),
9555 "abc1234".into(),
9556 "work".into(),
9557 cfg.clone(),
9558 );
9559 run.config.graph.implementers = 2;
9560 for n in [2, 1, 3, 2] {
9561 t.set_implementers(n, &cfg).unwrap();
9562 let mut dispatched = cfg.clone();
9563 apply_task_implementers(&mut dispatched, &t);
9564 assert_eq!(dispatched.graph.implementers, usize::from(n));
9565 assert_eq!(implementers_match(&t, &run), n == 2);
9566 for proposed in [
9567 Starter::Resume(run.id.clone()),
9568 Starter::Review("old-branch".into()),
9569 ] {
9570 let expected = if n == 2 {
9571 proposed.clone()
9572 } else {
9573 Starter::Start
9574 };
9575 assert_eq!(
9576 starter_with_implementers(&t, Some(&run), proposed),
9577 expected
9578 );
9579 }
9580 assert_eq!(
9581 run.config.graph.implementers, 2,
9582 "history must not be rewritten"
9583 );
9584 }
9585 t.solo = false;
9586 t.overrides = None;
9587 assert!(implementers_match(&t, &run));
9588 }
9589
9590 #[test]
9591 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9592 let mut solo_cfg = Config::default();
9598 solo_cfg.graph.implementers = 3;
9599 let mut solo_task = task();
9600 solo_task.solo = true;
9601 apply_task_implementers(&mut solo_cfg, &solo_task);
9602 assert_eq!(solo_cfg.graph.implementers, 1);
9603
9604 let mut plain_cfg = Config::default();
9605 plain_cfg.graph.implementers = 3;
9606 let plain_task = task();
9607 assert!(!plain_task.solo);
9608 apply_task_implementers(&mut plain_cfg, &plain_task);
9609 assert_eq!(
9610 plain_cfg.graph.implementers, 3,
9611 "a task that did not ask to run alone keeps the config's candidates"
9612 );
9613 }
9614
9615 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9616 QuotaLoss {
9617 seat: seat.into(),
9618 node: "judge".into(),
9619 at: at.parse().unwrap(),
9620 reset: reset.map(str::to_string),
9621 }
9622 }
9623
9624 #[test]
9625 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
9626 let old: Vec<QuotaLoss> = (1..=4)
9627 .map(|i| {
9628 loss(
9629 &format!("judge-{i}"),
9630 "2026-09-23T05:23:00Z",
9631 Some("2:40pm (Asia/Tokyo)"),
9632 )
9633 })
9634 .collect();
9635 let fresh = losses_this_attempt(&old, &old);
9636 assert!(fresh.is_empty());
9637 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
9638 }
9640
9641 #[test]
9642 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
9643 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
9644 let now = Timestamp::now();
9645 let mut after = old.clone();
9646 after.push(loss("judge-2", &now.to_string(), None));
9647 let fresh = losses_this_attempt(&old, &after);
9648 assert_eq!(fresh, vec![after[1].clone()]);
9649 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
9650 assert_eq!(
9651 until,
9652 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
9653 );
9654 }
9655
9656 #[test]
9657 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
9658 let before = vec![
9661 loss("judge-1", "2026-09-23T05:23:00Z", None),
9662 loss("judge-2", "2026-09-23T05:24:00Z", None),
9663 ];
9664 let after = vec![
9665 loss("judge-2", "2026-09-23T05:24:00Z", None),
9666 loss("judge-1", "2026-09-24T01:00:00Z", None),
9667 ];
9668 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
9669 }
9670
9671 #[test]
9672 fn merge_overrides_are_parsed_or_refused() {
9673 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
9674 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
9675 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
9676 assert!(merge_mode("squash").is_err());
9677 }
9678
9679 #[test]
9680 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
9681 let now = Timestamp::now();
9682 let fallback = Duration::from_secs(300);
9683 let cap = Duration::from_secs(1800);
9684
9685 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
9687
9688 let soon = now + jiff::SignedDuration::from_secs(600);
9690 assert_eq!(
9691 quota_wait(Some(soon), now, fallback, cap),
9692 Duration::from_secs(600)
9693 );
9694
9695 let past = now - jiff::SignedDuration::from_secs(60);
9698 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
9699
9700 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
9703 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
9704 }
9705
9706 #[test]
9707 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9708 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9709
9710 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9711 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9712
9713 let already_past =
9717 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9718 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9719
9720 assert!(
9721 parse_reset_hint("session limit reached", now, now).is_none(),
9722 "free text with no recognised shape is not guessed at"
9723 );
9724 assert!(
9725 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9726 "an unresolvable zone name is not guessed at either"
9727 );
9728 }
9729
9730 #[test]
9731 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9732 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9733
9734 let at = parse_reset_hint(
9735 "You've hit your usage limit. Visit \
9736 https://chatgpt.com/codex/settings/usage to purchase more \
9737 credits or try again at Sep 19th, 2026 5:10 PM.",
9738 now,
9739 now,
9740 )
9741 .expect("the codex reset wording is a recognised shape");
9742 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9743
9744 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9749 .expect("an explicit year needs no rollover");
9750 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9751
9752 assert!(
9753 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9754 "a two-digit year is not the documented shape and is not guessed at"
9755 );
9756 assert!(
9757 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9758 "a four-letter month name is not the documented three-letter abbreviation"
9759 );
9760 assert!(
9761 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9762 "an explicit zone on the dated shape is a format nobody has \
9763 documented, and is refused rather than guessed at as UTC"
9764 );
9765 }
9766
9767 #[test]
9768 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9769 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9770 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9771
9772 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9773 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9774
9775 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9776 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9777
9778 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9779 assert!(
9780 parse_reset_hint(bad, now, recorded).is_none(),
9781 "{bad:?} must not be guessed at"
9782 );
9783 }
9784 }
9785
9786 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9790 let config = dir.join("magi.toml");
9791 std::fs::write(
9792 &config,
9793 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9794 )
9795 .unwrap();
9796 let opts = Opts {
9797 poll: Duration::from_secs(30),
9798 config: Some(config),
9799 repo: dir.join("repo"),
9803 ..Opts::default()
9804 };
9805 let home = dir.join("home");
9814 let stamps = home.join("service-ticks");
9817 std::fs::create_dir_all(&stamps).unwrap();
9818 for service in ["waiter", "deputies", "release-watch"] {
9819 std::fs::write(
9820 stamps.join(service),
9821 Timestamp::now().as_second().to_string(),
9822 )
9823 .unwrap();
9824 }
9825 for operation in [
9826 "conductor",
9827 "maintenance",
9828 "housekeeping",
9829 "disk-pressure",
9830 "cache",
9831 "fetch",
9832 ] {
9833 std::fs::write(
9834 stamps.join(format!("daemon-{operation}")),
9835 serde_json::to_vec(&Timestamp::now()).unwrap(),
9836 )
9837 .unwrap();
9838 }
9839 let worktrees = dir.join("wt");
9840 (
9841 opts,
9842 Queue::at(dir.join("queue")),
9843 home.join("daemon.json"),
9844 home,
9845 worktrees,
9846 )
9847 }
9848
9849 #[test]
9850 fn a_stop_is_idempotent_and_once_set_stays_set() {
9851 let stop = Stop::new();
9852 assert!(!stop.stopped());
9853
9854 stop.stop();
9855 assert!(stop.stopped());
9856 stop.stop();
9857 assert!(stop.stopped(), "a second stop is not a toggle");
9858
9859 let shared = stop.clone();
9860 assert!(
9861 shared.stopped(),
9862 "a clone is the same stop; that is how the loop and its caller share one"
9863 );
9864 }
9865
9866 #[test]
9867 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9868 let stop = Stop::new();
9869 stop.busy.store(1, Ordering::SeqCst);
9870 assert!(
9871 !stop.finishing(),
9872 "a busy loop nobody has asked to stop is just running"
9873 );
9874
9875 stop.stop();
9876 assert!(
9877 stop.finishing(),
9878 "a stop asked for mid-run has not landed until the run is settled"
9879 );
9880
9881 stop.busy.store(0, Ordering::SeqCst);
9882 assert!(
9883 !stop.finishing(),
9884 "once the run is settled the stop has landed and there is nothing to finish"
9885 );
9886 }
9887
9888 #[test]
9889 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9890 let stop = Stop::new();
9891 stop.busy.store(2, Ordering::SeqCst);
9892 stop.stop();
9893 assert!(stop.finishing(), "two runs still in flight");
9894
9895 stop.busy.store(1, Ordering::SeqCst);
9896 assert!(
9897 stop.finishing(),
9898 "one run finished, but a sibling is still working"
9899 );
9900
9901 stop.busy.store(0, Ordering::SeqCst);
9902 assert!(
9903 !stop.finishing(),
9904 "the last run out is what actually lands the stop"
9905 );
9906 }
9907
9908 #[tokio::test]
9909 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9910 let dir = tempfile::tempdir().unwrap();
9911 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9912 let stop = Stop::new();
9913 stop.stop();
9914
9915 let began = std::time::Instant::now();
9916 tokio::time::timeout(
9917 Duration::from_secs(2),
9918 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9919 )
9920 .await
9921 .expect("a stopped loop must return, not sit out its poll interval")
9922 .expect("the loop's own setup and teardown must not fail");
9923 assert!(
9924 began.elapsed() < opts.poll,
9925 "returned only after {:?}, which is a poll interval, not a stop",
9926 began.elapsed()
9927 );
9928 }
9929
9930 #[tokio::test]
9931 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9932 let dir = tempfile::tempdir().unwrap();
9933 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9934 let stop = Stop::new();
9935
9936 let asker = {
9939 let stop = stop.clone();
9940 tokio::spawn(async move {
9941 tokio::time::sleep(Duration::from_millis(20)).await;
9942 stop.stop();
9943 })
9944 };
9945
9946 let began = std::time::Instant::now();
9947 tokio::time::timeout(
9948 Duration::from_secs(2),
9949 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9950 )
9951 .await
9952 .expect("a stop asked for while idle must wake the wait")
9953 .expect("the loop's own setup and teardown must not fail");
9954 asker.await.unwrap();
9955 assert!(
9956 began.elapsed() < opts.poll,
9957 "returned only after {:?}, so the stop waited on the sleep",
9958 began.elapsed()
9959 );
9960 }
9961
9962 #[tokio::test]
9963 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9964 let dir = tempfile::tempdir().unwrap();
9965 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9966 let stop = Stop::new();
9967 stop.stop();
9968
9969 tokio::time::timeout(
9970 Duration::from_secs(2),
9971 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9972 )
9973 .await
9974 .expect("a stopped loop must return")
9975 .expect("the loop's own setup and teardown must not fail");
9976
9977 assert!(
9978 home.is_dir(),
9979 "the loop did publish a status file, so its removal is the teardown and not an absence"
9980 );
9981 assert!(
9982 !status_file.exists(),
9983 "a stopped loop clears its status file"
9984 );
9985 assert!(
9986 read_status(&home).is_none(),
9987 "a reader must see no daemon at all, not a heartbeat that merely stopped"
9988 );
9989 }
9990
9991 #[tokio::test]
9992 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
9993 let dir = tempfile::tempdir().unwrap();
9994 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
9995
9996 let mut settled = RunState::new(
9997 dir.path().join("repo"),
9998 "main".to_owned(),
9999 "abc1234".to_owned(),
10000 "fixture".to_owned(),
10001 Config::default(),
10002 );
10003 settled.status = RunStatus::Ready;
10004 let run_dir = home.join("runs").join(&settled.id);
10005 std::fs::create_dir_all(&run_dir).unwrap();
10006 std::fs::write(
10007 run_dir.join("run.json"),
10008 serde_json::to_string_pretty(&settled).unwrap(),
10009 )
10010 .unwrap();
10011 let questions = Questions::at(home.join("questions"));
10012 let mut question = ask::Question::new(
10013 settled.id.clone(),
10014 "review".to_owned(),
10015 "reviewer-1".to_owned(),
10016 "Continue?".to_owned(),
10017 String::new(),
10018 Vec::new(),
10019 );
10020 questions.put(&mut question).unwrap();
10021
10022 run_service_worker(
10023 &home,
10024 serde_json::to_value(ServiceWorker {
10025 opts,
10026 queue: queue.root().to_path_buf(),
10027 worktrees_root: worktrees,
10028 operation: "housekeeping".to_owned(),
10029 })
10030 .unwrap(),
10031 )
10032 .await
10033 .unwrap();
10034
10035 assert_eq!(
10036 questions.get(&question.id).unwrap().status,
10037 ask::QuestionStatus::Abandoned,
10038 "the independent housekeeping worker settles ended-run questions"
10039 );
10040 }
10041
10042 #[test]
10043 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10044 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10045
10046 assert!(
10047 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10048 "never checked before: due at once"
10049 );
10050
10051 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10052 assert!(
10053 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10054 "well inside the interval: not due yet"
10055 );
10056
10057 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10058 assert!(
10059 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10060 "exactly at the edge: not yet due, same convention as `clean::due`"
10061 );
10062
10063 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10064 assert!(
10065 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10066 "past the interval: due again"
10067 );
10068 }
10069
10070 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10075 let config = dir.join("magi.toml");
10076 std::fs::write(
10082 &config,
10083 format!(
10084 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10085 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10086 cache_dir.display()
10087 ),
10088 )
10089 .unwrap();
10090 Opts {
10091 config: Some(config),
10092 repo: dir.join("repo"),
10093 ..Opts::default()
10094 }
10095 }
10096
10097 #[tokio::test]
10098 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10099 let dir = tempfile::tempdir().unwrap();
10100 let home = dir.path().join("home");
10101 let cache_dir = dir.path().join("cache");
10102 std::fs::create_dir_all(&cache_dir).unwrap();
10103 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10104 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10105
10106 let running = Stop::new();
10109 let mut last_checked = None;
10110 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10111 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10112 .await;
10113 assert_eq!(
10114 crate::disk::dir_size(&cache_dir),
10115 0,
10116 "over the cap on the first check ever: pruned at once, no idle queue required"
10117 );
10118 assert_eq!(last_checked, Some(t0));
10119
10120 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10122 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10123 maybe_prune_cache_between_runs(
10124 &opts.repo,
10125 &opts,
10126 &home,
10127 &running,
10128 &mut last_checked,
10129 too_soon,
10130 )
10131 .await;
10132 assert_eq!(
10133 crate::disk::dir_size(&cache_dir),
10134 10,
10135 "too soon since the last check: left alone rather than rescanned every call"
10136 );
10137 assert_eq!(
10138 last_checked,
10139 Some(t0),
10140 "an idle check does not reset the clock"
10141 );
10142
10143 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10145 maybe_prune_cache_between_runs(
10146 &opts.repo,
10147 &opts,
10148 &home,
10149 &running,
10150 &mut last_checked,
10151 due_again,
10152 )
10153 .await;
10154 assert_eq!(
10155 crate::disk::dir_size(&cache_dir),
10156 0,
10157 "due again: pruned back under the cap"
10158 );
10159 }
10160
10161 #[tokio::test]
10169 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
10170 let dir = tempfile::tempdir().unwrap();
10171 let home = dir.path().join("home");
10172 let cache_dir = dir.path().join("cache");
10173 std::fs::create_dir_all(&cache_dir).unwrap();
10174 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10175 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10176
10177 let stop = Stop::new();
10178 stop.stop();
10179 assert!(
10180 !stop.finishing(),
10181 "no run is in flight at a between-runs boundary, so nothing else \
10182 would tell the operator this stop had not taken effect yet"
10183 );
10184
10185 let mut last_checked = None;
10186 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10187 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10188 .await;
10189 assert_eq!(
10190 crate::disk::dir_size(&cache_dir),
10191 10,
10192 "over its cap, and due for the first check ever, but a stop outranks \
10193 it: the cap is a standing policy the next start measures again"
10194 );
10195 assert_eq!(
10196 last_checked, None,
10197 "a check that never happened must not claim the interval"
10198 );
10199 }
10200
10201 #[tokio::test]
10204 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10205 let dir = tempfile::tempdir().unwrap();
10206 let cache_dir = dir.path().join("cache");
10207 std::fs::create_dir_all(&cache_dir).unwrap();
10208 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10209
10210 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10211 opts.poll = Duration::from_millis(20);
10212 opts.max_attempts = 1_000;
10213
10214 let queue = Queue::at(dir.path().join("queue"));
10215 for i in 0..128 {
10216 let mut task = Task::new(
10217 format!("queued work {i}"),
10218 format!("queued work {i}"),
10219 opts.repo.clone(),
10220 Source::Human,
10221 );
10222 queue.put(&mut task).unwrap();
10223 }
10224
10225 let home = dir.path().join("home");
10226 let worktrees = dir.path().join("wt");
10227 run_service_worker(
10228 &home,
10229 serde_json::to_value(ServiceWorker {
10230 opts,
10231 queue: queue.root().to_path_buf(),
10232 worktrees_root: worktrees,
10233 operation: "cache".to_owned(),
10234 })
10235 .unwrap(),
10236 )
10237 .await
10238 .unwrap();
10239 let after = queue.list();
10240 assert_eq!(after.len(), 128);
10241 assert!(
10242 after
10243 .iter()
10244 .all(|task| task.status.runnable() && task.attempts == 0)
10245 );
10246
10247 assert_eq!(
10248 crate::disk::dir_size(&cache_dir),
10249 0,
10250 "an oversized cache must not be left to grow unboundedly just because the \
10251 queue kept the loop busy the whole time"
10252 );
10253 }
10254
10255 #[test]
10256 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10257 let dir = tempfile::tempdir().unwrap();
10258 let queue = Queue::at(dir.path().join("queue"));
10259 let questions = Questions::at(dir.path().join("questions"));
10260 let mut task = task();
10261 queue.put(&mut task).unwrap();
10262
10263 let mut task_question = ask::Question::new(
10264 task.id.clone(),
10265 crate::conduct::NODE.to_owned(),
10266 "conduct".to_owned(),
10267 "Which backend?".to_owned(),
10268 String::new(),
10269 Vec::new(),
10270 );
10271 questions.put(&mut task_question).unwrap();
10272 task.block(vec![task_question.id.clone()], None);
10273 queue.put(&mut task).unwrap();
10274
10275 let mut run_question = ask::Question::new(
10276 "20260101-000000-run1".to_owned(),
10277 "review".to_owned(),
10278 "reviewer-1".to_owned(),
10279 "Run question".to_owned(),
10280 String::new(),
10281 Vec::new(),
10282 );
10283 questions.put(&mut run_question).unwrap();
10284
10285 let mut coincidental = ask::Question::new(
10290 task.id.clone(),
10291 "review".to_owned(),
10292 "reviewer-1".to_owned(),
10293 "Unrelated review question".to_owned(),
10294 String::new(),
10295 Vec::new(),
10296 );
10297 questions.put(&mut coincidental).unwrap();
10298
10299 reconcile_task_questions(&queue, &questions);
10300 assert!(questions.get(&task_question.id).unwrap().status.open());
10301 assert!(questions.get(&run_question.id).unwrap().status.open());
10302 assert!(questions.get(&coincidental.id).unwrap().status.open());
10303
10304 task.release();
10305 queue.put(&mut task).unwrap();
10306 reconcile_task_questions(&queue, &questions);
10307 assert_eq!(
10308 questions.get(&task_question.id).unwrap().status,
10309 ask::QuestionStatus::Abandoned
10310 );
10311 assert!(
10312 questions.get(&run_question.id).unwrap().status.open(),
10313 "run questions remain the run janitor's responsibility"
10314 );
10315 assert!(
10316 questions.get(&coincidental.id).unwrap().status.open(),
10317 "a non-conductor question must not be abandoned just because its \
10318 run id coincides with a task id"
10319 );
10320 }
10321
10322 #[test]
10323 fn a_freshly_started_running_task_is_never_stalled() {
10324 let dir = tempfile::tempdir().unwrap();
10325 let mut t = task();
10326 t.start("run-1".to_owned());
10327 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10330 }
10331
10332 #[test]
10333 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10334 let dir = tempfile::tempdir().unwrap();
10335 let mut t = task();
10336 t.start("run-1".to_owned());
10337 t.updated_at = Timestamp::now()
10338 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10339 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10340 assert_eq!(
10341 stalled_tasks(
10342 &Queue::at(dir.path().join("q")),
10343 dir.path(),
10344 Timestamp::now()
10345 )
10346 .len(),
10347 0,
10348 "the task was never written to this queue"
10349 );
10350 }
10351
10352 #[test]
10353 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10354 let dir = tempfile::tempdir().unwrap();
10355 let mut t = task();
10356 t.id = "20260903-080340-0167".to_owned();
10357 t.start("20260903-080619-01c2".to_owned());
10358 t.updated_at = Timestamp::now()
10359 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10360
10361 let mut status = Status::new();
10362 status.current = vec![Current {
10363 worker: None,
10364 task: t.id.clone(),
10365 run: "20260903-080619-01c2".to_owned(),
10366 }];
10367 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10368
10369 assert!(
10370 !is_stalled(&t, dir.path(), Timestamp::now()),
10371 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10372 );
10373 }
10374
10375 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10379 let path = queue.path_of(id);
10380 let body = std::fs::read_to_string(&path).unwrap();
10381 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10382 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10383 v["updated_at"] = serde_json::Value::String(old.to_string());
10384 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10385 }
10386
10387 #[test]
10388 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10389 let dir = tempfile::tempdir().unwrap();
10402 let queue = Queue::at(dir.path().join("queue"));
10403 let home = dir.path().join("home");
10404
10405 let mut t = task();
10406 t.id = "20260101-000001-lock".to_owned();
10407 t.start("run-1".to_owned());
10408 queue.put(&mut t).unwrap();
10409 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10410 std::fs::write(
10411 dir.path().join("queue").join(format!("{}.lock", t.id)),
10412 "not a pid",
10413 )
10414 .unwrap();
10415
10416 let now = Timestamp::now();
10417 assert!(
10418 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10419 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10420 and reclaim must leave the task alone"
10421 );
10422 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10423
10424 let stalled = stalled_tasks(&queue, &home, now);
10425 assert_eq!(
10426 stalled.len(),
10427 1,
10428 "reclaim's inability to claim it yet must not hide it from the conductor"
10429 );
10430 assert_eq!(stalled[0].id, t.id);
10431 }
10432
10433 #[test]
10434 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10435 let dir = tempfile::tempdir().unwrap();
10436 crate::run::set_home(dir.path().join("run-home"));
10437 let queue = Queue::at(dir.path().join("queue"));
10438 let home = dir.path().join("home");
10439 let questions = Questions::at(dir.path().join("questions"));
10440
10441 let mut t = task();
10442 t.id = "20260101-000003-dead".to_owned();
10443 t.start("missing-run".to_owned());
10444 queue.put(&mut t).unwrap();
10445 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10446
10447 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10450 assert_eq!(
10451 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10452 [&t.id]
10453 );
10454 assert_eq!(
10455 reclaim_orphaned_running(&queue, 2, &Default::default()),
10456 [t.id.clone()]
10457 );
10458 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10459
10460 crate::conduct::apply(
10463 &queue,
10464 &questions,
10465 &crate::conduct::Verdict {
10466 decisions: vec![crate::conduct::Decision {
10467 id: t.id.clone(),
10468 recovery: Some(crate::conduct::Recovery::Requeue),
10469 ..crate::conduct::Decision::default()
10470 }],
10471 },
10472 )
10473 .unwrap();
10474 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10475 }
10476
10477 #[test]
10478 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10479 let dir = tempfile::tempdir().unwrap();
10480 let queue = Queue::at(dir.path().join("queue"));
10481 let home = dir.path().join("home");
10482
10483 let mut fresh = task();
10484 fresh.id = "20260101-000001-aaaa".to_owned();
10485 fresh.start("run-1".to_owned());
10486 queue.put(&mut fresh).unwrap();
10487
10488 let mut old = task();
10489 old.id = "20260101-000002-bbbb".to_owned();
10490 old.start("run-2".to_owned());
10491 queue.put(&mut old).unwrap();
10492 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10493
10494 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10495 assert_eq!(stalled.len(), 1);
10496 assert_eq!(stalled[0].id, old.id);
10497 }
10498
10499 #[test]
10500 fn queued_and_finished_task_views_partition_by_status() {
10501 let dir = tempfile::tempdir().unwrap();
10502 let queue = Queue::at(dir.path().join("queue"));
10503
10504 let mut queued = task();
10505 queued.id = "20260101-000001-aaaa".to_owned();
10506 queue.put(&mut queued).unwrap();
10507
10508 let mut failed = task();
10509 failed.id = "20260101-000002-bbbb".to_owned();
10510 failed.start("run-1".to_owned());
10511 failed.fail("gate red", 5);
10512 queue.put(&mut failed).unwrap();
10513
10514 let mut held = task();
10515 held.id = "20260101-000003-cccc".to_owned();
10516 held.hold_machine(None);
10517 queue.put(&mut held).unwrap();
10518
10519 let mut running = task();
10520 running.id = "20260101-000004-dddd".to_owned();
10521 running.start("run-2".to_owned());
10522 queue.put(&mut running).unwrap();
10523
10524 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
10525 assert_eq!(queued_ids, [queued.id.clone()]);
10526
10527 let mut finished_ids: Vec<String> =
10528 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
10529 finished_ids.sort_unstable();
10530 let mut want = vec![failed.id.clone(), held.id.clone()];
10531 want.sort_unstable();
10532 assert_eq!(finished_ids, want);
10533 }
10534
10535 #[test]
10536 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
10537 for short in [false, true] {
10538 let dir = tempfile::tempdir().unwrap();
10539 let queue = Queue::at(dir.path().join("queue"));
10540 let questions = ask::Questions::at(dir.path().join("questions"));
10541 let mut running = task();
10542 running.id = "20261013-192213-feb2".into();
10543 running.start("existing-run".into());
10544 queue.put(&mut running).unwrap();
10545 let mut middle = task();
10546 middle.id = "20261013-193611-ca5b".into();
10547 middle.block(
10548 vec![if short {
10549 "feb2".into()
10550 } else {
10551 running.id.clone()
10552 }],
10553 None,
10554 );
10555 queue.put(&mut middle).unwrap();
10556 let mut waiting = task();
10557 waiting.id = "20261013-202849-ee37".into();
10558 waiting.block(
10559 vec![if short {
10560 "ca5b".into()
10561 } else {
10562 middle.id.clone()
10563 }],
10564 Some("wait for implementation".into()),
10565 );
10566 queue.put(&mut waiting).unwrap();
10567 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
10568 let inv = crate::blockers::Inventory::load(&queue, &questions);
10569 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
10570 assert!(inv.stuck().is_empty());
10571 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10572 assert!(report.quarantined.is_empty());
10573 assert!(report.asked.is_empty());
10574 resolve_blockers(&queue, &questions);
10575 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10576 assert!(questions.list().is_empty());
10577 running.succeed();
10580 queue.put(&mut running).unwrap();
10581 resolve_blockers(&queue, &questions);
10582 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
10583 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10584 middle = queue.get(&middle.id).unwrap();
10585 middle.succeed();
10586 queue.put(&mut middle).unwrap();
10587 resolve_blockers(&queue, &questions);
10588 let after = queue.get(&waiting.id).unwrap();
10589 assert_eq!(after.status, TaskStatus::Queued);
10590 assert!(after.blocked_by.is_empty());
10591 assert_eq!(after.instruction, waiting.instruction);
10592 }
10593 }
10594
10595 #[test]
10596 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
10597 for future_schema in [false, true] {
10598 let name = "20261013-193611-ca5b";
10599 let mut future = task();
10600 future.id = name.into();
10601 future.schema = crate::queue::SCHEMA + 1;
10602 let contents = if future_schema {
10603 serde_json::to_string(&future).unwrap()
10604 } else {
10605 "corrupt".into()
10606 };
10607 let dir = tempfile::tempdir().unwrap();
10608 let queue = Queue::at(dir.path().join("queue"));
10609 let questions = ask::Questions::at(dir.path().join("questions"));
10610 let mut waiting = task();
10611 waiting.id = "20261013-202849-ee37".into();
10612 waiting.block(vec!["ca5b".into()], None);
10613 queue.put(&mut waiting).unwrap();
10614 std::fs::write(queue.path_of(name), contents).unwrap();
10615 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
10616 assert!(
10617 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
10618 );
10619 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10620 assert!(report.asked.is_empty());
10621 resolve_blockers(&queue, &questions);
10622 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10623 assert!(questions.list().is_empty());
10624 }
10625 let dir = tempfile::tempdir().unwrap();
10626 let queue = Queue::at(dir.path().join("queue"));
10627 let questions = ask::Questions::at(dir.path().join("questions"));
10628 let mut waiting = task();
10629 waiting.id = "20261013-202849-ee37".into();
10630 waiting.block(vec!["ca5b".into()], None);
10631 queue.put(&mut waiting).unwrap();
10632 let mut done = task();
10633 done.id = "20261013-193611-ca5b".into();
10634 done.succeed();
10635 queue.put(&mut done).unwrap();
10636 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
10639 assert!(queue.get("ca5b").is_err());
10640 assert_eq!(
10641 crate::queue::dependency(&queue, &questions, "ca5b"),
10642 crate::queue::Dependency::Ambiguous
10643 );
10644 resolve_blockers(&queue, &questions);
10645 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10646 assert!(report.asked.is_empty());
10647 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10648 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
10649 }
10650
10651 #[test]
10652 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
10653 for held in [false, true] {
10654 for question in [false, true] {
10655 for reference in ["ca5b", "20261013-193611-ca5b"] {
10656 let dir = tempfile::tempdir().unwrap();
10657 let queue = Queue::at(dir.path().join("queue"));
10658 let questions = ask::Questions::at(dir.path().join("questions"));
10659 let mut prerequisite = task();
10660 prerequisite.id = "20261013-193611-ca5b".into();
10661 prerequisite.start("existing-run".into());
10662 queue.put(&mut prerequisite).unwrap();
10663 let mut waiting = task();
10664 waiting.id = "20261013-202849-ee37".into();
10665 if held {
10666 waiting.hold_machine(Some("waiting for separate consent".into()));
10667 }
10668 queue.put(&mut waiting).unwrap();
10669 let reason = "Deleting saved data requires separate owner consent";
10670 crate::conduct::apply(
10671 &queue,
10672 &questions,
10673 &crate::conduct::Verdict {
10674 decisions: vec![crate::conduct::Decision {
10675 id: waiting.id.clone(),
10676 blocked_by: vec![reference.into()],
10677 question: question.then(|| "May the saved data be deleted?".into()),
10678 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
10679 reason: Some(reason.into()),
10680 ..crate::conduct::Decision::default()
10681 }],
10682 },
10683 )
10684 .unwrap();
10685 let before = queue.get(&waiting.id).unwrap();
10686 if question {
10687 assert_eq!(before.status, TaskStatus::Blocked);
10688 let q = questions.get(&before.blocked_by[0]).unwrap();
10689 assert_eq!(q.summary, "May the saved data be deleted?");
10690 assert!(q.status.open());
10691 } else {
10692 assert_eq!(before.status, TaskStatus::Held);
10693 let recorded_reason = before.hold_reason.as_deref().unwrap();
10694 assert!(recorded_reason.starts_with(reason));
10695 if held {
10696 assert!(recorded_reason.contains("waiting for separate consent"));
10697 }
10698 }
10699 prerequisite.succeed();
10700 queue.put(&mut prerequisite).unwrap();
10701 resolve_blockers(&queue, &questions);
10702 let after = queue.get(&waiting.id).unwrap();
10703 assert_eq!(after.status, before.status);
10704 assert_eq!(after.blocked_by, before.blocked_by);
10705 assert_eq!(after.hold_reason, before.hold_reason);
10706 assert_eq!(after.instruction, waiting.instruction);
10707 if question {
10708 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10709 }
10710 assert!(
10711 queue.next_runnable().is_none(),
10712 "dependency completion is not consent"
10713 );
10714 }
10715 }
10716 }
10717 }
10718
10719 #[test]
10720 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10721 let dir = tempfile::tempdir().unwrap();
10722 let queue = Queue::at(dir.path().join("queue"));
10723 let questions = ask::Questions::at(dir.path().join("questions"));
10724
10725 let mut dep = task();
10726 dep.id = "20260101-000001-dep0".to_owned();
10727 dep.succeed();
10728 queue.put(&mut dep).unwrap();
10729
10730 let mut still_going = task();
10731 still_going.id = "20260101-000002-dep1".to_owned();
10732 queue.put(&mut still_going).unwrap();
10733
10734 let mut blocked = task();
10735 blocked.id = "20260101-000003-main".to_owned();
10736 blocked.block(
10737 vec![dep.id.clone(), still_going.id.clone()],
10738 Some("waits on both".to_owned()),
10739 );
10740 queue.put(&mut blocked).unwrap();
10741
10742 resolve_blockers(&queue, &questions);
10743
10744 let after = queue.get(&blocked.id).unwrap();
10745 assert_eq!(
10746 after.status,
10747 TaskStatus::Blocked,
10748 "one dependency is still outstanding"
10749 );
10750 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10751 }
10752
10753 #[test]
10754 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10755 let dir = tempfile::tempdir().unwrap();
10756 let queue = Queue::at(dir.path().join("queue"));
10757 let questions = ask::Questions::at(dir.path().join("questions"));
10758
10759 let mut q = crate::ask::Question::new(
10760 "20260101-000001-main".to_owned(),
10761 crate::conduct::NODE.to_owned(),
10762 "conduct".to_owned(),
10763 "Which backend?".to_owned(),
10764 String::new(),
10765 Vec::new(),
10766 );
10767 questions.put(&mut q).unwrap();
10768 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10769 .unwrap();
10770 questions.put(&mut q).unwrap();
10771
10772 let mut blocked = task();
10773 blocked.id = "20260101-000001-main".to_owned();
10774 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10775 queue.put(&mut blocked).unwrap();
10776
10777 resolve_blockers(&queue, &questions);
10778
10779 let after = queue.get(&blocked.id).unwrap();
10780 assert_eq!(
10781 after.status,
10782 TaskStatus::Queued,
10783 "the only blocker resolved"
10784 );
10785 assert_eq!(after.answers.len(), 1);
10786 assert_eq!(after.answers[0].question, "Which backend?");
10787 assert_eq!(after.answers[0].answer, "SQLite");
10788
10789 let instruction = instruction_for(&after);
10791 assert!(instruction.contains("Which backend?"));
10792 assert!(instruction.contains("SQLite"));
10793 }
10794
10795 #[test]
10796 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10797 let dir = tempfile::tempdir().unwrap();
10798 let queue = Queue::at(dir.path().join("queue"));
10799 let questions = ask::Questions::at(dir.path().join("questions"));
10800
10801 let mut q = crate::ask::Question::new(
10802 "20260101-000001-main".to_owned(),
10803 crate::conduct::NODE.to_owned(),
10804 "conduct".to_owned(),
10805 "Is the setup done?".to_owned(),
10806 String::new(),
10807 Vec::new(),
10808 );
10809 q.abandon("no answer within 60s of asking");
10810 questions.put(&mut q).unwrap();
10811
10812 let mut blocked = task();
10813 blocked.id = "20260101-000001-main".to_owned();
10814 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10815 queue.put(&mut blocked).unwrap();
10816
10817 resolve_blockers(&queue, &questions);
10818
10819 let after = queue.get(&blocked.id).unwrap();
10820 assert_eq!(
10821 after.status,
10822 TaskStatus::Held,
10823 "never left blocked on nothing"
10824 );
10825 assert!(!after.operator_held(), "a machine hold, for triage");
10826 assert!(
10827 after
10828 .hold_reason
10829 .as_deref()
10830 .unwrap_or_default()
10831 .contains("went unanswered")
10832 );
10833 }
10834
10835 #[test]
10836 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10837 let dir = tempfile::tempdir().unwrap();
10843 let queue = Queue::at(dir.path().join("queue"));
10844 let questions = ask::Questions::at(dir.path().join("questions"));
10845
10846 let mut q = crate::ask::Question::new(
10847 "20260101-000001-main".to_owned(),
10848 crate::conduct::NODE.to_owned(),
10849 "conduct".to_owned(),
10850 "How should this be handled?".to_owned(),
10851 String::new(),
10852 Vec::new(),
10853 );
10854 questions.put(&mut q).unwrap();
10855 q.answer(crate::ask::Answer::Text(
10856 "leave it held, a human will look at it later".to_owned(),
10857 ))
10858 .unwrap();
10859 questions.put(&mut q).unwrap();
10860
10861 let mut held = task();
10862 held.id = "20260101-000001-main".to_owned();
10863 held.hold_machine(Some("out of attempts".to_owned()));
10864 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10865 queue.put(&mut held).unwrap();
10866
10867 resolve_blockers(&queue, &questions);
10868
10869 let after = queue.get(&held.id).unwrap();
10870 assert_eq!(after.status, TaskStatus::Held);
10871 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10872 assert_eq!(
10873 after.answers[0].answer,
10874 "leave it held, a human will look at it later"
10875 );
10876 }
10877
10878 #[test]
10879 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10880 let dir = tempfile::tempdir().unwrap();
10881 let queue = Queue::at(dir.path().join("queue"));
10882 let questions = ask::Questions::at(dir.path().join("questions"));
10883 let mut dep = task();
10884 dep.id = "20260101-000001-gone".to_owned();
10885 queue.put(&mut dep).unwrap();
10886 let mut blocked = task();
10887 blocked.id = "20260101-000003-main".to_owned();
10888 blocked.block(vec![dep.id.clone()], None);
10889 queue.put(&mut blocked).unwrap();
10890
10891 let claim = queue.claim(&blocked.id).unwrap();
10893 queue.remove(&dep.id, false, &questions).unwrap();
10894 drop(claim);
10895 resolve_blockers(&queue, &questions);
10896
10897 let after = queue.get(&blocked.id).unwrap();
10898 assert_eq!(after.status, TaskStatus::Queued);
10899 assert!(after.blocked_by.is_empty());
10900 }
10901
10902 #[test]
10903 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10904 let dir = tempfile::tempdir().unwrap();
10910 let queue = Queue::at(dir.path().join("queue"));
10911 let questions = ask::Questions::at(dir.path().join("questions"));
10912
10913 let mut still_going = task();
10914 still_going.id = "20260101-000002-dep1".to_owned();
10915 queue.put(&mut still_going).unwrap();
10916
10917 let mut blocked = task();
10918 blocked.id = "20260101-000003-main".to_owned();
10919 blocked.block(
10920 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10921 Some("waits on both".to_owned()),
10922 );
10923 queue.put(&mut blocked).unwrap();
10924
10925 resolve_blockers(&queue, &questions);
10926
10927 let after = queue.get(&blocked.id).unwrap();
10928 assert_eq!(
10929 after.status,
10930 TaskStatus::Held,
10931 "a missing dependency must not leave the task blocked forever"
10932 );
10933 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10934 assert!(after.blocked_by.is_empty());
10935 let reason = after.hold_reason.as_deref().unwrap_or_default();
10936 assert!(
10937 reason.contains("20260101-000001-gone"),
10938 "the missing id must be named so an operator can tell what happened: {reason}"
10939 );
10940 assert!(
10941 reason.contains(&still_going.id),
10942 "the still-valid dependency must not silently vanish from the record: {reason}"
10943 );
10944 }
10945
10946 #[test]
10947 fn instruction_for_is_unchanged_without_any_answers() {
10948 let t = task();
10949 assert_eq!(instruction_for(&t), t.instruction);
10950 }
10951
10952 #[test]
10953 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10954 let dir = tempfile::tempdir().unwrap();
10955 let q = Queue::at(dir.path().join("queue"));
10956 let src = dir.path().join("shot.png");
10957 std::fs::write(&src, "x").unwrap();
10958 let mut t = task();
10959 q.attach(&mut t, &[src]).unwrap();
10960 let paths = task_attachments(&q, &t).unwrap();
10961 assert_eq!(paths.len(), 1);
10962 assert!(paths[0].is_absolute() && paths[0].is_file());
10963 std::fs::remove_file(&paths[0]).unwrap();
10964 let err = task_attachments(&q, &t).unwrap_err().to_string();
10965 assert!(err.contains("shot.png"), "{err}");
10966 }
10967
10968 #[tokio::test]
10969 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
10970 let dir = tempfile::tempdir().unwrap();
10971 let queue_path = dir.path().join("queue");
10972 let queue = Queue::at(queue_path.clone());
10973 let mut t = task();
10974 t.source = Source::Agent {
10975 run: "originating-chat".into(),
10976 node: crate::queue::CHAT_NODE.into(),
10977 };
10978 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
10979 stage: "config-fetch",
10980 target: "origin/main".into(),
10981 });
10982 record_config_failure(&mut t, &fetch);
10983 t.hold_machine(Some(format!("{fetch:#}")));
10984 let fetch_key = t.chat_notice_due().unwrap();
10985 t.chat_notices.push(fetch_key.clone());
10986 queue.put(&mut t).unwrap();
10987
10988 let queue = Queue::at(queue_path);
10989 let mut t = queue.get(&t.id).unwrap();
10990 let a = dir.path().join("a.toml");
10991 let b = dir.path().join("b.toml");
10992 for (path, text, new_cause) in [
10993 (&a, "[graph", true),
10994 (&a, "[verify", false),
10995 (&a, "[graph]\nimplementers = 'invalid-type'", true),
10996 (&b, "[graph]\nimplementers = 'invalid-type'", true),
10997 ] {
10998 t.release_operator("test", None);
10999 std::fs::write(path, text).unwrap();
11000 let error = Config::discover_fetched(dir.path(), Some(path))
11001 .await
11002 .unwrap_err();
11003 record_config_failure(&mut t, &error);
11004 t.hold_machine(Some(format!("{error:#}")));
11005 let due = t.chat_notice_due();
11006 assert_eq!(due.is_some(), new_cause, "{error:#}");
11007 if let Some(key) = due {
11008 assert_ne!(key, fetch_key);
11009 t.chat_notices.push(key);
11010 }
11011 }
11012 t.succeed();
11013 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11014 }
11015
11016 #[test]
11017 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11018 let dir = tempfile::tempdir().unwrap();
11019 let queue_path = dir.path().join("queue");
11020 let queue = Queue::at(queue_path.clone());
11021 let mut t = task();
11022 t.source = crate::queue::Source::Agent {
11023 run: "originating-chat".into(),
11024 node: crate::queue::CHAT_NODE.into(),
11025 };
11026 let a = dir.path().join("a.png");
11027 let b = dir.path().join("b.png");
11028 std::fs::write(&a, "a").unwrap();
11029 std::fs::write(&b, "b").unwrap();
11030 queue.attach(&mut t, &[a, b]).unwrap();
11031 let paths = task_attachments(&queue, &t).unwrap();
11032 for path in &paths {
11033 std::fs::remove_file(path).unwrap();
11034 }
11035 let missing = task_attachments(&queue, &t).unwrap_err();
11036 t.attempts += 1;
11037 let reason = missing.to_string();
11038 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11039 let first = t.chat_notice_due().expect("first attachment notice");
11040 t.chat_notices.push(first.clone());
11041 queue.put(&mut t).unwrap();
11042
11043 let queue = Queue::at(queue_path.clone());
11044 let mut t = queue.get(&t.id).unwrap();
11045 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11046 t.release_operator("test", None);
11047 t.attempts += 1;
11048 let missing = task_attachments(&queue, &t).unwrap_err();
11049 t.fail_startup_target(
11050 "startup-attachments",
11051 missing.name,
11052 "same missing attachment with regenerated wording",
11053 1,
11054 );
11055 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11056
11057 std::fs::write(&paths[0], "restored").unwrap();
11058 t.release_operator("test", None);
11059 t.attempts += 1;
11060 let missing = task_attachments(&queue, &t).unwrap_err();
11061 let reason = missing.to_string();
11062 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11063 let second = t
11064 .chat_notice_due()
11065 .expect("different attachment must notify");
11066 assert_ne!(first, second);
11067 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11068 }
11069
11070 #[test]
11071 fn resumed_instruction_is_unchanged_without_any_answers() {
11072 let t = task();
11073 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11074 }
11075
11076 #[test]
11077 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11078 let mut t = task();
11079 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11080 let old = t.instruction.clone();
11084
11085 let refreshed = resumed_instruction(&old, &t);
11086 assert!(refreshed.starts_with(&old), "the original text is kept");
11087 assert!(refreshed.contains("Which backend?"));
11088 assert!(refreshed.contains("SQLite"));
11089 }
11090
11091 #[test]
11092 fn resumed_instruction_keeps_an_original_answers_heading() {
11093 let mut t = task();
11094 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11095 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11096
11097 let refreshed = resumed_instruction(&t.instruction, &t);
11098
11099 assert!(
11100 refreshed.starts_with(&t.instruction),
11101 "an answers heading in the original instruction is not the appended block"
11102 );
11103 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11104 assert!(refreshed.contains("Which backend?"));
11105 assert!(refreshed.contains("SQLite"));
11106
11107 let repeated = resumed_instruction(&refreshed, &t);
11108 assert_eq!(
11109 repeated, refreshed,
11110 "only the final appended block is refreshed"
11111 );
11112 }
11113
11114 #[test]
11115 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11116 let mut t = task();
11117 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11118
11119 let once = resumed_instruction(&t.instruction, &t);
11123 let twice = resumed_instruction(&once, &t);
11124 assert_eq!(once, twice);
11125 assert_eq!(once.matches("Which backend?").count(), 1);
11126
11127 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
11129 let refreshed = resumed_instruction(&once, &t);
11130 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
11131 assert!(refreshed.contains("Which backend?"));
11132 assert!(refreshed.contains("Which cache?"));
11133 }
11134
11135 #[test]
11136 fn prepare_instruction_covers_all_three_starters() {
11137 let mut t = task();
11138 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11139
11140 assert_eq!(
11143 prepare_instruction(&Starter::Start, None, &t),
11144 Some(instruction_for(&t))
11145 );
11146
11147 let old = t.instruction.clone();
11150 assert_eq!(
11151 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
11152 Some(resumed_instruction(&old, &t))
11153 );
11154
11155 assert_eq!(
11159 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
11160 None
11161 );
11162 }
11163
11164 #[test]
11165 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
11166 assert_eq!(
11167 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
11168 Starter::Review("magi/eba2/A".to_owned())
11169 );
11170 }
11171
11172 #[test]
11173 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
11174 assert_eq!(
11175 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
11176 Starter::Start,
11177 "a vanished review branch must not fall back to resuming the old run either"
11178 );
11179 }
11180
11181 #[test]
11182 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11183 let mut t = task();
11184 t.start("old-run".to_owned());
11185 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11186 t.release();
11187 let branch = t.review_branch.take();
11188 assert_eq!(
11189 choose_starter(branch.as_deref(), true, Some("old-run")),
11190 Starter::Review("magi/eba2/A".to_owned()),
11191 "a review wins over resuming the old run"
11192 );
11193 }
11194
11195 #[test]
11196 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11197 assert_eq!(
11198 choose_starter(None, false, Some("some-run")),
11199 Starter::Resume("some-run".to_owned())
11200 );
11201 assert_eq!(choose_starter(None, false, None), Starter::Start);
11202 }
11203
11204 #[test]
11205 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11206 let mut released = task();
11207 released.start("stalled-run".to_owned());
11208 released.requeue();
11209 let unfinished = (!released.fresh_start)
11210 .then(|| Some("stalled-run".to_owned()))
11211 .flatten();
11212 assert_eq!(
11213 choose_starter(None, false, unfinished.as_deref()),
11214 Starter::Start,
11215 "release keeps run history but must not resume it"
11216 );
11217 assert_eq!(released.runs, ["stalled-run"]);
11218 }
11219
11220 fn stopped_public_run(status: RunStatus) -> RunState {
11221 let mut state = run_state(status);
11222 state.driver_exited = true;
11223 state.config.graph.land = true;
11224 state.config.graph.review_rounds = 1;
11225 state.reviews = vec![review_round(1)];
11226 state.merge = Some(crate::run::MergeOutcome {
11227 mode: MergeMode::Pr,
11228 ok: false,
11229 detail: "landing stopped".into(),
11230 empty: false,
11231 });
11232 state.pr = Some(crate::run::PrRecord {
11233 url: "https://example.test/pull/1".into(),
11234 number: 1,
11235 state: "open".into(),
11236 checks: "red".into(),
11237 round: 1,
11238 rounds: 1,
11239 red_at_merge: Vec::new(),
11240 });
11241 state
11242 }
11243
11244 #[test]
11245 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11246 for status in [RunStatus::Blocked, RunStatus::Gating] {
11247 let home = tempfile::tempdir().unwrap();
11248 let mut state = stopped_public_run(status);
11249 state.save_under(home.path()).unwrap();
11250 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11251 assert!(!exhausted_review_budget(&state));
11252 let mut task = held_task_with(&state.id);
11253 task.release();
11254 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11255 assert_eq!(
11256 choose_starter(None, false, unfinished.as_deref()),
11257 Starter::Resume(state.id.clone())
11258 );
11259 let question = action_question(&state.id, resume_action(&state.id));
11260 assert_eq!(
11261 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11262 ActionDecision::Resume(state.id.clone())
11263 );
11264 assert_eq!(
11266 crate::resume::validate(home.path(), &state.id).unwrap().id,
11267 state.id
11268 );
11269 let queue = Queue::at(home.path().join("queue"));
11270 let mut controlled = Task::new(
11271 "resume land".into(),
11272 state.instruction.clone(),
11273 state.repo.clone(),
11274 Source::Agent {
11275 run: "talk".into(),
11276 node: crate::queue::CHAT_NODE.into(),
11277 },
11278 );
11279 controlled.runs.push(state.id.clone());
11280 controlled.hold_machine(Some("land stopped".into()));
11281 queue.put(&mut controlled).unwrap();
11282 let resumed =
11283 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11284 assert_eq!(resumed.status, TaskStatus::Queued);
11285 assert_eq!(
11286 resumed.resume_override.unwrap().pinned_run,
11287 Some(state.id.clone())
11288 );
11289 assert_eq!(resumed.attempts, 0);
11290 }
11291 }
11292
11293 #[test]
11294 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11295 for review_rounds in [0, 1] {
11296 let home = tempfile::tempdir().unwrap();
11297 let mut state = run_state(RunStatus::Blocked);
11298 state.config.graph.review_rounds = review_rounds;
11299 if review_rounds > 0 {
11300 let mut review = review_round(1);
11301 review.clean = true;
11302 state.reviews = vec![review];
11303 }
11304 state.gate_ran = true;
11305 state.gate = vec![crate::run::CommandOutcome {
11306 command: "verify".into(),
11307 code: Some(1),
11308 output_tail: "gate failed after repair budget exhausted".into(),
11309 duration_ms: 0,
11310 resource_blocked: false,
11311 }];
11312 state.save_under(home.path()).unwrap();
11313 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11314 assert!(exhausted_review_budget(&state));
11315 let mut task = held_task_with(&state.id);
11316 task.release();
11317 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11318 let question = action_question(&state.id, resume_action(&state.id));
11319 assert_ne!(
11320 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11321 ActionDecision::Resume(state.id.clone())
11322 );
11323 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11324
11325 state.gate_ran = false;
11327 state.gate.clear();
11328 assert!(!exhausted_review_budget(&state));
11329 assert_eq!(
11330 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11331 Some(state.id)
11332 );
11333 }
11334 }
11335
11336 #[test]
11337 fn resource_blocked_verification_resumes_the_same_run() {
11338 for (kind, stage) in [
11339 ("cache", "clean"),
11340 ("coordination", "clean"),
11341 ("cache", "review-e2e"),
11342 ("coordination", "no-reviews"),
11343 ("coordination", "handoff"),
11344 ] {
11345 let home = tempfile::tempdir().unwrap();
11346 let mut state = run_state(RunStatus::Blocked);
11347 state.config.graph.review_rounds = 1;
11348 let mut review = review_round(1);
11349 review.clean = true;
11350 state.reviews = vec![review];
11351 if stage == "review-e2e" {
11352 let review = &mut state.reviews[0];
11353 review.clean = false;
11354 review.blocking = 1;
11355 review.e2e = vec![crate::run::CommandOutcome {
11356 command: "verify".into(),
11357 code: None,
11358 output_tail: "resource wait expired".into(),
11359 duration_ms: 0,
11360 resource_blocked: true,
11361 }];
11362 } else if stage == "no-reviews" {
11363 state.config.graph.review_rounds = 0;
11364 state.reviews.clear();
11365 } else if stage == "handoff" {
11366 let review = &mut state.reviews[0];
11369 review.clean = false;
11370 review.blocking = 1;
11371 review.e2e = vec![crate::run::CommandOutcome {
11372 command: "verify".into(),
11373 code: Some(0),
11374 output_tail: String::new(),
11375 duration_ms: 0,
11376 resource_blocked: false,
11377 }];
11378 }
11379 state.gate_ran = false;
11380 state.resource_block = Some(crate::run::ResourceBlock {
11381 cause: crate::run::ResourceBlockCause::Other,
11382 node: if stage == "review-e2e" {
11383 "review"
11384 } else {
11385 "gate"
11386 }
11387 .into(),
11388 kind: kind.into(),
11389 detail: "resource wait expired".into(),
11390 at: jiff::Timestamp::now(),
11391 });
11392 state.save_under(home.path()).unwrap();
11393 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11394 assert!(!exhausted_review_budget(&state));
11395 let mut task = held_task_with(&state.id);
11396 task.release();
11397 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11398 assert_eq!(
11399 choose_starter(None, false, unfinished.as_deref()),
11400 Starter::Resume(state.id.clone())
11401 );
11402 let question = action_question(&state.id, resume_action(&state.id));
11403 assert_eq!(
11404 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11405 ActionDecision::Resume(state.id.clone())
11406 );
11407 assert_eq!(
11408 crate::resume::validate(home.path(), &state.id).unwrap().id,
11409 state.id
11410 );
11411 let queue = Queue::at(home.path().join("queue"));
11412 let mut controlled = Task::new(
11413 "resume verification".into(),
11414 state.instruction.clone(),
11415 state.repo.clone(),
11416 Source::Agent {
11417 run: "talk".into(),
11418 node: crate::queue::CHAT_NODE.into(),
11419 },
11420 );
11421 controlled.runs.push(state.id.clone());
11422 controlled.hold_machine(Some("verification resource unavailable".into()));
11423 queue.put(&mut controlled).unwrap();
11424 let resumed =
11425 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11426 assert_eq!(resumed.status, TaskStatus::Queued);
11427 assert_eq!(
11428 resumed.resume_override.unwrap().pinned_run,
11429 Some(state.id.clone())
11430 );
11431 assert_eq!(resumed.attempts, 0);
11432
11433 state.config.graph.review_rounds = 1;
11435 let mut unresolved = review_round(1);
11436 unresolved.blocking = 1;
11437 state.reviews = vec![unresolved];
11438 state.resource_block = None;
11439 state.save_under(home.path()).unwrap();
11440 assert!(exhausted_review_budget(&state));
11441 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11442 }
11443 }
11444
11445 #[tokio::test]
11446 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11447 if !crate::standalone_test(
11448 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
11449 ) {
11450 return;
11451 }
11452 let home = tempfile::tempdir().unwrap();
11453 let mut owner = stopped_public_run(RunStatus::Blocked);
11454 owner.save_under(home.path()).unwrap();
11455 let history = vec![owner.id.clone()];
11456 let repo = owner.repo.clone();
11457 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11458 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11459 let initial =
11460 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11461 assert!(
11462 initial.is_pending(),
11463 "a busy record must not hold or resume the task"
11464 );
11465 owner.retired_to = Some("20260101-000001-bbbb".into());
11466 owner.save_under(home.path()).unwrap();
11467 drop(lock);
11468 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11469 .await
11470 .expect("ownership lock released")
11471 .expect("read settled owner");
11472 assert!(
11473 result.is_none(),
11474 "the retired owner must not be resumed from a stale read"
11475 );
11476 }
11477
11478 #[tokio::test]
11479 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11480 if !crate::standalone_test(
11481 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
11482 ) {
11483 return;
11484 }
11485 let home = tempfile::tempdir().unwrap();
11486 let mut owner = stopped_public_run(RunStatus::Blocked);
11487 let repo = tempfile::tempdir().unwrap();
11488 crate::git::git(repo.path(), &["init"]).await.unwrap();
11489 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11490 owner.id = "20260101-000000-aaaa".into();
11491 owner.save_under(home.path()).unwrap();
11492 let mut successor = run_state(RunStatus::Blocked);
11493 successor.id = "20260101-000001-bbbb".into();
11494 successor.save_under(home.path()).unwrap();
11495 let runs = vec![owner.id.clone(), successor.id.clone()];
11496 assert_eq!(
11497 public_resume_owner(home.path(), &runs, &owner.repo)
11498 .await
11499 .unwrap()
11500 .unwrap()
11501 .id,
11502 owner.id
11503 );
11504 let subdir = owner.repo.join("nested");
11505 std::fs::create_dir(&subdir).unwrap();
11506 assert_eq!(
11507 public_resume_owner(home.path(), &runs, &subdir)
11508 .await
11509 .unwrap()
11510 .unwrap()
11511 .id,
11512 owner.id
11513 );
11514 #[cfg(unix)]
11515 {
11516 let alias = home.path().join("repo-alias");
11517 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
11518 assert_eq!(
11519 public_resume_owner(home.path(), &runs, &alias.join("nested"))
11520 .await
11521 .unwrap()
11522 .unwrap()
11523 .id,
11524 owner.id
11525 );
11526 }
11527 let different = tempfile::tempdir().unwrap();
11528 crate::git::git(different.path(), &["init"]).await.unwrap();
11529 assert!(
11530 public_resume_owner(home.path(), &runs, different.path())
11531 .await
11532 .is_err()
11533 );
11534 std::fs::remove_file(
11535 home.path()
11536 .join("runs")
11537 .join(&successor.id)
11538 .join("run.json"),
11539 )
11540 .unwrap();
11541 assert!(
11542 public_resume_owner(home.path(), &runs, &owner.repo)
11543 .await
11544 .is_err()
11545 );
11546 }
11547
11548 #[test]
11549 fn an_ordinary_release_keeps_a_resumable_run_available() {
11550 let mut released = task();
11551 released.start("stalled-run".to_owned());
11552 released.release();
11553 let unfinished = (!released.fresh_start)
11554 .then(|| Some("stalled-run".to_owned()))
11555 .flatten();
11556 assert_eq!(
11557 choose_starter(None, false, unfinished.as_deref()),
11558 Starter::Resume("stalled-run".to_owned()),
11559 "manual release must preserve the normal resume path"
11560 );
11561 }
11562
11563 #[test]
11564 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
11565 let mut state = run_state(RunStatus::Blocked);
11566 state.config.graph.review_rounds = 3;
11567 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
11568 assert!(exhausted_review_budget(&state));
11569
11570 state.reviews.pop();
11572 assert!(!exhausted_review_budget(&state));
11573
11574 let mut stalled = run_state(RunStatus::Stalled);
11577 stalled.config.graph.review_rounds = 1;
11578 stalled.reviews = vec![review_round(1)];
11579 assert!(!exhausted_review_budget(&stalled));
11580 }
11581
11582 fn review_round(round: usize) -> crate::run::ReviewRound {
11583 crate::run::ReviewRound {
11584 round,
11585 head: "deadbeef".to_owned(),
11586 verified_head: None,
11587 verified_at: None,
11588 reviews: Vec::new(),
11589 e2e: Vec::new(),
11590 verify_retried: false,
11591 e2e_deferred: false,
11592 e2e_defer_reason: None,
11593 fix: None,
11594 blocking: 0,
11595 answered: 1,
11596 expected: 1,
11597 clean: false,
11598 progressed: true,
11599 vote_split: false,
11600 reconsideration: Vec::new(),
11601 verdict: None,
11602 }
11603 }
11604
11605 #[test]
11606 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
11607 let mut run = RunState::new(
11608 PathBuf::from("/repo"),
11609 "main".to_owned(),
11610 "abc1234def".to_owned(),
11611 "add retries".to_owned(),
11612 Config::default(),
11613 );
11614 run.status = RunStatus::Judging;
11615 run.parked = true;
11616 let mut task = Task::new(
11617 "add retries".to_owned(),
11618 "add retries".to_owned(),
11619 PathBuf::from("/repo"),
11620 crate::queue::Source::Human,
11621 );
11622 task.status = TaskStatus::Failed;
11623 task.runs = vec![run.id.clone()];
11624 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
11625 assert!(with(&task, &run), "parked after judging is the case");
11626 let mut parked_task = task.clone();
11627 parked_task.status = TaskStatus::Parked;
11628 assert!(with(&parked_task, &run), "the Parked status is the case");
11629
11630 let mut not_parked = run.clone();
11631 not_parked.parked = false;
11632 not_parked.status = RunStatus::Stalled;
11633 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
11634
11635 let mut fresh = task.clone();
11636 fresh.fresh_start = true;
11637 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
11638
11639 let mut review = task.clone();
11640 review.review_branch = Some("magi/x/A".to_owned());
11641 assert!(!with(&review, &run), "review is ranked before resume");
11642
11643 let mut held = task.clone();
11644 held.status = TaskStatus::Held;
11645 assert!(!with(&held, &run), "a hold stays visible to the conductor");
11646
11647 let mut released = run.clone();
11648 released.released_to = Some("20260901-000000-new1".to_owned());
11649 assert!(!with(&task, &released), "nothing left to resume into");
11650
11651 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
11652 "unreadable"
11653 )));
11654 }
11655
11656 #[test]
11657 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
11658 let mut released = RunState::new(
11659 PathBuf::from("/repo"),
11660 "main".to_owned(),
11661 "abc1234def".to_owned(),
11662 "add retries".to_owned(),
11663 Config::default(),
11664 );
11665 released.status = RunStatus::Blocked;
11666 assert_eq!(
11667 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11668 Some(released.id.clone())
11669 );
11670 released.released_to = Some("20260901-000000-new1".to_owned());
11671 assert_eq!(
11672 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11673 None,
11674 "there is nothing left to resume it into"
11675 );
11676 }
11677
11678 #[test]
11679 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
11680 let mut exhausted = RunState::new(
11689 PathBuf::from("/repo"),
11690 "main".to_owned(),
11691 "abc1234def".to_owned(),
11692 "add retries".to_owned(),
11693 Config::default(),
11694 );
11695 exhausted.status = RunStatus::Blocked;
11696 exhausted.config.graph.review_rounds = 1;
11697 exhausted.reviews = vec![review_round(1)];
11698
11699 assert_eq!(
11700 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
11701 None,
11702 "an exhausted `Blocked` run must not be offered as resumable"
11703 );
11704
11705 let mut has_budget_left = RunState::new(
11708 PathBuf::from("/repo"),
11709 "main".to_owned(),
11710 "abc1234def".to_owned(),
11711 "add retries".to_owned(),
11712 Config::default(),
11713 );
11714 has_budget_left.status = RunStatus::Blocked;
11715 has_budget_left.config.graph.review_rounds = 3;
11716 has_budget_left.reviews = vec![review_round(1)];
11717
11718 assert_eq!(
11719 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
11720 Ok(has_budget_left.clone())
11721 }),
11722 Some(has_budget_left.id.clone())
11723 );
11724 }
11725
11726 #[test]
11727 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
11728 let mut run = RunState::new(
11729 PathBuf::from("/repo"),
11730 "main".to_owned(),
11731 "abc1234def".to_owned(),
11732 "add retries".to_owned(),
11733 Config::default(),
11734 );
11735 run.status = RunStatus::Blocked;
11736 run.config.graph.review_rounds = 1;
11737 run.reviews = vec![review_round(1)];
11738 run.pr_publication_pending = true;
11739 assert_eq!(
11740 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11741 Some(run.id.clone()),
11742 "a spent review budget must not turn a failed publication into a re-competition"
11743 );
11744 }
11745
11746 #[test]
11747 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11748 let mut task = Task::new(
11749 "recover work".into(),
11750 "recover work".into(),
11751 PathBuf::from("/repo"),
11752 crate::queue::Source::Human,
11753 );
11754 let mut history = std::collections::BTreeMap::new();
11755 for index in 0..8 {
11756 let mut run = run_state(RunStatus::Blocked);
11757 run.id = format!("20260101-000000-{index:04}");
11758 run.base_commit = format!("upstream-{index}");
11759 if index > 0 {
11761 run.candidates.clear();
11762 }
11763 task.runs.extend([run.id.clone(), run.id.clone()]);
11764 history.insert(run.id.clone(), run);
11765 task.requeue();
11766 }
11767 let check = |task: &Task, starter: &Starter, limit| {
11768 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11769 };
11770 assert!(
11771 check(&task, &Starter::Start, 8)
11772 .unwrap()
11773 .contains("8 unretired")
11774 );
11775 assert!(check(&task, &Starter::Start, 9).is_none());
11776 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11777 task.release_operator("owner", None);
11778 assert_eq!(task.attempts, 0);
11779 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11780 assert!(check(&restored, &Starter::Start, 8).is_some());
11781 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11782 assert!(check(&restored, &Starter::Start, 0).is_none());
11783 }
11784
11785 #[test]
11786 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11787 let mut task = Task::new(
11788 "recover work".into(),
11789 "recover work".into(),
11790 PathBuf::from("/repo"),
11791 crate::queue::Source::Human,
11792 );
11793 let mut run = run_state(RunStatus::Superseded);
11794 run.candidates.clear();
11795 task.runs.push(run.id.clone());
11796 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11797 run.retired_to = Some("authority".into());
11798 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11799 run.retired_to = None;
11800 run.released_to = Some("authority".into());
11801 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11802 assert!(
11803 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11804 .unwrap()
11805 .contains("Cannot verify")
11806 );
11807 assert!(
11809 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11810 panic!("resume must not load historical attempts")
11811 })
11812 .is_none()
11813 );
11814 }
11815
11816 #[test]
11817 fn retired_resume_requires_a_valid_same_repository_authority() {
11818 let mut source = run_state(RunStatus::Blocked);
11819 let mut empty = run_state(RunStatus::Blocked);
11820 empty.retired_to = Some(source.id.clone());
11821 let runs = vec![source.id.clone(), empty.id.clone()];
11822 let resolve = |source: &RunState, empty: &RunState| {
11823 unfinished_run_with(&runs, "fixture", |id| {
11824 if id == source.id {
11825 Ok(source.clone())
11826 } else if id == empty.id {
11827 Ok(empty.clone())
11828 } else {
11829 bail!("unreadable authority")
11830 }
11831 })
11832 };
11833 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11834 assert_eq!(
11835 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11836 Starter::Resume(source.id.clone())
11837 );
11838 source.retired_to = Some(empty.id.clone());
11839 assert!(
11840 resolve(&source, &empty).is_none(),
11841 "cycles do not revive obsolete runs"
11842 );
11843 source.retired_to = None;
11844 source.repo = PathBuf::from("/another-repository");
11845 assert!(resolve(&source, &empty).is_none());
11846 source.repo = empty.repo.clone();
11847 source.released_to = Some("later".into());
11848 assert!(resolve(&source, &empty).is_none());
11849 empty.retired_to = Some("unlisted-run".into());
11850 assert!(resolve(&source, &empty).is_none());
11851 }
11852
11853 #[test]
11854 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11855 let mut older_stalled = RunState::new(
11863 PathBuf::from("/repo"),
11864 "main".to_owned(),
11865 "abc1234def".to_owned(),
11866 "add retries".to_owned(),
11867 Config::default(),
11868 );
11869 older_stalled.status = RunStatus::Stalled;
11870
11871 let mut newest_exhausted = RunState::new(
11872 PathBuf::from("/repo"),
11873 "main".to_owned(),
11874 "abc1234def".to_owned(),
11875 "add retries".to_owned(),
11876 Config::default(),
11877 );
11878 newest_exhausted.status = RunStatus::Blocked;
11879 newest_exhausted.config.graph.review_rounds = 1;
11880 newest_exhausted.reviews = vec![review_round(1)];
11881
11882 assert_eq!(
11883 unfinished_run_with(
11884 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11885 "t",
11886 |_| Ok(newest_exhausted.clone())
11887 ),
11888 None,
11889 "the newest run is exhausted, so nothing here is worth resuming - \
11890 least of all the older, already-superseded run"
11891 );
11892 }
11893
11894 #[test]
11895 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11896 assert_eq!(
11897 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11898 Err(anyhow::anyhow!("fixture is absent"))
11899 }),
11900 None
11901 );
11902 }
11903
11904 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11905 let mut q = ask::Question::new(
11906 run.to_owned(),
11907 "implement".to_owned(),
11908 "impl-A".to_owned(),
11909 "continue?".to_owned(),
11910 String::new(),
11911 vec!["resume で続行する".to_owned(), "other".to_owned()],
11912 );
11913 q.actions.insert("resume で続行する".to_owned(), action);
11914 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11915 .unwrap();
11916 q
11917 }
11918
11919 fn held_task_with(run: &str) -> Task {
11920 let mut t = task();
11921 t.runs = vec![run.to_owned()];
11922 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11923 t
11924 }
11925
11926 fn resume_action(run: &str) -> ask::ChoiceAction {
11927 ask::ChoiceAction::Resume { run: run.into() }
11928 }
11929
11930 #[test]
11931 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11932 for parked in [false, true] {
11933 let mut t = task();
11934 t.start("A".into());
11935 t.start("B".into());
11936 assert_eq!(t.runs, vec!["A", "B"]);
11937 t.start("A".into());
11938 let mut resumed = run_state(RunStatus::Blocked);
11939 resumed.id = "A".into();
11940 resumed.parked = parked;
11941 resumed.driver_exited = true;
11942 t.hold_machine(Some("resumed run stopped".into()));
11943 let q = action_question("A", resume_action("A"));
11944 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11945 assert_eq!(
11946 decide_action(&t, &q, &PHRASES_EN, |id| {
11947 assert_eq!(id, "A");
11948 Ok(resumed.clone())
11949 }),
11950 ActionDecision::Resume("A".into())
11951 );
11952 assert_eq!(
11953 action_standing(&t, &action_question("B", resume_action("B"))),
11954 ActionStanding::Stale
11955 );
11956 t.release_operator("cli", None);
11957 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11958 assert_eq!(id, "A");
11959 Ok(resumed.clone())
11960 });
11961 assert_eq!(
11962 choose_starter(None, false, unfinished.as_deref()),
11963 Starter::Resume("A".into())
11964 );
11965 assert_eq!(t.runs, vec!["B", "A"]);
11966 }
11967 }
11968
11969 #[test]
11970 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
11971 let mut state = crate::land::tests::repair_state(true, true);
11972 state.status = RunStatus::Landing;
11973 state.parked = true;
11974 state.driver_exited = true;
11975 let mut q = ask::Question::new(
11976 state.id.clone(),
11977 land::APPROVAL_NODE.into(),
11978 "land".into(),
11979 "Merge?".into(),
11980 String::new(),
11981 vec![land::APPROVE.into(), land::HOLD.into()],
11982 );
11983 q.id = state.id.clone();
11984 state.land_approval = Some(crate::run::LandApproval {
11985 question: q.id.clone(),
11986 head: "current-head".into(),
11987 });
11988 assert!(land::upgrade_repair_choice(&mut q, &state));
11989 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
11990 let mut task = held_task_with(&state.id);
11991 task.status = TaskStatus::Parked;
11992 assert_eq!(
11993 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
11994 ActionDecision::Resume(state.id.clone())
11995 );
11996 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
11997 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
11998 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
11999 assert_eq!(
12000 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12001 ActionDecision::Resume(state.id.clone()),
12002 "a repair answer newer than the manual hold resumes the existing owner"
12003 );
12004 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12005 for answered_at in [q.answered_at, None] {
12006 let mut older = q.clone();
12007 older.answered_at = answered_at;
12008 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
12009 panic!("a later manual hold must prevent dispatch")
12010 });
12011 let ActionDecision::Refuse(reason) = refused else {
12012 panic!("expected an actionable refusal, got {refused:?}");
12013 };
12014 assert!(reason.contains("later operator decision"));
12015 assert!(reason.contains(&format!("magi task release {}", task.id)));
12016 }
12017 task.operator_decided_at = None;
12018 let mut superseded = state.clone();
12019 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
12020 assert_eq!(
12021 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
12022 ActionDecision::Stale
12023 );
12024 for case in 1..4 {
12025 let mut changed = state.clone();
12026 match case {
12027 1 => changed.status = RunStatus::Blocked,
12028 2 => changed.released_to = Some("new-owner".into()),
12029 _ => changed.land_approval.as_mut().unwrap().head.clear(),
12030 }
12031 assert!(matches!(
12032 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
12033 ActionDecision::Refuse(_)
12034 ));
12035 }
12036 assert!(matches!(
12037 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
12038 ActionDecision::Refuse(_)
12039 ));
12040 task.runs.push("replacement".into());
12041 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12042 assert_eq!(
12043 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
12044 "old owner must not load"
12045 )),
12046 ActionDecision::Stale
12047 );
12048 }
12049
12050 #[test]
12051 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
12052 let mut state = crate::land::tests::repair_state(true, true);
12053 state.status = RunStatus::Landing;
12054 state.parked = true;
12055 state.driver_exited = true;
12056 let mut q = ask::Question::new(
12057 state.id.clone(),
12058 land::APPROVAL_NODE.into(),
12059 "land".into(),
12060 "Merge?".into(),
12061 String::new(),
12062 vec![land::APPROVE.into(), land::HOLD.into()],
12063 );
12064 q.id = state.id.clone();
12065 state.land_approval = Some(crate::run::LandApproval {
12066 question: q.id.clone(),
12067 head: "current-head".into(),
12068 });
12069 state.save().unwrap();
12070 land::upgrade_repair_choice(&mut q, &state);
12071 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12072 Questions::open().put(&mut q).unwrap();
12073 let tmp = tempfile::tempdir().unwrap();
12074 let queue = Queue::at(tmp.path().join("queue"));
12075 let mut task = held_task_with(&state.id);
12076 task.status = TaskStatus::Parked;
12077 queue.put(&mut task).unwrap();
12078 let empty = Arc::new(tokio::sync::Semaphore::new(0));
12079 let free = Arc::new(tokio::sync::Semaphore::new(1));
12080 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
12081 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
12082 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
12083 assert!(claim.permit.is_some());
12084 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
12085 drop(claim);
12086 Questions::open()
12087 .update(&q.id, |q| {
12088 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
12089 Ok(())
12090 })
12091 .unwrap();
12092 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
12093 assert!(claim.permit.is_none());
12094 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
12095 }
12096
12097 #[test]
12098 fn decide_action_resumes_only_the_latest_resumable_run() {
12099 let t = held_task_with("r1");
12100 let q = action_question("r1", resume_action("r1"));
12101 let load = |s: RunState| move |_: &str| Ok(s);
12102 assert_eq!(
12103 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
12104 ActionDecision::Resume("r1".into())
12105 );
12106 let q_other = action_question("r1", resume_action("r0"));
12108 assert!(matches!(
12109 decide_action(
12110 &t,
12111 &q_other,
12112 &PHRASES_EN,
12113 load(run_state(RunStatus::Blocked))
12114 ),
12115 ActionDecision::Refuse(_)
12116 ));
12117 assert!(matches!(
12119 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
12120 ActionDecision::Refuse(_)
12121 ));
12122 let mut released = run_state(RunStatus::Blocked);
12124 released.released_to = Some("elsewhere".into());
12125 assert!(matches!(
12126 decide_action(&t, &q, &PHRASES_EN, load(released)),
12127 ActionDecision::Refuse(_)
12128 ));
12129 assert!(matches!(
12131 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
12132 ActionDecision::Refuse(_)
12133 ));
12134 }
12135
12136 #[test]
12137 fn decide_action_ignores_a_question_about_an_earlier_run() {
12138 let mut t = held_task_with("r1");
12139 t.runs.push("r2".to_owned());
12140 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12141 assert_eq!(
12142 decide_action(
12143 &t,
12144 &action_question("r1", ask::ChoiceAction::Done),
12145 &PHRASES_EN,
12146 never
12147 ),
12148 ActionDecision::Stale
12149 );
12150 }
12151
12152 #[test]
12153 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
12154 let mut t = held_task_with("r1");
12155 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12156 assert_eq!(
12157 decide_action(
12158 &t,
12159 &action_question("r1", ask::ChoiceAction::Requeue),
12160 &PHRASES_EN,
12161 never
12162 ),
12163 ActionDecision::Requeue
12164 );
12165 let done_q = action_question("r1", ask::ChoiceAction::Done);
12166 assert_eq!(
12167 decide_action(&t, &done_q, &PHRASES_EN, never),
12168 ActionDecision::Done
12169 );
12170 t.mark_action_applied(&done_q.id);
12171 assert_eq!(
12172 decide_action(&t, &done_q, &PHRASES_EN, never),
12173 ActionDecision::Skip
12174 );
12175
12176 let mut plain = action_question("r1", ask::ChoiceAction::Done);
12178 plain.actions.clear();
12179 assert_eq!(
12180 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
12181 ActionDecision::Skip
12182 );
12183 let mut running = held_task_with("r1");
12185 running.status = TaskStatus::Running;
12186 assert_eq!(
12187 decide_action(
12188 &running,
12189 &action_question("r1", ask::ChoiceAction::Done),
12190 &PHRASES_EN,
12191 never
12192 ),
12193 ActionDecision::Skip
12194 );
12195 }
12196
12197 #[test]
12198 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12199 crate::run::pin_test_home();
12200 for legacy in [false, true] {
12201 let dir = tempfile::tempdir().unwrap();
12202 let queue = Queue::at(dir.path().join("queue"));
12203 let questions = Questions::at(dir.path().join("questions"));
12204 let run = "20260101-000000-hold";
12205 let mut task = held_task_with(run);
12206 task.status = TaskStatus::Parked;
12207 let mut q = ask::Question::new(
12208 run.into(),
12209 land::APPROVAL_NODE.into(),
12210 "land".into(),
12211 "Repair?".into(),
12212 String::new(),
12213 vec![land::REPAIR.into()],
12214 );
12215 q.actions.insert(land::REPAIR.into(), resume_action(run));
12216 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12217 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12219 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12220 questions.put(&mut q).unwrap();
12221 task.hold_manual(Some("Keep this PR paused".into()));
12222 if legacy {
12223 task.hold_source = None;
12224 }
12225 queue.put(&mut task).unwrap();
12226
12227 apply_task_choice_actions(&queue, &questions, dir.path());
12228 let after = queue.get(&task.id).unwrap();
12229 assert!(after.operator_held());
12230 assert_eq!(after.hold_source, task.hold_source);
12231 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12232 assert_eq!(after.runs, task.runs);
12233 assert_eq!(after.instruction, task.instruction);
12234 assert!(after.resume_override.is_none());
12235 assert!(after.action_applied(&q.id));
12236 let reason = after.hold_reason.as_deref().unwrap();
12237 assert!(reason.starts_with("Keep this PR paused"));
12238 assert!(reason.contains("later operator decision"));
12239 assert!(reason.contains(&format!("magi task release {}", task.id)));
12240
12241 for recovery in [
12242 crate::conduct::Recovery::Review,
12243 crate::conduct::Recovery::Requeue,
12244 ] {
12245 crate::conduct::apply(
12246 &queue,
12247 &questions,
12248 &crate::conduct::Verdict {
12249 decisions: vec![crate::conduct::Decision {
12250 id: task.id.clone(),
12251 recovery: Some(recovery),
12252 ..Default::default()
12253 }],
12254 },
12255 )
12256 .unwrap();
12257 }
12258 apply_task_choice_actions(&queue, &questions, dir.path());
12261 let still_held = queue.get(&task.id).unwrap();
12262 assert!(still_held.operator_held());
12263 assert_eq!(still_held.hold_reason, after.hold_reason);
12264 assert_eq!(still_held.runs, task.runs);
12265 assert!(still_held.review_branch.is_none());
12266 assert!(!still_held.fresh_start);
12267 assert!(queue.next_runnable().is_none());
12268 }
12269 }
12270
12271 #[test]
12272 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
12273 for held in [false, true] {
12274 for retry in [false, true] {
12275 let dir = tempfile::tempdir().unwrap();
12276 let queue = Queue::at(dir.path().join("queue"));
12277 let questions = Questions::at(dir.path().join("questions"));
12278 let home = dir.path().join("home");
12279 let mut t = task();
12280 if held {
12281 t.hold_machine(Some("driver state unknown".into()));
12282 }
12283 queue.put(&mut t).unwrap();
12284 let verdict = serde_json::from_value(serde_json::json!({
12285 "decisions": [{
12286 "id": t.id,
12287 "question": "Driver state unknown; retry the checks or hold?",
12288 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
12289 "actions": {
12290 "daemon: recheck and retry": {"do": "requeue"},
12291 "daemon: hold the task": {"do": "hold"}
12292 }
12293 }]
12294 }))
12295 .unwrap();
12296 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
12297 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12298 let mut q = questions.list().pop().unwrap();
12299 let choice = if retry {
12300 "daemon: recheck and retry"
12301 } else {
12302 "daemon: hold the task"
12303 };
12304 assert_eq!(q.actions.len(), 2);
12305 q.answer(ask::Answer::Choice(choice.into())).unwrap();
12306 questions.put(&mut q).unwrap();
12307
12308 questions.beat(&q.id, ask::WaiterKind::Deputy);
12310 for _ in 0..2 {
12311 resolve_blockers(&queue, &questions);
12312 apply_task_choice_actions(&queue, &questions, &home);
12313 let waiting = queue.get(&t.id).unwrap();
12314 assert_eq!(waiting.status, TaskStatus::Blocked);
12315 assert_eq!(waiting.blocked_by, [q.id.clone()]);
12316 assert!(waiting.answers.is_empty());
12317 assert!(!waiting.action_applied(&q.id));
12318 assert!(runnable(&queue).is_empty());
12319 let free = Arc::new(tokio::sync::Semaphore::new(1));
12320 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
12321 }
12322 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12323
12324 resolve_blockers(&queue, &questions);
12326 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12327 apply_task_choice_actions(&queue, &questions, &home);
12328 let after = queue.get(&t.id).unwrap();
12329 assert_eq!(
12330 after.status,
12331 if retry {
12332 TaskStatus::Queued
12333 } else {
12334 TaskStatus::Held
12335 }
12336 );
12337 assert!(after.action_applied(&q.id));
12338 assert_eq!(after.operator_held(), !retry);
12339 assert_eq!(after.answers.len(), 1);
12340
12341 let mut changed = after;
12343 if retry {
12344 changed.hold_manual(Some("later hold".into()));
12345 } else {
12346 changed.release_operator("test", None);
12347 }
12348 let expected = changed.status;
12349 queue.put(&mut changed).unwrap();
12350 apply_task_choice_actions(&queue, &questions, &home);
12351 assert_eq!(queue.get(&t.id).unwrap().status, expected);
12352 }
12353 }
12354 }
12355
12356 #[test]
12357 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
12358 let dir = tempfile::tempdir().unwrap();
12359 let queue = Queue::at(dir.path().join("queue"));
12360 let questions = Questions::at(dir.path().join("questions"));
12361 let home = dir.path().join("home");
12362 let mut state = run_state(RunStatus::Blocked);
12363 state.id = "20260101-000000-act1".to_owned();
12364 state.save_under(&home).unwrap();
12365
12366 let mut t = held_task_with(&state.id);
12367 queue.put(&mut t).unwrap();
12368 let mut q = action_question(&state.id, resume_action(&state.id));
12369 questions.put(&mut q).unwrap();
12370
12371 apply_task_choice_actions(&queue, &questions, &home);
12372 let after = queue.get(&t.id).unwrap();
12373 assert_eq!(after.status, TaskStatus::Queued);
12374 assert!(!after.fresh_start);
12375 assert!(after.action_applied(&q.id));
12376 let pin = after.resume_override.clone().unwrap();
12377 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
12378 assert!(pin.forced);
12379
12380 let mut again = queue.get(&t.id).unwrap();
12382 again.hold_machine(Some("later".into()));
12383 queue.put(&mut again).unwrap();
12384 apply_task_choice_actions(&queue, &questions, &home);
12385 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12386 }
12387
12388 #[test]
12389 fn a_delivered_answer_is_still_acted_on_exactly_once() {
12390 let dir = tempfile::tempdir().unwrap();
12391 let queue = Queue::at(dir.path().join("queue"));
12392 let questions = Questions::at(dir.path().join("questions"));
12393 let home = dir.path().join("home");
12394 let mut state = run_state(RunStatus::Blocked);
12395 state.id = "20260101-000000-act2".to_owned();
12396 state.save_under(&home).unwrap();
12397
12398 let mut t = held_task_with(&state.id);
12399 queue.put(&mut t).unwrap();
12400 let mut q = action_question(&state.id, resume_action(&state.id));
12401 q.answer_delivered = true;
12402 questions.put(&mut q).unwrap();
12403
12404 apply_task_choice_actions(&queue, &questions, &home);
12407 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12408 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
12409
12410 let mut again = queue.get(&t.id).unwrap();
12412 again.hold_machine(Some("later".into()));
12413 queue.put(&mut again).unwrap();
12414 apply_task_choice_actions(&queue, &questions, &home);
12415 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12416 }
12417
12418 #[test]
12419 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
12420 let dir = tempfile::tempdir().unwrap();
12421 let queue = Queue::at(dir.path().join("queue"));
12422 let questions = Questions::at(dir.path().join("questions"));
12423 let home = dir.path().join("home");
12424 let mut state = run_state(RunStatus::Blocked);
12425 state.id = "20260101-000000-act3".to_owned();
12426 state.save_under(&home).unwrap();
12427 let mut t = held_task_with(&state.id);
12428 queue.put(&mut t).unwrap();
12429 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
12430 questions.put(&mut q).unwrap();
12431
12432 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
12433 apply_task_choice_actions(&queue, &questions, &home);
12434 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12435
12436 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12437 apply_task_choice_actions(&queue, &questions, &home);
12438 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12439 }
12440
12441 #[test]
12442 fn action_standing_tells_the_reasons_apart() {
12443 let mut t = held_task_with("r1");
12444 let q = action_question("r1", ask::ChoiceAction::Requeue);
12445 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12446 let mut plain = q.clone();
12447 plain.actions.clear();
12448 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
12449
12450 t.mark_action_applied(&q.id);
12452 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
12453
12454 let mut t = held_task_with("r1");
12456 t.start("r2".to_owned());
12457 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
12458 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
12460 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12461 t.status = TaskStatus::Blocked;
12462 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12463
12464 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
12466 c.node = crate::conduct::NODE.to_owned();
12467 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
12468 assert!(!ActionStanding::Busy.daemon_owns());
12469 }
12470
12471 #[test]
12472 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
12473 let dir = tempfile::tempdir().unwrap();
12474 let queue = Queue::at(dir.path().join("queue"));
12475 let questions = Questions::at(dir.path().join("questions"));
12476 let home = dir.path().join("home");
12477 let mut state = run_state(RunStatus::Blocked);
12478 state.id = "20260101-000000-act4".to_owned();
12479 state.save_under(&home).unwrap();
12480 let mut t = held_task_with(&state.id);
12481 t.status = TaskStatus::Running;
12482 queue.put(&mut t).unwrap();
12483 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
12484 q.cwd = Some(dir.path().display().to_string());
12485 questions.put(&mut q).unwrap();
12486
12487 let running = queue.get(&t.id).unwrap();
12490 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
12491 assert!(matches!(
12492 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
12493 crate::waiter::Action::Deliver(_)
12494 ));
12495 apply_task_choice_actions(&queue, &questions, &home);
12497 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
12498 let mut back = queue.get(&t.id).unwrap();
12499 back.hold_machine(Some("later".into()));
12500 queue.put(&mut back).unwrap();
12501 apply_task_choice_actions(&queue, &questions, &home);
12502 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
12503 let mut again = queue.get(&t.id).unwrap();
12504 again.hold_machine(Some("again".into()));
12505 queue.put(&mut again).unwrap();
12506 apply_task_choice_actions(&queue, &questions, &home);
12507 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12508 }
12509}