1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.runs.last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id,
519 run: task.runs.last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if q.chosen_action().is_none() {
887 ActionStanding::NoAction
888 } else if task.action_applied(&q.id) {
889 ActionStanding::Applied
890 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 ActionStanding::Stale
892 } else if matches!(
893 task.status,
894 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
895 ) {
896 ActionStanding::Busy
897 } else {
898 ActionStanding::Pending
899 }
900}
901
902#[derive(Debug, Clone, PartialEq, Eq)]
904enum ActionDecision {
905 Skip,
907 Resume(String),
909 Requeue,
911 Hold,
913 Done,
915 Stale,
918 Refuse(String),
921}
922
923fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
931where
932 F: FnOnce(&str) -> Result<RunState>,
933{
934 let Some(action) = q.chosen_action() else {
935 return ActionDecision::Skip;
936 };
937 if task.action_applied(&q.id)
938 || matches!(
939 task.status,
940 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
941 )
942 {
943 return ActionDecision::Skip;
944 }
945 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
948 return ActionDecision::Stale;
949 }
950 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
951 let stopping = q.node == land::APPROVAL_NODE
952 && q.resolution()
953 .as_deref()
954 .is_some_and(crate::merge_decision::stops);
955 let decision_at = if repair || stopping {
959 q.answered_at.unwrap_or(q.asked_at)
960 } else {
961 q.asked_at
962 };
963 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
964 return if repair || stopping {
965 ActionDecision::Refuse(format!(
966 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
967 if repair { "repair" } else { "stop" },
968 q.id,
969 q.run,
970 task.id
971 ))
972 } else {
973 ActionDecision::Stale
974 };
975 }
976 match action {
977 a if a.maintenance() => ActionDecision::Skip,
978 ask::ChoiceAction::ResumeRun { .. }
979 | ask::ChoiceAction::ReleaseTask { .. }
980 | ask::ChoiceAction::RerunChecks { .. }
981 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
982 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
983 ask::ChoiceAction::Hold => ActionDecision::Hold,
984 ask::ChoiceAction::Done => ActionDecision::Done,
985 ask::ChoiceAction::Resume { run } => {
986 if task.runs.last() != Some(run) {
987 return ActionDecision::Refuse((p.resume_not_latest)(
988 q.short(),
989 ask::short_id(run),
990 ));
991 }
992 match load(run) {
993 Ok(s) if stopping => {
994 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
995 || !s
996 .land_approval
997 .as_ref()
998 .is_some_and(|binding| binding.question == q.id)
999 {
1000 ActionDecision::Stale
1001 } else if s.liveness(false) == Liveness::Live {
1002 ActionDecision::Skip
1003 } else if s.status == RunStatus::Landing
1004 && !s.released()
1005 && s.retired_to.is_none()
1006 {
1007 ActionDecision::Resume(run.clone())
1008 } else {
1009 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1010 }
1011 }
1012 Ok(s)
1013 if q.node == land::APPROVAL_NODE
1014 && q.resolution().as_deref() == Some(land::REPAIR) =>
1015 {
1016 if s.merge_repairs.iter().any(|r| r.question == q.id)
1017 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1018 {
1019 ActionDecision::Stale
1022 } else if s.liveness(false) == Liveness::Live {
1023 ActionDecision::Skip
1024 } else if s.status != RunStatus::Landing
1025 || s.released()
1026 || s.reviews.is_empty()
1027 || !implementers_match(task, &s)
1028 || !s
1029 .land_approval
1030 .as_ref()
1031 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1032 {
1033 ActionDecision::Refuse(format!(
1034 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1035 q.id, run, run
1036 ))
1037 } else {
1038 ActionDecision::Resume(run.clone())
1039 }
1040 }
1041 Ok(s) if s.liveness(false) == Liveness::Live => {
1042 ActionDecision::Refuse("run already has a live driver".into())
1043 }
1044 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
1045 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
1046 q.short(),
1047 ask::short_id(run),
1048 )),
1049 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1050 q.short(),
1051 ask::short_id(run),
1052 &format!("{e:#}"),
1053 )),
1054 }
1055 }
1056 }
1057}
1058
1059struct Phrases {
1073 graph_stopped: fn(&str, &str) -> String,
1075 quorum_lost: &'static str,
1076 quota_took_out: &'static str,
1078 run_ended: &'static str,
1080 waiting_for_answer: &'static str,
1082 recovered_running: &'static str,
1083 no_run_to_recover: &'static str,
1084 could_not_start: &'static str,
1085 resume_not_latest: fn(&str, &str) -> String,
1086 resume_cannot_progress: fn(&str, &str) -> String,
1087 resume_unreadable: fn(&str, &str, &str) -> String,
1088 handover_refused: fn(&crate::handover::Refused) -> String,
1091 handover_hint: &'static str,
1093 reviewers_never_answered: fn(usize, usize) -> String,
1096}
1097
1098const PHRASES_EN: Phrases = Phrases {
1099 graph_stopped: |status, detail| {
1100 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1101 },
1102 quorum_lost: "the judging panel lost its quorum",
1103 quota_took_out: "; quota took out ",
1104 run_ended: "run ended ",
1105 waiting_for_answer: " - waiting for operator answer to question ",
1106 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1107 no_run_to_recover: "task was `running` with no live daemon and no readable \
1108 run to recover; held for a human to check what happened",
1109 could_not_start: "could not start the run: ",
1110 resume_not_latest: |q, run| {
1111 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1112 },
1113 resume_cannot_progress: |q, run| {
1114 format!("question {q} asked to resume run {run}, which cannot make progress")
1115 },
1116 resume_unreadable: |q, run, e| {
1117 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1118 },
1119 handover_refused: |r| r.to_string(),
1120 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1121 reviewers_never_answered: |missing, rounds| {
1122 format!(
1123 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1124 refusing to call it clean"
1125 )
1126 },
1127};
1128
1129const PHRASES_JA: Phrases = Phrases {
1130 graph_stopped: |status, detail| {
1131 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1132 },
1133 quorum_lost: "審査パネルが定足数を失いました",
1134 quota_took_out: "。クォータで脱落: ",
1135 run_ended: "run 終了: ",
1136 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1137 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1138 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1139 何が起きたか人が確認するため保留にしました",
1140 could_not_start: "run を開始できませんでした: ",
1141 resume_not_latest: |q, run| {
1142 format!(
1143 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1144 )
1145 },
1146 resume_cannot_progress: |q, run| {
1147 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1148 },
1149 resume_unreadable: |q, run, e| {
1150 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1151 },
1152 handover_refused: |r| {
1153 use crate::handover::Refused;
1154 match r {
1155 Refused::Foreign { branch, path, why } => format!(
1156 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1157 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1158 から、やり直してください(詳細: {why})"
1159 ),
1160 Refused::Unsafe { branch, path, why } => format!(
1161 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1162 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1163 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1164 ),
1165 Refused::ReleaseFailed { branch, path, run } => format!(
1166 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1167 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1168 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1169 ),
1170 }
1171 },
1172 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1173 reviewers_never_answered: |missing, rounds| {
1174 format!(
1175 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1176 クリーンとは認めません"
1177 )
1178 },
1179};
1180
1181fn phrases(language: &str) -> &'static Phrases {
1182 if crate::lang::is_japanese(language) {
1183 &PHRASES_JA
1184 } else {
1185 &PHRASES_EN
1186 }
1187}
1188
1189fn language_of(task: &Task, fallback: &Path) -> String {
1193 crate::lang::of_repo(&repo_for(task, fallback))
1194}
1195
1196pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1200 if q.node == crate::conduct::NODE {
1201 return tasks.iter().find(|t| t.id == q.run);
1202 }
1203 tasks.iter().find(|t| t.runs.contains(&q.run))
1204}
1205
1206fn repair_choice_offers(questions: &Questions) {
1209 for q in questions.list().into_iter().filter(|q| {
1210 q.status.open()
1211 && q.node == crate::conduct::NODE
1212 && q.choices
1213 .iter()
1214 .any(|c| crate::choice_effect::describe(q, c).is_none())
1215 }) {
1216 let _ = questions.update(&q.id, |current| {
1217 if current.status.open() {
1218 crate::choice_effect::executable_conductor(current);
1219 if let Some(deputy) = &mut current.deputy {
1220 deputy.brief = crate::deputy::brief(
1221 ¤t.run,
1222 ¤t.detail,
1223 ¤t.choices,
1224 ¤t.actions,
1225 );
1226 }
1227 }
1228 Ok(())
1229 });
1230 }
1231}
1232
1233pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1236 apply_maintenance_choices(queue, questions, home).await;
1237 apply_task_choice_actions(queue, questions, home);
1238}
1239
1240async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1243 for q in questions.list() {
1244 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1245 continue;
1246 };
1247 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1250 continue;
1251 }
1252 let dir = home.join("choice-actions");
1253 if std::fs::create_dir_all(&dir).is_err() {
1254 continue;
1255 }
1256 let Ok(lock) = std::fs::OpenOptions::new()
1257 .create(true)
1258 .truncate(false)
1259 .write(true)
1260 .open(dir.join(format!("{}.lock", q.id)))
1261 else {
1262 continue;
1263 };
1264 if crate::pr_head::try_lock_file(&lock).is_err() {
1265 continue;
1266 }
1267 let fresh = !home
1268 .join("choice-actions")
1269 .join(format!("{}.json", q.id))
1270 .exists();
1271 let result = async {
1272 let receipt = dir.join(format!("{}.json", q.id));
1273 if receipt.exists() {
1274 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1275 anyhow::ensure!(
1276 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1277 "previous maintenance attempt did not confirm completion; inspect its receipt"
1278 );
1279 return Ok::<_, anyhow::Error>(());
1280 }
1281 let record = |status: &str| -> anyhow::Result<()> {
1284 let tmp = receipt.with_extension("tmp");
1285 std::fs::write(
1286 &tmp,
1287 serde_json::to_vec(&serde_json::json!({
1288 "action": action, "status": status,
1289 }))?,
1290 )?;
1291 std::fs::rename(tmp, &receipt)?;
1292 Ok(())
1293 };
1294 let applied = async {
1295 match action {
1296 ask::ChoiceAction::ResumeRun { run } => {
1297 crate::resume::validate(home, run)?;
1298 record("started")?;
1299 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1300 crate::resume::validate_unqueued(home, run)?;
1301 crate::resume::start(home, run.clone()).await?;
1302 }
1303 }
1304 ask::ChoiceAction::ReleaseTask {
1305 task,
1306 held_at,
1307 reason,
1308 } => {
1309 anyhow::ensure!(
1310 queue.resolve_id(task)? == *task,
1311 "task action requires an exact task ID"
1312 );
1313 let _claim = queue.claim(task)?;
1314 let mut task = queue.get(task)?;
1315 if task.action_applied(&q.id) {
1316 return Ok(());
1317 }
1318 if q.node == crate::conduct::NODE
1323 && q.run == task.id
1324 && task.status == TaskStatus::Blocked
1325 && task.blocked_from == Some(TaskStatus::Held)
1326 && task.blocked_by.as_slice() == [q.id.as_str()]
1327 {
1328 task.unblock(&q.id);
1329 }
1330 anyhow::ensure!(
1331 task.status == TaskStatus::Held
1332 && task.held_at == Some(*held_at)
1333 && task.hold_reason.as_deref() == reason.as_deref()
1334 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1335 "task no longer has the hold this question names"
1336 );
1337 task.release_operator("choice", Some(&q.id));
1338 task.mark_action_applied(&q.id);
1339 queue.put(&mut task)?;
1340 }
1341 ask::ChoiceAction::AbandonRun { run } => {
1342 record("started")?;
1343 crate::driver::abandon_proven_dead(home, run)?;
1344 }
1345 ask::ChoiceAction::RerunChecks {
1346 repo,
1347 pr,
1348 head,
1349 runs,
1350 } => {
1351 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1352 .await?;
1353 }
1354 _ => unreachable!(),
1355 }
1356 Ok::<_, anyhow::Error>(())
1357 }
1358 .await;
1359 match applied {
1360 Ok(()) => record("applied"),
1361 Err(e) => {
1362 record(&format!("refused or failed: {e:#}"))?;
1363 Err(e)
1364 }
1365 }
1366 }
1367 .await;
1368 if home
1373 .join("choice-actions")
1374 .join(format!("{}.json", q.id))
1375 .exists()
1376 && let Some(listed) = task_of_question(&queue.list(), &q)
1377 && !listed.action_applied(&q.id)
1378 && let Ok(_claim) = queue.claim(&listed.id)
1379 && let Ok(mut origin) = queue.get(&listed.id)
1380 {
1381 if !origin.operator_held()
1382 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1383 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1384 {
1385 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1386 origin.hold_machine(Some(format!(
1387 "Maintenance choice {} settled; inspect its target before releasing this task",
1388 q.id
1389 )));
1390 } else {
1391 origin.unblock(&q.id);
1392 }
1393 }
1394 origin.mark_action_applied(&q.id);
1395 let _ = queue.put(&mut origin);
1396 }
1397 if let Err(e) = result
1398 && fresh
1399 {
1400 notices::raise_in(
1401 home,
1402 Notice::warn(
1403 &format!("action:{}", q.id),
1404 format!("Choice action {} was not completed: {e:#}", q.id),
1405 )
1406 .link(if q.run_names_task() {
1407 Link::Task { id: q.run.clone() }
1408 } else {
1409 Link::Run { id: q.run.clone() }
1410 }),
1411 );
1412 }
1413 }
1414}
1415
1416pub(crate) async fn rerun_checks_once(
1423 home: &Path,
1424 request: &str,
1425 repo: &Path,
1426 pr: &str,
1427 head: &str,
1428 runs: &[String],
1429 started: impl FnOnce() -> anyhow::Result<()>,
1430) -> anyhow::Result<()> {
1431 use crate::release_watch::{GhForge, ReleaseForge};
1432 anyhow::ensure!(
1433 !runs.is_empty()
1434 && runs.len() <= 20
1435 && runs
1436 .iter()
1437 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1438 && !pr.starts_with('-'),
1439 "invalid rerun targets"
1440 );
1441 let snap = GhForge.snapshot(repo, pr).await?;
1442 anyhow::ensure!(
1443 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1444 "pull request head or lifecycle changed"
1445 );
1446 anyhow::ensure!(
1447 runs.iter().all(|r| snap
1448 .checks
1449 .iter()
1450 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1451 "named workflow is no longer failed on this pull request"
1452 );
1453 started()?;
1454 let dir = home.join("choice-actions");
1455 std::fs::create_dir_all(&dir)?;
1456 let rerun_lock = std::fs::OpenOptions::new()
1457 .create(true)
1458 .truncate(false)
1459 .write(true)
1460 .open(dir.join("reruns.lock"))?;
1461 crate::pr_head::try_lock_file(&rerun_lock)?;
1462 let ledger = dir.join("reruns.json");
1463 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1464 serde_json::from_slice(&std::fs::read(&ledger)?)?
1465 } else {
1466 Default::default()
1467 };
1468 for run in runs {
1469 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1470 if !recorded.insert(key) {
1471 continue;
1472 }
1473 let tmp = ledger.with_extension("tmp");
1474 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1475 std::fs::rename(tmp, &ledger)?;
1476 GhForge.rerun(repo, run).await?;
1477 }
1478 Ok(())
1479}
1480
1481pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1490 let tasks = queue.list();
1491 for q in questions.list() {
1492 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1494 continue;
1495 }
1496 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1497 continue;
1498 }
1499 let Some(listed) = task_of_question(&tasks, &q) else {
1500 if q.node == land::APPROVAL_NODE
1501 && q.resolution().as_deref().is_some_and(|choice| {
1502 choice == land::REPAIR || crate::merge_decision::stops(choice)
1503 })
1504 {
1505 notices::raise_in(
1506 home,
1507 Notice::warn(
1508 &format!("repair-owner:{}", q.id),
1509 format!(
1510 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1511 q.run, q.run
1512 ),
1513 ).link(Link::Run { id: q.run.clone() }),
1514 );
1515 }
1516 continue;
1517 };
1518 if listed.action_applied(&q.id) {
1519 continue;
1520 }
1521 let Ok(_claim) = queue.claim(&listed.id) else {
1522 continue;
1523 };
1524 let Ok(mut task) = queue.get(&listed.id) else {
1525 continue;
1526 };
1527 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1530 let blockers: Vec<_> = task
1531 .blocked_by
1532 .iter()
1533 .filter(|id| {
1534 crate::queue::dependency(queue, questions, id)
1535 == crate::queue::Dependency::Question(q.id.clone())
1536 })
1537 .cloned()
1538 .collect();
1539 if !blockers.is_empty() {
1540 for id in blockers {
1541 task.unblock(&id);
1542 }
1543 let answer = match &q.answer {
1544 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1545 None => String::new(),
1546 };
1547 task.record_answer(q.summary.clone(), answer);
1548 }
1549 }
1550 let language = language_of(&task, Path::new("."));
1551 let decision = decide_action(&task, &q, phrases(&language), |id| {
1552 RunState::load_under(id, home)
1553 });
1554 let ran = matches!(
1555 decision,
1556 ActionDecision::Resume(_)
1557 | ActionDecision::Requeue
1558 | ActionDecision::Hold
1559 | ActionDecision::Done
1560 );
1561 if ran && asker_may_still_read(questions, &q, home) {
1562 continue;
1571 }
1572 let done = matches!(decision, ActionDecision::Done);
1573 match decision {
1574 ActionDecision::Skip => continue,
1575 ActionDecision::Resume(run) => {
1576 task.release_operator("choice", Some(&q.id));
1577 task.resume_override = Some(crate::queue::OperatorResume {
1578 question_id: q.id.clone(),
1579 at: Timestamp::now(),
1580 conductor_rehold: None,
1581 forced: true,
1582 pinned_run: Some(run),
1583 });
1584 }
1585 ActionDecision::Stale => {}
1586 ActionDecision::Requeue => {
1587 task.release_operator("choice-requeue", Some(&q.id));
1588 task.requeue();
1589 }
1590 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1591 ActionDecision::Done => task.succeed(),
1592 ActionDecision::Refuse(why) => {
1593 if task.operator_held() {
1594 task.hold_reason = Some(match task.hold_reason.take() {
1597 Some(reason) => format!("{reason}\n\n{why}"),
1598 None => why,
1599 });
1600 } else {
1601 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1602 task.hold_machine(Some(why));
1603 }
1604 notices::raise(
1605 Notice::warn(
1606 &format!("action:{}", q.id),
1607 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1608 )
1609 .link(Link::Task {
1610 id: task.id.clone(),
1611 }),
1612 );
1613 }
1614 }
1615 task.mark_action_applied(&q.id);
1616 record(queue, &mut task);
1619 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1620 supersede_prior_runs(&task, home);
1621 }
1622 let _ = questions.update(&q.id, |r| {
1625 r.answer_delivered = true;
1626 Ok(())
1627 });
1628 }
1629}
1630
1631fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1635 let now = Timestamp::now();
1636 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1637 return true;
1638 }
1639 q.node != crate::conduct::NODE
1640 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1641 s.seats_active()
1642 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1643 })
1644}
1645
1646fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1657 let tasks = queue.list();
1658 let by_id: std::collections::BTreeMap<&str, &Task> =
1659 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1660 let referenced: Vec<&str> = tasks
1661 .iter()
1662 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1663 .collect();
1664
1665 for mut question in questions.list() {
1666 if !question.status.open() || question.node != crate::conduct::NODE {
1667 continue;
1668 }
1669 if referenced
1674 .iter()
1675 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1676 {
1677 continue;
1678 }
1679 let Some(task) = by_id.get(question.run.as_str()) else {
1680 continue;
1681 };
1682 question.abandon(format!(
1683 "task {} no longer waits for this answer",
1684 task.short()
1685 ));
1686 if let Err(e) = questions.put(&mut question) {
1687 tracing::warn!(
1688 "could not retire question {} for task {}: {e:#}",
1689 question.short(),
1690 task.short()
1691 );
1692 }
1693 }
1694}
1695
1696#[derive(Debug, Clone, Copy)]
1703pub struct Verdict {
1704 pub status: RunStatus,
1706 pub left_pr: bool,
1708 pub quota_hit: bool,
1710 pub parked: bool,
1712 pub no_viable_candidates: bool,
1719}
1720
1721pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1778 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1779}
1780
1781fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1783 if verdict.parked {
1789 task.park(detail);
1790 return;
1791 }
1792 match verdict.status {
1793 RunStatus::Merged | RunStatus::Ready
1794 if task.pr_recovery.as_ref().is_some_and(|s| {
1795 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1796 }) =>
1797 {
1798 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1799 }
1800 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1801 RunStatus::AlreadyInBase => task.already_landed(detail),
1802 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1803 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1804 task.stall(detail)
1805 }
1806 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1807 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1808 RunStatus::Blocked => task.fail(detail, max_attempts),
1809 RunStatus::VerifiedNoop => task.handed_off(detail),
1810 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1811 }
1812}
1813
1814pub fn supersede_prior_runs(task: &Task, home: &Path) {
1863 let now = Timestamp::now();
1864 let last_run_succeeded = task
1871 .runs
1872 .last()
1873 .and_then(|id| RunState::load_under(id, home).ok())
1874 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1875 for id in task.superseded_attempts(last_run_succeeded) {
1876 let mut state = match RunState::load_under(id, home) {
1877 Ok(s) => s,
1878 Err(e) => {
1879 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1880 continue;
1881 }
1882 };
1883 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1884 continue;
1885 }
1886 let daemon_claims = is_working_on(home, id, now);
1887 if state.liveness(daemon_claims) == Liveness::Live {
1888 continue;
1889 }
1890 state.status = RunStatus::Superseded;
1891 if let Err(e) = state.save_under(home) {
1892 tracing::warn!("could not mark run {id} superseded: {e:#}");
1893 }
1894 }
1895}
1896
1897fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1919 for task in queue.list() {
1920 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1921 continue;
1922 }
1923 supersede_prior_runs(&task, home);
1924 }
1925}
1926
1927fn settle_and_diagnose(
1934 task: &mut Task,
1935 verdict: Verdict,
1936 detail: &str,
1937 max_attempts: usize,
1938 state: &RunState,
1939) {
1940 let p = phrases(&state.config.graph.language);
1941 if let Some(d) = &state.merge_decision {
1944 use crate::merge_decision as md;
1945 if task.runs.last().is_some_and(|id| id != &state.id) {
1946 return;
1947 }
1948 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1949 task.succeed();
1950 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1951 if state.liveness(false) != Liveness::Dead {
1952 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1953 return;
1954 }
1955 let replacement = md::Replacement {
1956 question: d.question.clone(),
1957 run: state.id.clone(),
1958 pr: d.target.pr.clone(),
1959 agent: agent.to_owned(),
1960 context: format!(
1961 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1962 d.target.pr,
1963 state.id,
1964 serde_json::to_string(&state.reviews).unwrap_or_default()
1965 ),
1966 };
1967 task.requeue();
1968 task.pr_recovery = None;
1969 task.review_of = None;
1970 task.replacement = Some(replacement);
1971 } else {
1972 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1973 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1974 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1975 }
1976 task.mark_action_applied(&d.question);
1977 if let Some(pass) = state.passes.last() {
1978 task.record_pass_at(
1979 &state.id,
1980 pass.index,
1981 Some(pass.started_at),
1982 task.attempts,
1983 None,
1984 );
1985 }
1986 return;
1987 }
1988 if let Some(replacement) = &task.replacement
1989 && !state.parked
1990 && state.resource_block.is_none()
1991 && (state.viable().is_empty()
1992 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1993 {
1994 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1995 return;
1996 }
1997 let attempts_before = task.attempts;
1998 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2005 task.notice_cause(
2006 &format!("resource:{}:{}", block.kind, block.node),
2007 None,
2008 None,
2009 );
2010 task.last_error = Some(detail.to_owned());
2011 task.diagnostic = None;
2012 task.attempts = task.attempts.saturating_sub(1);
2013 if block.kind == "disk pressure" {
2014 task.hold_machine(Some(format!(
2015 "{}{}",
2016 crate::disk::RUNNING_PRESSURE,
2017 block.detail
2018 )));
2019 } else {
2020 task.hold_machine(Some(format!(
2021 "{}{}; no attempt was spent; resume to retry the affected operation",
2022 crate::cache::RESOURCE_HOLD,
2023 block.description()
2024 )));
2025 }
2026 if let Some(pass) = state.passes.last() {
2027 task.record_pass_at(
2028 &state.id,
2029 pass.index,
2030 Some(pass.started_at),
2031 attempts_before,
2032 None,
2033 );
2034 }
2035 return;
2036 }
2037 let unresolved = task.hold_notice_cause.clone();
2038 if !verdict.parked
2039 && task
2040 .pr_recovery
2041 .as_ref()
2042 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2043 && matches!(
2044 verdict.status,
2045 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2046 )
2047 {
2048 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2049 task.succeed();
2050 } else {
2051 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2052 }
2053 } else if !verdict.parked
2054 && task.pr_recovery.is_some()
2055 && matches!(
2056 verdict.status,
2057 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2058 )
2059 && !verdict.quota_hit
2060 {
2061 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2062 } else {
2063 settle_in(task, verdict, detail, max_attempts, p);
2064 }
2065 if !verdict.parked
2066 && !verdict.quota_hit
2067 && matches!(
2068 verdict.status,
2069 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2070 )
2071 {
2072 task.recovery_failed(state);
2073 }
2074 if state.prep_failure.is_some() {
2075 task.hold_notice_cause = unresolved;
2078 task.startup_notice_cause("startup-candidates");
2079 }
2080 if let Some(pass) = state.passes.last() {
2083 let parked = match &pass.end {
2084 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2085 _ => None,
2086 };
2087 task.record_pass_at(
2088 &state.id,
2089 pass.index,
2090 Some(pass.started_at),
2091 attempts_before,
2092 parked,
2093 );
2094 }
2095 if task.status == TaskStatus::Held {
2096 if state.prep_failure.is_none() {
2097 task.notice_cause(
2098 label(verdict.status),
2099 Some(&state.id),
2100 state.passes.last().map(|p| p.index),
2101 );
2102 }
2103 task.diagnostic = diagnostic(state);
2104 note_open_question(task, &state.id, p);
2105 }
2106}
2107
2108fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2123 let Some(home) = crate::run::try_home() else {
2124 return;
2125 };
2126 let open = Questions::at(home.join("questions")).open_for(run);
2127 let Some(q) = open.first() else {
2128 return;
2129 };
2130 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2131 decisions.sort_unstable();
2132 task.notice_cause(
2133 &format!("questions:{}", decisions.join(",")),
2134 Some(run),
2135 None,
2136 );
2137 let base = task.hold_reason.clone().unwrap_or_default();
2138 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2139}
2140
2141fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2152 match last_run {
2153 Some(state) => {
2154 let verdict = Verdict {
2155 status: state.status,
2156 left_pr: state.pr.is_some(),
2157 quota_hit: !state.quota.is_empty(),
2158 parked: state.parked,
2159 no_viable_candidates: state.viable().is_empty(),
2160 };
2161 let detail = format!(
2162 "{}{}",
2163 phrases(&state.config.graph.language).recovered_running,
2164 describe(&state)
2165 );
2166 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2167 }
2168 None => {
2169 let why = phrases(language).no_run_to_recover;
2172 task.last_error = Some(why.to_owned());
2173 task.notice_cause("missing-run", None, None);
2176 task.hold_machine(Some(why.to_owned()));
2177 }
2178 }
2179}
2180
2181fn reclaim_orphaned_running(
2203 queue: &Queue,
2204 max_attempts: usize,
2205 live_workers: &std::collections::BTreeSet<String>,
2206) -> Vec<String> {
2207 let mut reclaimed = Vec::new();
2208 for listed in queue.list() {
2209 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2210 continue;
2211 }
2212 let Ok(_claim) = queue.claim(&listed.id) else {
2213 continue;
2214 };
2215 let Ok(mut task) = queue.get(&listed.id) else {
2219 continue;
2220 };
2221 if task.status != TaskStatus::Running {
2222 continue;
2223 }
2224 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2225 if let Some(state) = &last_run
2235 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2236 {
2237 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2238 }
2239 let language = if last_run.is_none() {
2240 language_of(&task, Path::new("."))
2241 } else {
2242 String::new()
2243 };
2244 reclaim(&mut task, last_run, max_attempts, &language);
2245 if task.status == TaskStatus::Done {
2246 supersede_prior_runs(&task, &crate::run::home());
2247 }
2248 record(queue, &mut task);
2249 reclaimed.push(task.id.clone());
2250 }
2251 reclaimed
2252}
2253
2254fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2286 reclaim_abandoned_runs_with(
2287 home,
2288 now,
2289 crate::proc::pid_status,
2290 crate::proc::process_started_at,
2291 )
2292}
2293
2294fn reclaim_abandoned_runs_with<F, G>(
2300 home: &Path,
2301 now: Timestamp,
2302 query: F,
2303 identity: G,
2304) -> Vec<String>
2305where
2306 F: Fn(u32) -> Option<bool>,
2307 G: Fn(u32) -> Option<String>,
2308{
2309 let mut abandoned = Vec::new();
2310 for entry in std::fs::read_dir(home.join("runs"))
2311 .into_iter()
2312 .flatten()
2313 .flatten()
2314 {
2315 let id = entry.file_name().to_string_lossy().into_owned();
2316 if !crate::run::is_run_id(&id) {
2317 continue;
2318 }
2319 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2322 continue;
2323 };
2324 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2332 continue;
2333 };
2334 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2335 continue;
2336 };
2337 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2338 continue;
2339 }
2340 let daemon_claims = is_working_on(home, &id, now);
2352 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2353 != crate::run::Liveness::Dead
2354 {
2355 continue;
2356 }
2357 state.abandon("daemon");
2358 if let Err(e) = state.save_under(home) {
2359 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2360 continue;
2361 }
2362 if let Some(notice) = notices::run_ended(&state) {
2365 notices::raise_in(home, notice);
2366 }
2367 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2375 tracing::warn!("abandon questions for {id}: {e:#}");
2376 }
2377 abandoned.push(id);
2378 }
2379 abandoned
2380}
2381
2382#[derive(Debug, Clone, Serialize, Deserialize)]
2383struct TaskWorker {
2384 opts: Opts,
2385 queue: PathBuf,
2386 task: String,
2387 #[serde(default)]
2388 lane: PermitKind,
2389 #[serde(default)]
2390 standalone: bool,
2391}
2392
2393fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2394 Ok(crate::worker::active(home)?
2395 .into_iter()
2396 .filter(|job| job.kind == "task")
2397 .collect())
2398}
2399
2400fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2401 task_jobs(home)?
2402 .into_iter()
2403 .filter_map(
2404 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2405 Ok(request) if !request.standalone => Some(Ok(job)),
2406 Ok(_) => None,
2407 Err(error) => Some(Err(error.into())),
2408 },
2409 )
2410 .collect()
2411}
2412
2413#[must_use]
2415pub fn managed(home: &Path) -> bool {
2416 match crate::worker::active(home) {
2417 Err(error) => {
2418 tracing::error!("cannot determine scheduler ownership: {error:#}");
2419 true
2420 }
2421 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2422 Ok(_) => {
2423 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2424 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2425 }
2426 }
2427}
2428
2429pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2431 anyhow::ensure!(
2432 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2433 "the scheduler is stopping; wait for retirement before starting it again"
2434 );
2435 crate::worker::spawn(
2436 home,
2437 "scheduler",
2438 "scheduler",
2439 serde_json::to_value(opts.absolute()?)?,
2440 )
2441 .await
2442}
2443
2444fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2445 home.join("worker-parks").join(format!("{}.json", job.id))
2446}
2447
2448fn request_task_park(
2449 home: &Path,
2450 job: &crate::worker::Job,
2451 cause: crate::run::ParkCause,
2452) -> Result<()> {
2453 let path = park_request_path(home, job);
2454 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2455 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2456 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2457}
2458
2459pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2461 if control == crate::worker::Control::Park {
2462 for job in owned_task_jobs(home)? {
2463 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2464 }
2465 }
2466 crate::worker::request(home, "scheduler", control)
2467}
2468
2469pub async fn restart_detached(home: &Path) -> Result<()> {
2471 let active = crate::worker::active(home)?;
2472 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2473 if prior.is_none() && !managed(home) {
2474 return Ok(());
2475 }
2476 let opts: Opts = match prior {
2477 Some(job) => serde_json::from_value(job.payload)?,
2478 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2479 };
2480 request_control(home, crate::worker::Control::Stop)?;
2481 loop {
2482 let reserved = crate::worker::active(home)?
2483 .iter()
2484 .any(|job| job.kind == "scheduler");
2485 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2486 break;
2487 }
2488 tokio::time::sleep(Duration::from_millis(100)).await;
2489 }
2490 spawn_detached(home, &opts).await?;
2491 Ok(())
2492}
2493
2494pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2496 let opts: Opts = serde_json::from_value(payload)?;
2497 let _ = home;
2498 serve_until(opts, Stop::new()).await
2499}
2500
2501pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2503 let request: TaskWorker = serde_json::from_value(payload)?;
2504 let queue = Queue::at(request.queue);
2505 let key = format!("task:{}", request.task);
2506 let job = task_jobs(home)?
2507 .into_iter()
2508 .find(|job| job.key == key)
2509 .context("task worker reservation is missing")?;
2510 let park_path = park_request_path(home, &job);
2511 let _claim = loop {
2514 match queue.claim(&request.task) {
2515 Ok(claim) => break claim,
2516 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2519 return Err(e.context("claim the task for its worker"));
2520 }
2521 Err(_) => {
2522 sweep_stale_claims(&queue, STALE_CLAIM);
2523 tokio::time::sleep(Duration::from_millis(50)).await;
2524 }
2525 }
2526 };
2527 let mut task = queue.get(&request.task)?;
2528 if !task.status.runnable()
2529 || (!request.standalone
2530 && request.lane
2531 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2532 {
2533 return Ok(());
2534 }
2535 let stop = Stop::new();
2536 let watcher = {
2537 let home = home.to_path_buf();
2538 let stop = stop.clone();
2539 let park_path = park_path.clone();
2540 tokio::spawn(async move {
2541 loop {
2542 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2543 let cause = std::fs::read(&park_path)
2544 .ok()
2545 .and_then(|body| serde_json::from_slice(&body).ok())
2546 .unwrap_or(crate::run::ParkCause::Operator);
2547 stop.park(cause);
2548 return;
2549 }
2550 tokio::time::sleep(Duration::from_millis(200)).await;
2551 }
2552 })
2553 };
2554 let quota = if request.standalone {
2555 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2556 } else {
2557 let status = Arc::new(Mutex::new(Status::new()));
2558 Box::pin(attempt(
2559 &request.opts,
2560 &queue,
2561 &status,
2562 &stop,
2563 crate::graph::Pause::new(),
2564 &mut task,
2565 ))
2566 .await
2567 };
2568 watcher.abort();
2569 let _ = std::fs::remove_file(park_path);
2570 persist_cooldown(home, "a).await?;
2571 Ok(())
2572}
2573
2574pub async fn spawn_claimed(
2577 home: &Path,
2578 opts: &Opts,
2579 queue: &Queue,
2580 task: &Task,
2581 claim: crate::queue::Claim,
2582) -> Result<crate::worker::Job> {
2583 let request = TaskWorker {
2584 opts: opts.absolute()?,
2585 queue: std::path::absolute(queue.root())?,
2586 task: task.id.clone(),
2587 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2588 standalone: true,
2589 };
2590 let job = crate::worker::spawn(
2591 home,
2592 "task",
2593 &format!("task:{}", task.id),
2594 serde_json::to_value(request)?,
2595 )
2596 .await?;
2597 drop(claim);
2598 Ok(job)
2599}
2600
2601pub async fn run_detached_claimed(
2604 opts: &Opts,
2605 queue: &Queue,
2606 task: &mut Task,
2607 claim: crate::queue::Claim,
2608) -> Result<()> {
2609 let home = crate::run::home();
2610 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2611 loop {
2612 crate::worker::active(&home)?;
2615 let current = crate::worker::get(&home, &job.id)?;
2616 if matches!(
2617 current.status,
2618 crate::worker::Status::Finished | crate::worker::Status::Failed
2619 ) {
2620 *task = queue.get(&task.id)?;
2621 anyhow::ensure!(
2622 current.status != crate::worker::Status::Failed,
2623 "task worker failed: {}",
2624 current.result.as_deref().unwrap_or("unknown failure")
2625 );
2626 return Ok(());
2627 }
2628 tokio::time::sleep(Duration::from_millis(200)).await;
2629 }
2630}
2631
2632async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2633 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2634 let _lease = loop {
2635 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2636 break lease;
2637 }
2638 tokio::time::sleep(Duration::from_millis(10)).await;
2639 };
2640 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2641 let path = home.join("task-quota-cooldown.json");
2642 let temporary = path.with_extension("json.tmp");
2643 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2644 std::fs::rename(temporary, path)?;
2645 }
2646 }
2647 Ok(())
2648}
2649
2650fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2651 match std::fs::read(home.join("task-quota-cooldown.json")) {
2652 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2653 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2654 Err(error) => Err(error.into()),
2655 }
2656}
2657
2658pub async fn serve(opts: Opts) -> Result<()> {
2664 tracing::info!(
2665 "automatic update and restart requires magi web; standalone serve does not restart"
2666 );
2667 serve_until(opts, Stop::new()).await
2668}
2669
2670pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2681 let opts = opts.absolute()?;
2682 let home = crate::run::home();
2683 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2684 .context("another scheduler already owns this home")?;
2685 if let Some(reading) = read_status(&home)
2686 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2687 {
2688 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2689 }
2690 if !crate::worker::active(&home)?
2691 .iter()
2692 .any(|job| job.kind == "scheduler")
2693 {
2694 crate::worker::clear_control(&home, "scheduler")?;
2695 }
2696 std::fs::write(
2697 home.join("scheduler-options.json"),
2698 serde_json::to_vec(&opts)?,
2699 )?;
2700 let control = {
2701 let home = home.clone();
2702 let stop = stop.clone();
2703 tokio::spawn(async move {
2704 loop {
2705 match crate::worker::control(&home, "scheduler").ok().flatten() {
2706 Some(crate::worker::Control::Park) => {
2707 let _ = request_control(&home, crate::worker::Control::Park);
2708 stop.park(crate::run::ParkCause::Operator);
2709 break;
2710 }
2711 Some(crate::worker::Control::Stop) => {
2712 stop.stop();
2713 break;
2714 }
2715 None => {}
2716 }
2717 tokio::time::sleep(Duration::from_millis(100)).await;
2718 }
2719 })
2720 };
2721 let signal = {
2722 let stop = stop.clone();
2723 tokio::spawn(async move {
2724 if tokio::signal::ctrl_c().await.is_ok() {
2725 stop.stop();
2726 tracing::info!("scheduler shutdown requested; independent workers continue");
2727 }
2728 })
2729 };
2730
2731 let worktrees_root = opts
2732 .worktrees_root
2733 .clone()
2734 .unwrap_or_else(crate::run::default_worktree_root);
2735 let outcome = drive(
2736 &opts,
2737 &Queue::open(),
2738 &status_path(),
2739 &crate::run::home(),
2740 &worktrees_root,
2741 &stop,
2742 )
2743 .await;
2744
2745 signal.abort();
2746 control.abort();
2747 outcome
2748}
2749
2750async fn drive(
2763 opts: &Opts,
2764 queue: &Queue,
2765 status_file: &Path,
2766 home: &Path,
2767 worktrees_root: &Path,
2768 stop: &Stop,
2769) -> Result<()> {
2770 let recovery_cfg = prepare(&opts.repo, opts).ok();
2778 let daemon_cfg = recovery_cfg
2779 .as_ref()
2780 .map(|c| c.daemon.clone())
2781 .unwrap_or_default();
2782 let reconciled = crate::driver::reconcile(home);
2783 if !reconciled.is_empty() {
2784 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2785 }
2786 let mut initial_status = Status::new();
2787 let owned = owned_task_jobs(home)?;
2788 initial_status.current = current_work(home, Timestamp::now())
2789 .into_iter()
2790 .filter(|current| {
2791 owned
2792 .iter()
2793 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2794 })
2795 .collect();
2796 initial_status.idle = initial_status.current.is_empty();
2797 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2798 recovery_cfg.as_ref(),
2799 daemon_cfg.max_deputies,
2800 ));
2801 let status = Arc::new(Mutex::new(initial_status));
2802 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2803 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2804
2805 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2811
2812 tracing::info!(
2813 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2814 queue.root().display(),
2815 opts.poll.as_secs(),
2816 opts.max_attempts,
2817 concurrency,
2818 if daemon_cfg.pause_for_interrupts {
2819 ", interrupts enabled"
2820 } else {
2821 ""
2822 }
2823 );
2824
2825 let outcome = tokio::select! {
2826 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2827 max_concurrent: concurrency,
2828 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2829 recovery_cfg,
2830 }) => outcome,
2831 () = async {
2832 while !stop.stopped() {
2833 tokio::time::sleep(Duration::from_millis(25)).await;
2834 }
2835 } => {
2836 if stop.parking() {
2837 request_control(home, crate::worker::Control::Park)?;
2838 }
2839 Ok(())
2840 }
2841 };
2842
2843 stop.busy.store(0, Ordering::SeqCst);
2844 beat.abort();
2845 clear_status_at(status_file);
2846 outcome
2847}
2848
2849pub(crate) fn release_watch_settings_from_config(
2852 home: &Path,
2853 repo: &Path,
2854 cfg: &Config,
2855) -> (Vec<PathBuf>, u64) {
2856 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2857 .into_iter()
2858 .map(|r| r.path)
2859 .collect();
2860 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2863 .list()
2864 .into_iter()
2865 .map(|t| t.repo)
2866 .collect();
2867 let runs = home.join("runs");
2868 for id in crate::run::list_ids_in(&runs) {
2869 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2870 .ok()
2871 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2872 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2873 known.extend(repo);
2874 }
2875 for k in known {
2876 let k = k.canonicalize().unwrap_or(k);
2877 if k.join(".git").exists() && !paths.contains(&k) {
2878 paths.push(k);
2879 }
2880 }
2881 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2882 if !paths.contains(&own) {
2883 paths.push(own);
2884 }
2885 (paths, cfg.daemon.release_stall_minutes)
2886}
2887
2888const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2890
2891async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2897 loop {
2898 tokio::time::sleep(HEARTBEAT).await;
2899 let snapshot = {
2900 let mut guard = lock(&status);
2901 guard.updated_at = Timestamp::now();
2902 guard.clone()
2903 };
2904 if let Err(e) = write_status_to(&path, &snapshot) {
2905 tracing::warn!("could not refresh the daemon status file: {e:#}");
2908 }
2909 }
2910}
2911
2912#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2915enum LandResume {
2916 NotLanding,
2919 StillWaiting,
2924 Ready,
2928}
2929
2930fn land_resume_state(task: &Task) -> LandResume {
2934 let Some(run_id) = task.runs.last() else {
2935 return LandResume::NotLanding;
2936 };
2937 let Ok(state) = RunState::load(run_id) else {
2938 return LandResume::NotLanding;
2939 };
2940 if !state.parked {
2941 return LandResume::NotLanding;
2942 }
2943 if state.status == RunStatus::Landing
2944 && state.land_approval.as_ref().is_some_and(|binding| {
2945 ask::Questions::open()
2946 .get(&binding.question)
2947 .is_ok_and(|q| {
2948 q.run == state.id
2949 && q.resolution()
2950 .as_deref()
2951 .is_some_and(crate::merge_decision::stops)
2952 })
2953 })
2954 {
2955 return LandResume::Ready;
2956 }
2957 if !implementers_match(task, &state) {
2958 return LandResume::NotLanding;
2959 }
2960 let node = if state.status == RunStatus::Landing {
2964 land::APPROVAL_NODE
2965 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
2966 crate::github_text::ASK_NODE
2967 } else {
2968 return LandResume::NotLanding;
2969 };
2970 let store = ask::Questions::open();
2971 let waiting = store
2972 .list()
2973 .into_iter()
2974 .filter(|q| &q.run == run_id && q.node == node)
2975 .max_by(|a, b| a.id.cmp(&b.id));
2976 let Some(mut q) = waiting else {
2977 return LandResume::Ready;
2978 };
2979 if !q.status.open() {
2980 if q.resolution().as_deref() == Some(land::REPAIR) {
2983 return if state.liveness(false) == Liveness::Live {
2984 LandResume::StillWaiting
2985 } else {
2986 LandResume::NotLanding
2987 };
2988 }
2989 return LandResume::Ready;
2990 }
2991 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
2998 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
2999 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3000 q.abandon(format!(
3001 "no answer within {}s of asking",
3002 timeout.as_secs().max(1)
3003 ));
3004 if store.put(&mut q).is_ok() {
3007 return LandResume::Ready;
3008 }
3009 }
3010 LandResume::StillWaiting
3011}
3012
3013const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3021
3022const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3034
3035#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3048enum Interrupt {
3049 Idle,
3051 Parking {
3063 parked: Vec<String>,
3064 interrupt_task: String,
3065 },
3066 Running {
3074 parked: Vec<String>,
3075 interrupt_task: String,
3076 },
3077 Resuming { parked: Vec<String> },
3084}
3085
3086fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3107 match state {
3108 Interrupt::Idle => {
3109 if in_flight.len() != 1 {
3121 return Interrupt::Idle;
3122 }
3123 match runnable.iter().find(|t| t.interrupt) {
3124 Some(t) => Interrupt::Parking {
3125 parked: in_flight.to_vec(),
3126 interrupt_task: t.id.clone(),
3127 },
3128 None => Interrupt::Idle,
3129 }
3130 }
3131 Interrupt::Parking {
3132 parked,
3133 interrupt_task,
3134 } => {
3135 if in_flight.iter().any(|id| parked.contains(id)) {
3136 Interrupt::Parking {
3138 parked,
3139 interrupt_task,
3140 }
3141 } else if in_flight.contains(&interrupt_task) {
3142 Interrupt::Running {
3143 parked,
3144 interrupt_task,
3145 }
3146 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3147 Interrupt::Parking {
3151 parked,
3152 interrupt_task,
3153 }
3154 } else {
3155 Interrupt::Resuming { parked }
3160 }
3161 }
3162 Interrupt::Running {
3163 parked,
3164 interrupt_task,
3165 } => {
3166 if in_flight.contains(&interrupt_task) {
3167 Interrupt::Running {
3168 parked,
3169 interrupt_task,
3170 }
3171 } else {
3172 Interrupt::Resuming { parked }
3178 }
3179 }
3180 Interrupt::Resuming { parked } => {
3181 if in_flight.iter().any(|id| parked.contains(id)) {
3182 Interrupt::Idle
3188 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3189 Interrupt::Resuming { parked }
3190 } else {
3191 Interrupt::Idle
3194 }
3195 }
3196 }
3197}
3198
3199fn advance_interrupt_tick(
3205 enabled: bool,
3206 state: Interrupt,
3207 in_flight: &[String],
3208 runnable: &[Task],
3209) -> Interrupt {
3210 if !enabled {
3211 return Interrupt::Idle;
3212 }
3213 advance_interrupt(state, in_flight, runnable)
3214}
3215
3216fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3221 match state {
3222 Interrupt::Idle => candidates,
3223 Interrupt::Parking {
3224 parked,
3225 interrupt_task,
3226 } => {
3227 if in_flight.iter().any(|id| parked.contains(id)) {
3228 Vec::new()
3229 } else {
3230 candidates
3231 .into_iter()
3232 .filter(|t| &t.id == interrupt_task)
3233 .collect()
3234 }
3235 }
3236 Interrupt::Running { .. } => Vec::new(),
3237 Interrupt::Resuming { parked } => candidates
3245 .into_iter()
3246 .find(|t| parked.contains(&t.id))
3247 .into_iter()
3248 .collect(),
3249 }
3250}
3251
3252struct DispatchLimits {
3256 max_concurrent: usize,
3259 pause_for_interrupts: bool,
3261 recovery_cfg: Option<Config>,
3262}
3263
3264#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3273enum PermitKind {
3274 None,
3279 Urgent,
3286 #[default]
3289 Ordinary,
3290}
3291
3292fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3295 if priority {
3296 PermitKind::None
3297 } else if urgent {
3298 PermitKind::Urgent
3299 } else {
3300 PermitKind::Ordinary
3301 }
3302}
3303
3304fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3305 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3306 (0usize, 0usize),
3307 |(ordinary, urgent), request| match request.lane {
3308 PermitKind::Ordinary => (ordinary + 1, urgent),
3309 PermitKind::Urgent => (ordinary, urgent + 1),
3310 PermitKind::None => (ordinary, urgent),
3311 },
3312 );
3313 (
3314 limit.saturating_sub(ordinary),
3315 1usize.saturating_sub(urgent),
3316 )
3317}
3318
3319struct DispatchClaim {
3320 task: Task,
3321 claim: crate::queue::Claim,
3322 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3323}
3324
3325fn claim_dispatch(
3328 queue: &Queue,
3329 candidate: &Task,
3330 cooling_down: bool,
3331 ordinary: &Arc<tokio::sync::Semaphore>,
3332 urgent: &Arc<tokio::sync::Semaphore>,
3333) -> Option<DispatchClaim> {
3334 let Ok(claim) = queue.claim(&candidate.id) else {
3335 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3336 return None;
3337 };
3338 let task = match queue.get(&candidate.id) {
3339 Ok(t) if t.status.runnable() => t,
3340 Ok(_) => return None,
3341 Err(e) => {
3342 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3343 return None;
3344 }
3345 };
3346 let resume = land_resume_state(&task);
3347 if resume == LandResume::StillWaiting {
3348 return None;
3349 }
3350 let priority = resume == LandResume::Ready;
3351 if !priority && cooling_down {
3352 return None;
3353 }
3354 let permit = match permit_kind(priority, task.urgent) {
3357 PermitKind::None => None,
3358 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3359 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3360 };
3361 let stamp = Timestamp::now();
3364 let mut task = task;
3365 task.dispatched_at = Some(stamp);
3366 if let Err(e) = queue.modify(&task.id, |t| {
3367 t.dispatched_at = Some(stamp);
3368 true
3369 }) {
3370 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3371 }
3372 Some(DispatchClaim {
3373 task,
3374 claim,
3375 permit,
3376 })
3377}
3378
3379#[derive(Serialize, Deserialize)]
3398struct ServiceWorker {
3399 opts: Opts,
3400 queue: PathBuf,
3401 worktrees_root: PathBuf,
3402 operation: String,
3403}
3404
3405fn load_conductor(home: &Path) -> Result<Conductor> {
3406 match std::fs::read(home.join("conductor-state.json")) {
3407 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3408 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3409 Err(error) => Err(error.into()),
3410 }
3411}
3412
3413fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3414 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3415 .into_iter()
3416 .map(|c| c.task)
3417 .collect();
3418 let stalled = stalled_tasks(queue, home, Timestamp::now());
3419 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3420 let finished = finished_tasks(queue)
3421 .into_iter()
3422 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3423 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3424 .collect();
3425 let queued = queued_tasks(queue)
3426 .into_iter()
3427 .filter(|t| !live.contains(&t.id))
3428 .collect();
3429 (queued, stalled, finished)
3430}
3431
3432async fn start_service(
3433 home: &Path,
3434 opts: &Opts,
3435 queue: &Queue,
3436 worktrees_root: &Path,
3437 operation: &str,
3438 cfg: Option<&Config>,
3439 busy: bool,
3440) -> Result<()> {
3441 let fetch_cfg = if operation == "fetch" {
3444 match prepare(&opts.repo, opts) {
3445 Ok(cfg) => Some(cfg),
3446 Err(error) => {
3447 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3448 return Ok(());
3449 }
3450 }
3451 } else {
3452 None
3453 };
3454 let cfg = fetch_cfg.as_ref().or(cfg);
3455 let ticks = home.join("service-ticks");
3456 std::fs::create_dir_all(&ticks)?;
3457 let receipt = ticks.join(format!("daemon-{operation}"));
3458 let last = std::fs::read(&receipt)
3459 .ok()
3460 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3461 let interval = match operation {
3462 "disk-pressure" => 30,
3463 "cache" => CACHE_CHECK_INTERVAL_SECS,
3464 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3465 _ => opts.poll.as_secs().max(1),
3466 };
3467 let now = Timestamp::now();
3468 if !cache_check_due(last, now, interval) {
3469 return Ok(());
3470 }
3471 let key = format!("service:daemon-{operation}");
3472 if crate::worker::active(home)?
3473 .iter()
3474 .any(|job| job.key == key)
3475 {
3476 return Ok(());
3477 }
3478 let eligible = match operation {
3479 "conductor" => {
3480 let (queued, stalled, finished) = conductor_work(queue, home);
3481 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3482 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3483 }
3484 "maintenance" => Questions::at(home.join("questions"))
3485 .list()
3486 .iter()
3487 .any(|q| {
3488 q.chosen_action().is_some_and(|action| action.maintenance())
3489 && !home
3490 .join("choice-actions")
3491 .join(format!("{}.json", q.id))
3492 .exists()
3493 }),
3494 "housekeeping" => !busy && runnable(queue).is_empty(),
3495 "cache" => {
3496 !busy
3497 && !runnable(queue).is_empty()
3498 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3499 }
3500 "disk-pressure" => busy,
3501 "fetch" => {
3502 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3503 }
3504 _ => false,
3505 };
3506 if !eligible {
3507 return Ok(());
3508 }
3509 let request = ServiceWorker {
3510 opts: opts.absolute()?,
3511 queue: std::path::absolute(queue.root())?,
3512 worktrees_root: std::path::absolute(worktrees_root)?,
3513 operation: operation.to_owned(),
3514 };
3515 if let Err(error) =
3516 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3517 {
3518 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3519 }
3520 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3521 Ok(())
3522}
3523
3524pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3527 let request: ServiceWorker = serde_json::from_value(payload)?;
3528 let opts = &request.opts;
3529 let queue = Queue::at(request.queue);
3530 let questions = Questions::at(home.join("questions"));
3531 match request.operation.as_str() {
3532 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3533 "conductor" => {
3534 let path = home.join("conductor-state.json");
3535 let mut conductor = load_conductor(home)?;
3536 let (queued, stalled, finished) = conductor_work(&queue, home);
3537 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3538 && conductor.worth_a_look(&queue, &stalled, &finished)
3539 {
3540 let cfg = prepare(&opts.repo, opts)?;
3541 conductor
3542 .maybe_run(
3543 &cfg,
3544 &opts.repo,
3545 &queue,
3546 &questions,
3547 home,
3548 &queued,
3549 &stalled,
3550 &finished,
3551 opts.max_attempts,
3552 )
3553 .await;
3554 let temporary = path.with_extension("json.tmp");
3555 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3556 std::fs::rename(temporary, path)?;
3557 }
3558 }
3559 "fetch" => {
3560 let cfg = prepare(&opts.repo, opts)?;
3561 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3562 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3563 tracing::debug!("fetch origins: {outcome:?}");
3564 }
3565 }
3566 "cache" => {
3567 if task_jobs(home)?.is_empty() {
3568 let mut checked = None;
3569 maybe_prune_cache_between_runs(
3570 &opts.repo,
3571 opts,
3572 home,
3573 &Stop::new(),
3574 &mut checked,
3575 Timestamp::now(),
3576 )
3577 .await;
3578 }
3579 }
3580 "disk-pressure" => {
3581 if let Ok(cfg) = prepare(&opts.repo, opts) {
3582 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3583 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3584 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3585 }
3586 for current in current_work(home, Timestamp::now()) {
3587 if let Ok(state) = RunState::load_under(¤t.run, home)
3588 && let Some(reason) = disk_gate(&state.repo, &state.config)
3589 {
3590 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3591 job.key.strip_prefix("task:") == Some(current.task.as_str())
3592 }) {
3593 request_task_park(
3594 home,
3595 &job,
3596 crate::run::ParkCause::Interrupt {
3597 reason: reason.clone(),
3598 },
3599 )?;
3600 }
3601 notices::raise_in(
3602 home,
3603 Notice::warn(
3604 "disk:pressure",
3605 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3606 ),
3607 );
3608 }
3609 }
3610 }
3611 }
3612 "housekeeping" => {
3613 if task_jobs(home)?.is_empty() {
3614 if let Ok(cfg) = prepare(&opts.repo, opts) {
3615 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3616 }
3617 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3618 resweep_superseded_attempts(&queue, home);
3619 triage_held(&queue, home, opts).await;
3620 }
3621 }
3622 operation => bail!("unknown daemon service {operation}"),
3623 }
3624 Ok(())
3625}
3626
3627fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3631 let key = format!("task:{}", request.task);
3632 let payload = serde_json::to_value(request)?;
3633 if crate::worker::active(home)?
3634 .iter()
3635 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3636 {
3637 tracing::warn!(
3638 "task {} acquired a competing worker reservation: {error:#}",
3639 crate::run::short_of(&request.task)
3640 );
3641 return Ok(());
3642 }
3643 tracing::error!("task worker startup stopped dispatch: {error:#}");
3644 notices::raise_in(
3645 home,
3646 Notice::error(
3647 "loop:worker-startup",
3648 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
3651 ).link(Link::Task { id: request.task.clone() }),
3652 );
3653 Err(error).with_context(|| {
3654 format!(
3655 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3656 crate::run::short_of(&request.task)
3657 )
3658 })
3659}
3660
3661async fn poll(
3662 opts: &Opts,
3663 queue: &Queue,
3664 status: &Arc<Mutex<Status>>,
3665 home: &Path,
3666 worktrees_root: &Path,
3667 stop: &Stop,
3668 limits: DispatchLimits,
3669) -> Result<()> {
3670 let mut attempted = Vec::new();
3671 let interrupt_path = home.join("scheduler-interrupt.json");
3672 let mut interrupt = match std::fs::read(&interrupt_path) {
3673 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3674 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3675 Err(error) => return Err(error.into()),
3676 };
3677 let mut previous = std::collections::BTreeSet::<String>::new();
3678 while !stop.stopped() {
3679 let jobs: Vec<_> = crate::worker::active(home)?
3680 .into_iter()
3681 .filter(|job| job.kind == "task")
3682 .collect();
3683 let requests: Vec<_> = jobs
3684 .iter()
3685 .map(|job| {
3686 serde_json::from_value::<TaskWorker>(job.payload.clone())
3687 .with_context(|| format!("read live task worker {}", job.id))
3688 })
3689 .collect::<Result<_>>()?;
3690 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3691 let owned: std::collections::BTreeSet<_> = requests
3692 .iter()
3693 .filter(|r| !r.standalone)
3694 .map(|r| r.task.clone())
3695 .collect();
3696 let current: Vec<_> = requests
3697 .iter()
3698 .zip(&jobs)
3699 .filter(|(request, _)| !request.standalone)
3700 .map(|(request, job)| Current {
3701 worker: Some(job.id.clone()),
3702 task: request.task.clone(),
3703 run: queue
3704 .get(&request.task)
3705 .ok()
3706 .and_then(|task| task.runs.last().cloned())
3707 .unwrap_or_default(),
3708 })
3709 .collect();
3710 {
3711 let mut snapshot = lock(status);
3712 snapshot.polls += 1;
3713 snapshot.completed += previous.difference(&owned).count();
3714 snapshot.current = current;
3715 snapshot.idle = owned.is_empty();
3716 }
3717 previous = owned.clone();
3718 stop.busy.store(owned.len(), Ordering::SeqCst);
3719
3720 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3723 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3724 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3725 let mut permits = Vec::new();
3726 sweep_stale_claims(queue, STALE_CLAIM);
3727 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3728 reclaim_abandoned_runs(home, Timestamp::now());
3729 let questions = Questions::at(home.join("questions"));
3730
3731 crate::task_chat::sweep(
3734 queue,
3735 &crate::talk::Talks::at(home.join("talks")),
3736 &questions,
3737 home,
3738 );
3739 repair_choice_offers(&questions);
3740 resolve_blockers(queue, &questions);
3741 apply_task_choice_actions(queue, &questions, home);
3742 triage::apply_answers(queue, &questions);
3743 reconcile_task_questions(queue, &questions);
3744 let talks = crate::talk::Talks::at(home.join("talks"));
3745 crate::chat_notify::sweep(queue, &talks, home);
3746 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
3747 crate::chat_watch::sweep(home, &talks);
3748 let mut turns = tokio::task::JoinSet::new();
3749 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3750 while turns.join_next().await.is_some() {}
3752
3753 crate::consult::start_services(
3754 home,
3755 &opts.repo,
3756 limits.recovery_cfg.as_ref(),
3757 opts.config.as_deref(),
3758 )
3759 .await?;
3760 for operation in [
3761 "conductor",
3762 "maintenance",
3763 "housekeeping",
3764 "disk-pressure",
3765 "cache",
3766 "fetch",
3767 ] {
3768 start_service(
3769 home,
3770 opts,
3771 queue,
3772 worktrees_root,
3773 operation,
3774 limits.recovery_cfg.as_ref(),
3775 !live.is_empty(),
3776 )
3777 .await?;
3778 }
3779 let latest = crate::worker::active(home)?;
3780 if latest.iter().any(|job| job.key == "service:daemon-cache")
3781 || latest
3782 .iter()
3783 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3784 {
3785 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3786 continue;
3787 }
3788 let candidates: Vec<Task> = runnable(queue)
3789 .into_iter()
3790 .filter(|task| !live.contains(&task.id))
3791 .filter(|task| !opts.once || !attempted.contains(&task.id))
3792 .collect();
3793 let in_flight: Vec<_> = owned.iter().cloned().collect();
3794 let prior_interrupt = std::mem::discriminant(&interrupt);
3795 interrupt = advance_interrupt_tick(
3796 limits.pause_for_interrupts,
3797 interrupt,
3798 &in_flight,
3799 &candidates,
3800 );
3801 if prior_interrupt != std::mem::discriminant(&interrupt) {
3804 let temporary = interrupt_path.with_extension("json.tmp");
3805 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3806 std::fs::rename(temporary, &interrupt_path)?;
3807 }
3808 if let Interrupt::Parking {
3809 parked,
3810 interrupt_task,
3811 } = &interrupt
3812 {
3813 for job in jobs.iter().filter(|job| {
3814 parked
3815 .iter()
3816 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3817 }) {
3818 if crate::worker::control(home, &job.key)?.is_none() {
3819 request_task_park(
3820 home,
3821 job,
3822 crate::run::ParkCause::Interrupt {
3823 reason: format!(
3824 "task {} asked to run first",
3825 crate::run::short_of(interrupt_task)
3826 ),
3827 },
3828 )?;
3829 }
3830 }
3831 }
3832 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3833 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3834 let mut started = false;
3835 let mut resync = false;
3836 for candidate in candidates {
3837 if stop.stopped() {
3838 break;
3839 }
3840 let Some(DispatchClaim {
3841 task,
3842 claim,
3843 permit,
3844 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3845 else {
3846 continue;
3847 };
3848 if crate::worker::active(home)?
3849 .iter()
3850 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3851 {
3852 drop(claim);
3853 resync = true;
3854 break;
3855 }
3856 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3857 let request = TaskWorker {
3858 opts: opts.absolute()?,
3859 queue: std::path::absolute(queue.root())?,
3860 task: task.id.clone(),
3861 lane,
3862 standalone: false,
3863 };
3864 if let Err(error) = crate::worker::spawn(
3867 home,
3868 "task",
3869 &format!("task:{}", task.id),
3870 serde_json::to_value(&request)?,
3871 )
3872 .await
3873 {
3874 task_worker_spawn_error(home, &request, error)?;
3875 drop(claim);
3876 resync = true;
3877 break;
3878 }
3879 drop(claim);
3880 permits.extend(permit);
3881 attempted.push(task.id);
3882 started = true;
3883 }
3884 if started || resync {
3885 continue;
3886 }
3887 if owned.is_empty() && opts.once {
3888 while crate::worker::active(home)?
3889 .iter()
3890 .any(|job| job.key == "service:daemon-housekeeping")
3891 {
3892 stop.idle(RECHECK_WHILE_BUSY).await;
3893 if stop.stopped() {
3894 return Ok(());
3895 }
3896 }
3897 break;
3898 }
3899 stop.idle(if live.is_empty() {
3900 opts.poll
3901 } else {
3902 RECHECK_WHILE_BUSY.min(opts.poll)
3903 })
3904 .await;
3905 }
3906 stop.busy.store(0, Ordering::SeqCst);
3909 Ok(())
3910}
3911
3912async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
3916 let busy = |error: &anyhow::Error| {
3917 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
3918 error.kind() == std::io::ErrorKind::WouldBlock
3919 || cfg!(windows) && error.raw_os_error() == Some(33)
3920 })
3921 };
3922 match crate::pr_head::owner_lock(home, id) {
3923 Ok(lock) => return Ok(lock),
3924 Err(error) if busy(&error) => {}
3925 Err(error) => return Err(error),
3926 }
3927 let wait = RunState::load_under(id, home)?
3928 .config
3929 .graph
3930 .timeout_resource_wait;
3931 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
3932}
3933
3934async fn public_resume_owner(
3938 home: &Path,
3939 runs: &[String],
3940 repo: &Path,
3941) -> Result<Option<RunState>> {
3942 let mut owner = None;
3943 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3944 let _lock = resume_owner_lock(home, id).await?;
3945 let state = RunState::load_under(id, home)?;
3946 if state.released()
3947 || state.retired_to.is_some()
3948 || state.pr_taken_over_by.is_some()
3949 || !crate::handover::landing_checkpoint(&state)
3950 {
3951 continue;
3952 }
3953 anyhow::ensure!(
3954 crate::git::toplevel(&state.repo).await?.canonicalize()?
3955 == crate::git::toplevel(repo).await?.canonicalize()?,
3956 "public owner repository differs; inspect run lineage"
3957 );
3958 anyhow::ensure!(
3959 owner.is_none(),
3960 "multiple retained public owners; inspect run lineage"
3961 );
3962 anyhow::ensure!(
3963 crate::handover::can_resume(&state),
3964 "public owner cannot resume"
3965 );
3966 let claimed = is_working_on_other(home, id, Timestamp::now())?;
3967 anyhow::ensure!(
3968 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
3969 "public owner's driver is live or unknown; wait before retrying"
3970 );
3971 owner = Some(state);
3972 }
3973 Ok(owner)
3974}
3975
3976async fn attempt(
3982 opts: &Opts,
3983 queue: &Queue,
3984 status: &Arc<Mutex<Status>>,
3985 stop: &Stop,
3986 interrupt_pause: crate::graph::Pause,
3987 task: &mut Task,
3988) -> Vec<QuotaLoss> {
3989 let repo = repo_for(task, &opts.repo);
3990 tracing::info!(
3991 "task {} — {} (repo {})",
3992 task.short(),
3993 task.title,
3994 repo.display()
3995 );
3996
3997 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
3998 Ok(owner) => owner,
3999 Err(e) => {
4000 task.hold_machine(Some(format!(
4001 "Cannot verify existing public owner: {e:#}; no attempt started."
4002 )));
4003 record(queue, task);
4004 return Vec::new();
4005 }
4006 };
4007 let resume_runs = public_owner
4008 .as_ref()
4009 .map(|s| vec![s.id.clone()])
4010 .unwrap_or_else(|| task.runs.clone());
4011
4012 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4016 && previous
4017 .merge_decision
4018 .as_ref()
4019 .is_some_and(|decision| !decision.closed)
4020 && previous.status != RunStatus::Landing
4021 && !crate::merge_decision::resumable_hold(&previous)
4022 {
4023 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4024 record(queue, task);
4025 return Vec::new();
4026 }
4027 let stop_owner = resume_runs
4028 .last()
4029 .and_then(|id| RunState::load(id).ok())
4030 .filter(|state| {
4031 crate::merge_decision::resumable_hold(state)
4032 || state.status == RunStatus::Landing
4033 && state.land_approval.as_ref().is_some_and(|binding| {
4034 Questions::open().get(&binding.question).is_ok_and(|q| {
4035 q.run == state.id
4036 && q.resolution()
4037 .as_deref()
4038 .is_some_and(crate::merge_decision::stops)
4039 })
4040 })
4041 });
4042 if let Some(owner) = stop_owner {
4043 let mut runner = match Runner::resume(&owner.id) {
4044 Ok(runner) => runner,
4045 Err(e) => {
4046 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4047 record(queue, task);
4048 return Vec::new();
4049 }
4050 };
4051 runner.on_pause(stop.pause());
4052 runner.watch_interrupt(interrupt_pause);
4053 let quota_before = runner.state.quota.clone();
4054 task.start(owner.id.clone());
4055 record(queue, task);
4056 lock(status).current.push(Current {
4057 worker: None,
4058 task: task.id.clone(),
4059 run: owner.id,
4060 });
4061 runner.state.next_pass_task_entry = true;
4062 let result = runner.execute().await;
4063 return finish_attempt(
4064 opts.max_attempts,
4065 queue,
4066 task,
4067 &runner.state,
4068 "a_before,
4069 result,
4070 );
4071 }
4072
4073 if let Some(owner) = &public_owner
4074 && (task.fresh_start || !may_resume_run(owner, task))
4075 {
4076 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4077 record(queue, task);
4078 return Vec::new();
4079 }
4080
4081 let unfinished = (!task.fresh_start)
4082 .then(|| unfinished_run(&resume_runs, task.short()))
4083 .flatten()
4084 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4085 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4086 if !task.fresh_start
4087 && task.pr_recovery.is_none()
4088 && task.review_branch.is_none()
4089 && task.resume_override.is_none()
4090 && previous
4091 .as_ref()
4092 .is_some_and(|run| run.retired_to.is_some())
4093 && unfinished.is_none()
4094 {
4095 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4096 record(queue, task);
4097 return Vec::new();
4098 }
4099
4100 let saved_recovery = previous.as_ref().filter(|run| {
4101 task.replacement.is_none()
4102 && task.pr_recovery.as_ref().is_some_and(|selected| {
4103 selected.check_task(&task.id, &task.instruction).is_ok()
4104 && selected_starter(selected, Some(run), unfinished.as_deref())
4105 == Starter::Resume(run.id.clone())
4106 })
4107 });
4108
4109 let explicit = match &task.overrides {
4113 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4114 Some(_) => opts.config.as_deref(),
4115 None => opts.config.as_deref(),
4116 };
4117 let (mut config, layers) = if let Some(run) = saved_recovery {
4118 let mut saved = run.clone();
4121 if let Err(e) = saved.configure_recovery_resume() {
4122 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4123 record(queue, task);
4124 return Vec::new();
4125 }
4126 (saved.config, Vec::new())
4127 } else {
4128 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4129 Ok(loaded) => loaded,
4130 Err(e) => {
4131 let reason = format!("[config] {e:#}");
4132 record_config_failure(task, &e);
4133 task.last_error = Some(reason.clone());
4134 task.hold_machine(Some(reason));
4135 record(queue, task);
4136 return Vec::new();
4137 }
4138 };
4139 if let Err(e) = apply_options(&mut config, opts, task) {
4140 task.hold_manual(Some(format!(
4141 "[config] {e:#}; correct the merge override before releasing the task"
4142 )));
4143 record(queue, task);
4144 return Vec::new();
4145 }
4146 if let Some(selected) = &task.pr_recovery
4147 && let Err(e) = selected.configure(&mut config)
4148 {
4149 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4150 record(queue, task);
4151 return Vec::new();
4152 }
4153 (config, layers)
4154 };
4155 let config_evidence = crate::pr_recovery::ConfigEvidence {
4156 layers,
4157 overrides: task.overrides.clone(),
4158 daemon_merge: task
4159 .overrides
4160 .as_ref()
4161 .is_none_or(|o| o.inherit_daemon_options)
4162 .then(|| opts.merge.clone())
4163 .flatten(),
4164 };
4165 apply_task_implementers(&mut config, task);
4166 if let Some(replacement) = &task.replacement {
4167 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4168 old.liveness(false) == Liveness::Dead
4169 && old.merge_decision.as_ref().is_some_and(|d| {
4170 d.closed
4171 && d.question == replacement.question
4172 && d.replacement_run.as_ref().is_none_or(|run| {
4173 task.runs.last() == Some(run)
4174 && RunState::load(run).is_ok_and(|current| {
4175 current.replaces.as_ref().is_some_and(|pin| {
4176 pin.question == replacement.question
4177 && pin.agent == replacement.agent
4178 })
4179 })
4180 })
4181 })
4182 });
4183 if !safe {
4184 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4185 record(queue, task);
4186 return Vec::new();
4187 }
4188 match config.agent(&replacement.agent) {
4189 Ok(agent) if crate::agent::installed(agent) => {}
4190 _ => {
4191 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4192 record(queue, task);
4193 return Vec::new();
4194 }
4195 }
4196 config.graph.implementers = 1;
4197 config.graph.land_approval = true;
4198 config.merge.mode = MergeMode::Pr;
4199 config.graph.selected_implementer = Some(replacement.agent.clone());
4200 }
4201 let p = phrases(&config.graph.language);
4202 let start_failed = p.could_not_start;
4203 if let Ok(base) =
4208 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4209 .await
4210 && let Ok(head) =
4211 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4212 {
4213 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4214 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4215 let source = crate::git::rev_parse(&repo, branch)
4216 .await
4217 .unwrap_or_default();
4218 inputs.push_str(&format!(";review:{branch}:{source}"));
4219 }
4220 if let Some(reason) = task.recovery_exhausted(
4221 &head,
4222 &inputs,
4223 task.attempt_limit
4224 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4225 ) {
4226 task.hold_unrequested_retry(Some(reason));
4227 record(queue, task);
4228 return Vec::new();
4229 }
4230 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4231 }
4232
4233 if let Some(reason) = disk_gate(&repo, &config) {
4241 task.notice_cause("disk-space", None, None);
4242 task.last_error = Some(reason.clone());
4243 task.hold_machine(Some(reason.clone()));
4244 record(queue, task);
4245 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4246 notices::raise(
4249 Notice::warn(
4250 &format!("disk:{}", repo.display()),
4251 format!("Task {} held for want of free disk space: {reason}. Free some, then release it from the queue.", task.id),
4252 )
4253 .link(Link::Task {
4254 id: task.id.clone(),
4255 }),
4256 );
4257 return Vec::new();
4258 }
4259
4260 let review_branch = take_review_branch(task, previous.as_ref());
4285 let branch_exists = match &review_branch {
4286 Some(branch) => crate::git::branch_exists(&repo, branch)
4287 .await
4288 .unwrap_or(false),
4289 None => false,
4290 };
4291 let proposed = review_retry_starter(
4295 task,
4296 previous.as_ref(),
4297 review_branch.as_deref(),
4298 branch_exists,
4299 unfinished.as_deref(),
4300 );
4301 let retry_selected = task.replacement.is_some()
4302 && task.resumption.is_some()
4303 && !task.fresh_start
4304 && previous.as_ref().is_some_and(|run| {
4305 run.replaces.is_some()
4306 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4307 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4308 });
4309 let stop_pin = task
4310 .resume_override
4311 .as_ref()
4312 .filter(|pin| {
4313 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4314 q.node == land::APPROVAL_NODE
4315 && q.resolution()
4316 .as_deref()
4317 .is_some_and(crate::merge_decision::stops)
4318 })
4319 })
4320 .and_then(|pin| pin.pinned_run.clone());
4321 let starter = if let Some(owner) = &public_owner {
4322 Starter::Resume(owner.id.clone())
4323 } else if let Some(run) = stop_pin {
4324 Starter::Resume(run)
4327 } else if retry_selected {
4328 Starter::Resume(previous.as_ref().unwrap().id.clone())
4329 } else if let Some(selected) = &task.pr_recovery {
4330 if selected.check_task(&task.id, &task.instruction).is_ok() {
4331 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4332 } else {
4333 Starter::Review(selected.branch.clone())
4334 }
4335 } else {
4336 starter_with_implementers(task, previous.as_ref(), proposed)
4337 };
4338 if let Some(pin) = &task.resume_override
4341 && let Some(run) = &pin.pinned_run
4342 && starter != Starter::Resume(run.clone())
4343 {
4344 let run = run.clone();
4345 task.notice_cause("resume-owner", Some(&run), None);
4346 task.hold_machine(Some(format!(
4347 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4348 task.id
4349 )));
4350 record(queue, task);
4351 return Vec::new();
4352 }
4353 if task.replacement.is_some()
4354 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4355 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4356 {
4357 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4358 record(queue, task);
4359 return Vec::new();
4360 }
4361 if let Some(reason) = retained_attempt_hold(
4362 task,
4363 &starter,
4364 config.disk.retained_attempt_limit,
4365 RunState::load,
4366 ) {
4367 if let Starter::Review(branch) = &starter {
4370 task.review_branch = Some(branch.clone());
4371 }
4372 task.hold_unrequested_retry(Some(reason));
4373 record(queue, task);
4374 return Vec::new();
4375 }
4376 let attachments = match task_attachments(queue, task) {
4381 Ok(a) => a,
4382 Err(e) => {
4383 task.attempts += 1;
4384 task.fail_startup_target(
4385 "startup-attachments",
4386 e.name.clone(),
4387 format!("{start_failed}{e:#}"),
4388 opts.max_attempts,
4389 );
4390 record(queue, task);
4391 return Vec::new();
4392 }
4393 };
4394 let startup_pause = stop.pause();
4395 let startup_interrupt = interrupt_pause.clone();
4396 let wait = crate::repo_lock::WaitPolicy::new(
4397 Duration::from_secs(config.graph.timeout_resource_wait),
4398 move || startup_pause.parked() || startup_interrupt.parked(),
4399 );
4400 let started = wait.scope(Box::pin(async { match &starter {
4403 Starter::Review(_) if task.pr_recovery.is_some() => {
4404 let selected = task.pr_recovery.clone().expect("selected recovery");
4405 match selected.check_task(&task.id, &task.instruction) {
4406 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4407 .await
4408 .map(|mut r| {
4409 r.state.attachments = attachments.clone();
4410 r
4411 }),
4412 Err(e) => Err(e),
4413 }
4414 }
4415 Starter::Review(branch) => {
4416 tracing::info!(
4417 "task {} reopens `{branch}` as a review-only pass",
4418 task.short()
4419 );
4420 let takeover = crate::handover::Takeover {
4423 earlier: task.earlier_attempts().to_vec(),
4424 home: crate::run::home(),
4425 choice: take_divergence_answer(branch, &config.merge.remote, task),
4426 task_instruction: Some(instruction_for(task)),
4427 };
4428 Runner::review_taking_over(
4429 &repo,
4430 branch,
4431 config,
4432 Some(takeover),
4433 crate::run::Origin::queue(&task.id),
4434 )
4435 .await
4436 }
4437 Starter::Resume(id) => {
4438 tracing::info!("resuming run {id} rather than competing again");
4439 Runner::resume(id).and_then(|mut r| {
4440 if retry_selected {
4441 for candidate in &mut r.state.candidates {
4444 candidate.failed = None;
4445 candidate.empty = false;
4446 candidate.commits = 0;
4447 }
4448 r.state.status = RunStatus::Implementing;
4449 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4450 }
4451 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4452 r.state.sources = task.sources.clone();
4453 }
4454 if r.state.prep_failure.is_some() {
4455 let approval = task
4456 .resumption
4457 .as_ref()
4458 .context("preparation retry requires an explicit task release")?;
4459 r.retry_preparation(approval.at)?;
4460 }
4461 if let Some(instruction) =
4462 prepare_instruction(&starter, Some(&r.state.instruction), task)
4463 {
4464 r.state.instruction = instruction;
4465 }
4466 r.state.attachments = attachments.clone();
4467 if task.replacement.is_some() {
4468 r.state.config.merge.mode = MergeMode::Pr;
4469 r.state.config.graph.land_approval = true;
4470 }
4471 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4475 .overrides
4476 .as_ref()
4477 .and_then(|o| o.merge.as_deref())
4478 .and_then(|m| merge_mode(m).ok())
4479 {
4480 r.state.config.merge.mode = mode;
4481 }
4482 Ok(r)
4483 })
4484 }
4485 Starter::Start => {
4486 if let Some(branch) = &review_branch {
4487 tracing::warn!(
4488 "conductor chose review for task {} but branch `{branch}` no longer \
4489 exists; requeuing as a fresh competition instead",
4490 task.short()
4491 );
4492 }
4493 let instruction = prepare_instruction(&starter, None, task)
4494 .unwrap_or_else(|| task.instruction.clone());
4495 Runner::start_naming(
4496 &repo,
4497 instruction,
4498 &task.title,
4499 config,
4500 crate::run::Origin::queue(&task.id),
4501 )
4502 .await
4503 .map(|mut r| {
4504 r.state.attachments = attachments.clone();
4505 r.state.sources = task.sources.clone();
4506 r
4507 })
4508 }
4509 }})).await;
4510 let mut runner = match started {
4511 Ok(r) => r,
4512 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4513 task.last_error = Some(e.to_string());
4514 if stop.parking() || interrupt_pause.parked() {
4515 let attempts = task.attempts;
4516 task.park(e.to_string());
4517 task.attempts = attempts;
4519 } else {
4520 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4521 }
4522 record(queue, task);
4523 return Vec::new();
4524 }
4525 Err(e) if task.pr_recovery.is_some() => {
4529 task.startup_notice_cause("startup-public-head");
4530 task.attempts += 1;
4531 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4532 task.link_run(&partial.run);
4533 task.record_pass(&partial.run, 0, task.attempts, None);
4534 }
4535 task.last_error = Some(format!("{start_failed}{e:#}"));
4536 task.hold_unrequested_retry(Some(format!(
4537 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4538 )));
4539 record(queue, task);
4540 return Vec::new();
4541 }
4542 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4543 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4544 Some(r) => (p.handover_refused)(r),
4545 None => format!("{e:#}"),
4546 };
4547 task.startup_notice_cause("startup-handover");
4548 let reason = format!("{start_failed}{detail}");
4549 task.last_error = Some(reason.clone());
4550 let branch = match &starter {
4553 Starter::Review(branch) => Some(branch.clone()),
4554 _ => None,
4555 };
4556 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4560 record(queue, task);
4561 tracing::warn!(
4562 "holding {} for a branch it cannot take over: {e:#}",
4563 task.short()
4564 );
4565 return Vec::new();
4566 }
4567 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4572 let d = e
4573 .downcast_ref::<crate::reconcile::Diverged>()
4574 .expect("checked by the guard");
4575 let mut q = ask::Question::new(
4576 task.id.clone(),
4577 crate::reconcile::NODE.to_owned(),
4578 crate::reconcile::SEAT.to_owned(),
4579 d.summary(),
4580 d.detail(),
4581 d.choices(),
4582 );
4583 match Questions::open().put(&mut q) {
4584 Ok(()) => {
4585 task.last_error = Some(format!("{e:#}"));
4586 task.review_branch = Some(d.branch.clone());
4589 task.review_instruction = Some(task.instruction.clone());
4590 task.block(vec![q.id.clone()], Some(d.summary()));
4591 }
4592 Err(put) => {
4593 tracing::warn!("could not file the divergence question: {put:#}");
4594 task.attempts += 1;
4595 task.fail_startup(
4596 "startup-question-store",
4597 format!("{start_failed}{e:#}"),
4598 opts.max_attempts,
4599 );
4600 }
4601 }
4602 record(queue, task);
4603 return Vec::new();
4604 }
4605 Err(e) => {
4606 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4612 && let Starter::Review(branch) = &starter
4613 {
4614 task.review_branch = Some(branch.clone());
4615 task.review_instruction = Some(task.instruction.clone());
4616 task.last_error = Some(format!("{start_failed}{e:#}"));
4617 task.status = crate::queue::TaskStatus::Failed;
4618 record(queue, task);
4619 return Vec::new();
4620 }
4621 task.attempts += 1;
4622 task.last_error = Some(format!("{start_failed}{e:#}"));
4623 task.startup_notice_cause("startup-source");
4624 task.hold_unrequested_retry(Some(format!(
4625 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4626 )));
4627 record(queue, task);
4628 return Vec::new();
4629 }
4630 };
4631 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4632 runner.state.recovery_config = Some(config_evidence);
4633 }
4634 if let Some(replacement) = &task.replacement {
4635 if matches!(starter, Starter::Start) {
4636 runner.state.instruction.push_str(&format!(
4637 "\n\n# Replacement review context\n{}",
4638 replacement.context
4639 ));
4640 }
4641 runner.state.replaces = Some(replacement.clone());
4642 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4643 if let Ok(mut old) = RunState::load(&replacement.run)
4644 && let Some(d) = &mut old.merge_decision
4645 {
4646 d.replacement_run = Some(runner.state.id.clone());
4647 old.event(
4648 "replacement",
4649 format!(
4650 "replacement run {} owns the next attempt; old PR remains closed",
4651 runner.state.id
4652 ),
4653 );
4654 if let Err(e) = old.save() {
4655 task.hold_manual(Some(format!(
4656 "Could not link replacement history: {e}; no agents dispatched"
4657 )));
4658 record(queue, task);
4659 return Vec::new();
4660 }
4661 }
4662 }
4663 runner.state.task_instruction = Some(task.instruction.clone());
4664 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4667 if runner.state.origin_chat.is_none() {
4668 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4669 }
4670 runner.on_pause(stop.pause());
4672 runner.watch_interrupt(interrupt_pause);
4676
4677 let run = runner.state.id.clone();
4680 task.start(run.clone());
4681 record(queue, task);
4682 lock(status).current.push(Current {
4683 worker: None,
4684 task: task.id.clone(),
4685 run,
4686 });
4687
4688 let quota_before = runner.state.quota.clone();
4691 runner.state.next_pass_task_entry = true;
4692 let result = runner.execute().await;
4693 finish_attempt(
4694 opts.max_attempts,
4695 queue,
4696 task,
4697 &runner.state,
4698 "a_before,
4699 result,
4700 )
4701}
4702
4703pub fn finish_attempt(
4708 max_attempts: usize,
4709 queue: &Queue,
4710 task: &mut Task,
4711 state: &RunState,
4712 quota_before: &[QuotaLoss],
4713 result: Result<()>,
4714) -> Vec<QuotaLoss> {
4715 let detail = match result {
4716 Ok(()) => describe(state),
4717 Err(e) => format!("{e:#}"),
4718 };
4719 let fresh = losses_this_attempt(quota_before, &state.quota);
4720 let verdict = Verdict {
4721 status: state.status,
4722 left_pr: state.pr.is_some(),
4725 quota_hit: !fresh.is_empty(),
4731 parked: state.parked,
4735 no_viable_candidates: state.viable().is_empty(),
4738 };
4739 if let Some(replacement) = &state.replaces
4740 && let Some(pr) = &state.pr
4741 && let Ok(mut old) = RunState::load(&replacement.run)
4742 && let Some(d) = &mut old.merge_decision
4743 {
4744 d.replacement_pr = Some(pr.url.clone());
4745 let _ = old.save();
4746 }
4747 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4748 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4749 task.hold_unrequested_retry(Some(format!(
4750 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4751 task.short()
4752 )));
4753 }
4754 if task.status == TaskStatus::Done {
4755 supersede_prior_runs(task, &crate::run::home());
4756 }
4757 record(queue, task);
4758 tracing::info!(
4759 "task {} is {} after run {} ({})",
4760 task.short(),
4761 task.status.as_str(),
4762 state.short(),
4763 label(state.status)
4764 );
4765 fresh
4766}
4767
4768pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4772 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4775 && !awaiting_resume_with(task, RunState::load)
4776 && withheld_text_wait(task).is_none()
4780 && !(task.fresh_start && task.replacement.is_some())
4781 {
4782 let why = task.last_error.clone().map_or_else(
4783 || "the run did not finish".to_owned(),
4784 |e| format!("the run did not finish: {e}"),
4785 );
4786 task.hold_unrequested_retry(Some(format!(
4787 "{why}. It was started by hand, so it is not retried \
4788 automatically; `magi task release` retries it."
4789 )));
4790 record(queue, task);
4791 }
4792}
4793
4794#[must_use]
4798pub fn loop_would_resume(run: &str) -> bool {
4799 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4800}
4801
4802#[must_use]
4812pub fn foreign_loop(
4813 reading: Option<&Reading>,
4814 now: Timestamp,
4815 own_pid: u32,
4816) -> Option<Option<u32>> {
4817 let reading = reading.filter(|r| r.running(now))?;
4818 match reading.pid {
4819 Some(pid) if pid == own_pid => None,
4820 pid => Some(pid),
4821 }
4822}
4823
4824fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4825 reading.worker_protocol != Some(1)
4826 && (foreign_loop(Some(reading), now, own_pid).is_some()
4827 || reading
4828 .pid
4829 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4830}
4831
4832pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4843 run_claimed_until(opts, queue, task, &Stop::new()).await;
4844}
4845
4846async fn run_claimed_until(
4847 opts: &Opts,
4848 queue: &Queue,
4849 task: &mut Task,
4850 stop: &Stop,
4851) -> Vec<QuotaLoss> {
4852 let status = Arc::new(Mutex::new(Status::new()));
4853 let mut quota = Box::pin(attempt(
4856 opts,
4857 queue,
4858 &status,
4859 stop,
4860 crate::graph::Pause::new(),
4861 task,
4862 ))
4863 .await;
4864 let mut resumes = 0;
4873 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
4874 let Some(timeout) = withheld_text_wait(task) else {
4875 break;
4876 };
4877 eprintln!(
4878 "waiting for the owner's answer on a withheld pull request text \
4879 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4880 );
4881 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4884 let ready = loop {
4885 match land_resume_state(task) {
4886 LandResume::StillWaiting
4887 if !stop.parking() && std::time::Instant::now() < limit =>
4888 {
4889 tokio::time::sleep(WITHHELD_POLL).await;
4890 }
4891 LandResume::Ready => break true,
4892 _ => break false,
4893 }
4894 };
4895 if !ready || stop.parking() {
4896 break;
4897 }
4898 resumes += 1;
4899 quota.extend(
4900 Box::pin(attempt(
4901 opts,
4902 queue,
4903 &status,
4904 stop,
4905 crate::graph::Pause::new(),
4906 task,
4907 ))
4908 .await,
4909 );
4910 }
4911 hold_if_runnable(queue, task);
4912 quota
4913}
4914
4915const MAX_WITHHELD_RESUMES: u32 = 3;
4918
4919const WITHHELD_POLL: Duration = Duration::from_secs(15);
4921
4922fn withheld_text_wait(task: &Task) -> Option<u64> {
4925 let id = task.runs.last()?;
4926 let s = RunState::load(id).ok()?;
4927 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4928 .then_some(s.config.graph.answer_timeout)
4929}
4930
4931fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4941 after
4942 .iter()
4943 .filter(|q| !before.contains(q))
4944 .cloned()
4945 .collect()
4946}
4947
4948fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4951 if quota.is_empty() {
4952 return None;
4953 }
4954 let with_hint = quota.iter().find(|q| q.reset.is_some());
4955 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4956 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4957 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4958 Some(
4959 now.checked_add(jiff::SignedDuration::from_secs(secs))
4960 .unwrap_or(Timestamp::MAX),
4961 )
4962}
4963
4964fn implementers_match(task: &Task, state: &RunState) -> bool {
4966 task.replacement.as_ref().map_or_else(
4967 || {
4968 task.implementers()
4969 .is_none_or(|n| n == state.config.graph.implementers)
4970 },
4971 |replacement| {
4972 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
4973 && state.config.graph.implementers == 1
4974 },
4975 )
4976}
4977
4978fn starter_with_implementers(
4980 task: &Task,
4981 previous: Option<&RunState>,
4982 proposed: Starter,
4983) -> Starter {
4984 if previous.is_some_and(|s| !implementers_match(task, s)) {
4985 task.review_of
4986 .as_ref()
4987 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
4988 } else {
4989 proposed
4990 }
4991}
4992
4993fn apply_task_implementers(config: &mut Config, task: &Task) {
4995 if let Some(n) = task.implementers() {
4996 config.graph.implementers = n;
4997 }
4998}
4999
5000#[cfg(test)]
5005fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5006 let explicit = match &task.overrides {
5007 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5008 _ => opts.config.as_deref(),
5009 };
5010 let (mut config, _) = Config::discover(repo, explicit)?;
5011 apply_options(&mut config, opts, task)?;
5012 Ok(config)
5013}
5014
5015fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5016 if task
5017 .overrides
5018 .as_ref()
5019 .is_none_or(|o| o.inherit_daemon_options)
5020 && let Some(mode) = opts.merge.as_deref()
5021 {
5022 config.merge.mode = merge_mode(mode)?;
5023 }
5024 if let Some(o) = &task.overrides {
5025 o.apply(config);
5026 }
5027 Ok(())
5028}
5029
5030fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5032 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5033 if let Some(mode) = &opts.merge {
5034 config.merge.mode = merge_mode(mode)?;
5035 }
5036 Ok(config)
5037}
5038
5039async fn maybe_prune_cache_between_runs(
5044 repo: &Path,
5045 opts: &Opts,
5046 home: &Path,
5047 stop: &Stop,
5048 last_checked: &mut Option<Timestamp>,
5049 now: Timestamp,
5050) {
5051 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5052 return;
5053 }
5054 *last_checked = Some(now);
5055 let cfg = match prepare(repo, opts) {
5056 Ok(cfg) => cfg,
5057 Err(e) => {
5058 tracing::warn!("cache check: no config: {e:#}");
5059 return;
5060 }
5061 };
5062 match clean::prune_cache_if_over_limit(&cfg, home) {
5063 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5064 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5065 pruned.files,
5066 pruned.freed
5067 ),
5068 Ok(_) => {}
5069 Err(e) => {
5070 tracing::warn!("housekeep: prune cache: {e:#}");
5071 notices::raise_in(
5072 home,
5073 Notice::warn(
5074 "housekeep:cache",
5075 format!(
5076 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5077 e.to_string()
5078 .lines()
5079 .next()
5080 .unwrap_or("unknown cleanup error")
5081 ),
5082 ),
5083 );
5084 }
5085 }
5086}
5087
5088fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5092 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5093}
5094
5095async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5114 let cfg = match prepare(repo, opts) {
5115 Ok(cfg) => cfg,
5116 Err(e) => {
5117 tracing::warn!("housekeep: no config: {e:#}");
5118 return;
5119 }
5120 };
5121 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5130 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5131 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5136 let mut extra = Vec::new();
5137 if out.unreadable > 0 {
5138 extra.push(format!("{} unreadable", out.unreadable));
5139 }
5140 if out.orphaned_worktrees > 0 {
5141 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5142 }
5143 let detail = if extra.is_empty() {
5144 String::new()
5145 } else {
5146 format!(" ({})", extra.join(", "))
5147 };
5148 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5149 }
5150 if out.external_merges_recorded > 0 {
5151 tracing::info!(
5152 "housekeep: recorded {} run(s) as merged externally",
5153 out.external_merges_recorded
5154 );
5155 }
5156 if out.stale_pr_states_repaired > 0 {
5157 tracing::info!(
5158 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5159 out.stale_pr_states_repaired
5160 );
5161 }
5162 if out.cache_files > 0 {
5163 tracing::info!(
5164 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5165 out.cache_files,
5166 out.cache_freed
5167 );
5168 }
5169 if out.questions_abandoned > 0 {
5170 tracing::info!(
5171 "housekeep: abandoned {} question(s) left open by a finished run",
5172 out.questions_abandoned
5173 );
5174 }
5175}
5176
5177async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5186 let questions = Questions::at(home.join("questions"));
5187 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5188 if report.is_empty() {
5189 return;
5190 }
5191 if !report.quarantined.is_empty() {
5192 tracing::info!(
5193 "triage: held {} blocked task(s) whose blocked-on task or \
5194 question no longer exists: {}",
5195 report.quarantined.len(),
5196 report.quarantined.join(", ")
5197 );
5198 }
5199 if !report.resumed.is_empty() {
5200 tracing::info!(
5201 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5202 report.resumed.len(),
5203 report.resumed.join(", ")
5204 );
5205 }
5206 if !report.asked.is_empty() {
5207 tracing::info!(
5208 "triage: asked about {} held task(s): {}",
5209 report.asked.len(),
5210 report.asked.join(", ")
5211 );
5212 }
5213 if !report.answered.is_empty() {
5214 tracing::info!(
5215 "triage: applied {} operator answer(s): {}",
5216 report.answered.len(),
5217 report.answered.join(", ")
5218 );
5219 }
5220}
5221
5222pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5229 disk_gate_with(repo, config, crate::disk::free_bytes)
5230}
5231
5232fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5236 repo: &Path,
5237 config: &Config,
5238 free_bytes: F,
5239) -> Option<String> {
5240 let min = config.disk.min_free_bytes;
5241 if min == 0 {
5242 return None;
5243 }
5244 let mut paths = vec![repo.to_owned()];
5245 if let Some(cache) = config.cache_dir() {
5246 paths.push(cache);
5247 }
5248 if let Some(root) = &config.graph.worktree_root {
5249 paths.push(root.clone());
5250 }
5251 for path in paths {
5252 let probe = if path == repo {
5253 path.as_path()
5254 } else {
5255 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5256 };
5257 let reason = match free_bytes(probe) {
5258 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5259 Err(e) => Some(crate::disk::unmeasured_in(
5260 probe,
5261 &e,
5262 &config.graph.language,
5263 )),
5264 };
5265 if reason.is_some() {
5266 return reason;
5267 }
5268 }
5269 None
5270}
5271
5272const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5279
5280const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5284
5285fn quota_wait(
5294 reset_at: Option<Timestamp>,
5295 now: Timestamp,
5296 fallback: Duration,
5297 cap: Duration,
5298) -> Duration {
5299 match reset_at {
5300 Some(at) if at > now => {
5301 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5302 Duration::from_secs(secs).min(cap)
5303 }
5304 _ => fallback,
5305 }
5306}
5307
5308fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5322 parse_reset_hint_zoned(text, now)
5323 .or_else(|| parse_reset_hint_dated(text))
5324 .or_else(|| parse_reset_hint_relative(text, recorded))
5325}
5326
5327fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5331 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5332 let mut rest = rest.trim();
5333 if rest.is_empty() {
5334 return None;
5335 }
5336 let mut total: i64 = 0;
5337 let mut matched = false;
5338 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5339 if let Some((digits, tail)) = rest.split_once(unit)
5340 && !digits.is_empty()
5341 && digits.bytes().all(|b| b.is_ascii_digit())
5342 {
5343 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5344 rest = tail;
5345 matched = true;
5346 }
5347 }
5348 if !rest.is_empty() || !matched {
5349 return None;
5350 }
5351 recorded
5352 .checked_add(jiff::SignedDuration::from_secs(total))
5353 .ok()
5354}
5355
5356fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5360 let clock = clock.trim().to_lowercase();
5361 let (digits, pm) = clock
5362 .strip_suffix("am")
5363 .map(|d| (d, false))
5364 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5365 let (h, m) = digits.trim().split_once(':')?;
5366 let mut hour: i8 = h.trim().parse().ok()?;
5367 let minute: i8 = m.trim().parse().ok()?;
5368 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5369 return None;
5370 }
5371 if pm && hour != 12 {
5372 hour += 12;
5373 } else if !pm && hour == 12 {
5374 hour = 0;
5375 }
5376 Some((hour, minute))
5377}
5378
5379fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5384 let open = text.find('(')?;
5385 let close = text.rfind(')')?;
5386 if close <= open {
5387 return None;
5388 }
5389 let zone = text[open + 1..close].trim();
5390 let (hour, minute) = parse_12h_clock(&text[..open])?;
5391 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5392 let candidate = now
5393 .to_zoned(tz)
5394 .with()
5395 .hour(hour)
5396 .minute(minute)
5397 .second(0)
5398 .millisecond(0)
5399 .microsecond(0)
5400 .nanosecond(0)
5401 .build()
5402 .ok()?;
5403 let mut at = candidate.timestamp();
5404 if at <= now {
5405 at += jiff::SignedDuration::from_hours(24);
5406 }
5407 Some(at)
5408}
5409
5410fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5419 let words: Vec<&str> = text.split_whitespace().collect();
5420 if words.len() < 5 {
5421 return None;
5422 }
5423 (0..=words.len() - 5)
5424 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5425}
5426
5427fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5433 if trailing.is_some_and(|next| next.starts_with('(')) {
5434 return None;
5435 }
5436 let month = month_number(window[0])?;
5437 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5438 let day_digits = ["st", "nd", "rd", "th"]
5439 .iter()
5440 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5441 let day: i8 = day_digits.parse().ok()?;
5442 let year_token = window[2];
5443 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5444 return None;
5445 }
5446 let year: i16 = year_token.parse().ok()?;
5447 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5451 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5452 let date = jiff::civil::Date::new(year, month, day).ok()?;
5453 let candidate = date
5454 .at(hour, minute, 0, 0)
5455 .to_zoned(jiff::tz::TimeZone::UTC)
5456 .ok()?;
5457 Some(candidate.timestamp())
5458}
5459
5460fn month_number(name: &str) -> Option<i8> {
5463 const NAMES: [&str; 12] = [
5464 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5465 ];
5466 let lower = name.to_lowercase();
5467 NAMES
5468 .iter()
5469 .position(|n| *n == lower.as_str())
5470 .map(|i| i as i8 + 1)
5471}
5472
5473#[cfg(test)]
5475fn exhausted_review_budget(state: &RunState) -> bool {
5476 !crate::handover::can_resume(state)
5479 && state.status == RunStatus::Blocked
5480 && !state.released()
5481 && state.retired_to.is_none()
5482}
5483
5484fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5521 unfinished_run_with(runs, short, RunState::load)
5522}
5523
5524pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5527where
5528 F: FnMut(&str) -> Result<RunState>,
5529{
5530 let mut id = runs.last()?.clone();
5531 let mut visited = std::collections::BTreeSet::new();
5532 let mut repo = None;
5533 loop {
5534 if !runs.contains(&id) || !visited.insert(id.clone()) {
5535 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5536 return None;
5537 }
5538 let state = match load(&id) {
5539 Ok(state) if state.id == id => state,
5540 Ok(_) => return None,
5541 Err(e) => {
5542 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5543 return None;
5544 }
5545 };
5546 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5547 return None;
5548 }
5549 repo = Some(state.repo.clone());
5550 if let Some(authority) = state.retired_to {
5551 id = authority;
5552 continue;
5553 }
5554 return (crate::handover::can_resume(&state)
5555 && state.liveness(false) != crate::run::Liveness::Live)
5556 .then_some(id);
5557 }
5558}
5559
5560fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5568where
5569 F: FnOnce(&str) -> Result<RunState>,
5570{
5571 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5573 || task.fresh_start
5574 || task.review_branch.is_some()
5575 {
5576 return false;
5577 }
5578 let Some(id) = task.runs.last() else {
5579 return false;
5580 };
5581 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5582}
5583
5584#[derive(Debug, Clone, PartialEq, Eq)]
5587enum Starter {
5588 Review(String),
5591 Resume(String),
5593 Start,
5595}
5596
5597fn retained_attempt_hold(
5601 task: &Task,
5602 starter: &Starter,
5603 limit: usize,
5604 mut load: impl FnMut(&str) -> Result<RunState>,
5605) -> Option<String> {
5606 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5607 return None;
5608 }
5609 let mut retained = 0;
5610 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5611 let state = match load(id) {
5612 Ok(state) if state.id == *id => state,
5613 _ => {
5614 return Some(format!(
5615 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5616 ));
5617 }
5618 };
5619 if state.retired_to.is_some() || state.released() {
5620 continue;
5621 }
5622 retained += 1;
5625 }
5626 (retained >= limit).then(|| format!(
5627 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5628 ))
5629}
5630
5631fn selected_starter(
5633 selected: &crate::pr_recovery::Selection,
5634 previous: Option<&RunState>,
5635 unfinished: Option<&str>,
5636) -> Starter {
5637 if let Some(id) = unfinished
5638 && previous.is_some_and(|r| {
5639 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5640 })
5641 {
5642 Starter::Resume(id.to_owned())
5643 } else {
5644 Starter::Review(selected.branch.clone())
5645 }
5646}
5647
5648fn take_divergence_answer(
5652 branch: &str,
5653 remote: &str,
5654 task: &mut Task,
5655) -> Option<crate::reconcile::Choice> {
5656 let summary = crate::reconcile::summary_for(branch, remote);
5657 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5658 (a.question == summary)
5659 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5660 .flatten()
5661 .map(|c| (i, c))
5662 })?;
5663 task.answers.remove(idx);
5664 Some(choice)
5665}
5666
5667fn choose_starter(
5679 review_branch: Option<&str>,
5680 branch_exists: bool,
5681 unfinished: Option<&str>,
5682) -> Starter {
5683 match review_branch {
5684 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5685 Some(_) => Starter::Start,
5686 None => match unfinished {
5687 Some(id) => Starter::Resume(id.to_owned()),
5688 None => Starter::Start,
5689 },
5690 }
5691}
5692
5693fn review_retry_starter(
5697 task: &Task,
5698 previous: Option<&RunState>,
5699 branch: Option<&str>,
5700 exists: bool,
5701 unfinished: Option<&str>,
5702) -> Starter {
5703 if let Some(run) = previous
5704 && unfinished == Some(run.id.as_str())
5705 && run.reviewed_commits.is_some()
5706 && branch.is_some_and(|branch| {
5707 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
5708 })
5709 && may_resume_run(run, task)
5710 {
5711 return Starter::Resume(run.id.clone());
5712 }
5713 match (branch, unfinished, &task.review_of) {
5714 (None, None, Some(branch)) => Starter::Review(branch.clone()),
5715 _ => choose_starter(branch, exists, unfinished),
5716 }
5717}
5718
5719fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5721 let branch = task.review_branch.take();
5722 let instruction = task.review_instruction.take();
5723 branch.filter(|branch| match &instruction {
5724 Some(authored) => *authored == task.instruction,
5725 None => previous.map_or_else(
5728 || task.review_of.as_ref() == Some(branch),
5729 |run| compatible_run_instruction(run, task),
5730 ),
5731 })
5732}
5733
5734fn may_resume_run(run: &RunState, task: &Task) -> bool {
5737 task.resume_override
5738 .as_ref()
5739 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5740 || compatible_run_instruction(run, task)
5741}
5742
5743pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5745 if let Some(authored) = &run.task_instruction {
5746 return compatible_instruction(authored, task);
5747 }
5748 if compatible_instruction(&run.instruction, task) {
5749 return true;
5750 }
5751 if run.reviewed_commits.is_some() {
5754 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5755 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5756 }
5757 if let Some(branch) = &task.review_of {
5758 let canonical = format!(
5759 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5760 );
5761 let generated = format!(
5762 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5763 );
5764 return task.instruction == canonical && run.instruction.starts_with(&generated);
5765 }
5766 }
5767 false
5768}
5769
5770pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5772 strip_answers_block(old, task) == task.instruction
5773}
5774
5775fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5778 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5779 return fallback.to_path_buf();
5780 }
5781 task.repo.clone()
5782}
5783
5784const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5788
5789fn answers_block(task: &Task, count: usize) -> String {
5791 let mut s = ANSWERS_HEADER.to_owned();
5792 for a in &task.answers[..count] {
5793 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5794 }
5795 s
5796}
5797
5798fn resumption_block(r: &crate::queue::Resumption) -> String {
5801 format!(
5802 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5803 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5804 r.generation, r.at, r.instruction, r.answers_through,
5805 )
5806}
5807
5808fn append_answers(base: &str, task: &Task) -> String {
5810 let mut s = base.to_owned();
5811 if !task.answers.is_empty() {
5812 s.push_str(&answers_block(task, task.answers.len()));
5813 }
5814 if let Some(r) = &task.resumption {
5815 s.push_str(&resumption_block(r));
5816 }
5817 s
5818}
5819
5820fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5824 let mut instruction = instruction;
5825 for approval in task
5826 .resumption
5827 .iter()
5828 .chain(task.resumption_history.iter().rev())
5829 {
5830 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5831 instruction = base;
5832 break;
5833 }
5834 }
5835 for count in (1..=task.answers.len()).rev() {
5836 let block = answers_block(task, count);
5837 if let Some(base) = instruction.strip_suffix(&block) {
5838 return base;
5839 }
5840 }
5841 instruction
5842}
5843
5844fn instruction_for(task: &Task) -> String {
5852 append_answers(&task.instruction, task)
5853}
5854
5855fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5867 append_answers(strip_answers_block(old_instruction, task), task)
5868}
5869
5870fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5872 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5873 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5874 if let Some(cause) = task.hold_notice_cause.as_mut() {
5875 cause.target = failure.map(|f| f.target.clone());
5876 }
5877}
5878
5879#[derive(Debug, thiserror::Error)]
5880#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5881struct MissingAttachment {
5882 name: String,
5883 path: PathBuf,
5884}
5885
5886fn task_attachments(
5888 queue: &Queue,
5889 task: &Task,
5890) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5891 let paths = queue.attachment_paths(task);
5892 for (name, path) in task.attachments.iter().zip(&paths) {
5893 if !path.is_file() {
5894 return Err(MissingAttachment {
5895 name: name.clone(),
5896 path: path.clone(),
5897 });
5898 }
5899 }
5900 Ok(paths)
5901}
5902
5903fn prepare_instruction(
5914 starter: &Starter,
5915 old_instruction: Option<&str>,
5916 task: &Task,
5917) -> Option<String> {
5918 match starter {
5919 Starter::Start => Some(instruction_for(task)),
5920 Starter::Resume(_) => Some(resumed_instruction(
5921 old_instruction.expect("a resumed run always has a prior instruction"),
5922 task,
5923 )),
5924 Starter::Review(_) => None,
5925 }
5926}
5927
5928fn record(queue: &Queue, task: &mut Task) {
5932 if let Err(e) = queue.put(task) {
5933 tracing::error!("could not record task {}: {e:#}", task.short());
5934 let notice = Notice::error(
5935 "loop:record",
5936 format!(
5937 "Task {} state could not be saved: {}. Check the disk.",
5938 task.id,
5939 e.to_string().lines().next().unwrap_or("unknown save error")
5940 ),
5941 )
5942 .link(Link::Task {
5943 id: task.id.clone(),
5944 });
5945 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5947 Some(home) => notices::raise_in(home, notice),
5948 None => notices::raise(notice),
5949 }
5950 }
5951}
5952
5953fn runnable(queue: &Queue) -> Vec<Task> {
5959 let mut tasks: Vec<Task> = queue
5960 .list()
5961 .into_iter()
5962 .filter(|t| t.status.runnable())
5963 .collect();
5964 tasks.sort_unstable_by(crate::queue::dispatch_order);
5965 tasks
5966}
5967
5968fn describe(state: &RunState) -> String {
5982 let p = phrases(&state.config.graph.language);
5983 let mut detail = if state.status == RunStatus::Stalled {
5984 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
5985 seats.sort_unstable();
5986 seats.dedup();
5987 if seats.is_empty() {
5988 p.quorum_lost.to_owned()
5989 } else {
5990 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
5991 }
5992 } else {
5993 format!("{}{}", p.run_ended, state.status.display_label())
5994 };
5995 if let Some(last) = state.events.last() {
5996 let unanswered = state
6000 .reviews
6001 .last()
6002 .filter(|r| {
6003 state.status == RunStatus::Blocked
6004 && last.node == "review"
6005 && r.incomplete()
6006 && r.blocking == 0
6007 && r.round == state.config.graph.review_rounds
6008 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6009 })
6010 .map(|r| (r.expected - r.answered, r.round));
6011 match unanswered {
6012 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6013 detail.push_str(&format!(
6014 " ({}: {})",
6015 last.node,
6016 (p.reviewers_never_answered)(missing, rounds)
6017 ));
6018 }
6019 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6020 }
6021 }
6022 detail.push_str(&format!(" [run {}]", state.id));
6023 detail
6024}
6025
6026const DIAGNOSTIC_MAX: usize = 4_000;
6032
6033const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6038
6039fn diagnostic(state: &RunState) -> Option<String> {
6053 let mut parts: Vec<String> = Vec::new();
6054
6055 for o in state.gate.iter().filter(|o| !o.ok()) {
6057 parts.push(format!(
6058 "gate `{}` failed ({}):\n{}",
6059 o.command,
6060 notices::exit_status(o.code),
6061 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6062 ));
6063 }
6064
6065 if let Some(last) = state
6068 .events
6069 .iter()
6070 .rev()
6071 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6072 {
6073 parts.push(last.message.clone());
6074 }
6075
6076 if state.viable().is_empty() {
6083 for c in &state.candidates {
6084 if let Some(evidence) = &c.verified_noop {
6085 parts.push(format!(
6086 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6087 c.label
6088 ));
6089 } else if !c.summary.trim().is_empty() {
6090 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6091 } else if let Some(why) = &c.failed {
6092 parts.push(format!("candidate {}: {why}", c.label));
6093 }
6094 }
6095 }
6096
6097 if parts.is_empty() {
6098 return None;
6099 }
6100 Some(crate::run::tail(
6105 &parts.join("\n\n"),
6106 DIAGNOSTIC_MAX.saturating_sub(100),
6107 ))
6108}
6109
6110fn label(status: RunStatus) -> &'static str {
6118 status.as_str()
6119}
6120
6121pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6123 match mode {
6124 "none" => Ok(MergeMode::None),
6125 "local" => Ok(MergeMode::Local),
6126 "pr" => Ok(MergeMode::Pr),
6127 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6128 }
6129}
6130
6131fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6136 mutex
6137 .lock()
6138 .unwrap_or_else(std::sync::PoisonError::into_inner)
6139}
6140
6141#[cfg(test)]
6142mod tests {
6143 use super::*;
6144 use crate::queue::{Source, TaskStatus};
6145 use crate::run::{Candidate, CommandOutcome};
6146 use pretty_assertions::assert_eq;
6147
6148 fn task() -> Task {
6149 Task::new(
6150 "add retries".to_owned(),
6151 "add retries".to_owned(),
6152 PathBuf::from("/repo"),
6153 Source::Human,
6154 )
6155 }
6156
6157 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6158 let job = crate::worker::Job {
6159 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6160 sequence: 1,
6161 kind: "task".into(),
6162 key: format!("task:{}", request.task),
6163 payload: serde_json::to_value(request).unwrap(),
6164 generation: "a".repeat(64),
6165 version: None,
6166 status: crate::worker::Status::Starting,
6167 result: None,
6168 pid: None,
6169 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6170 finished_at: None,
6171 };
6172 let dir = home.join("workers/jobs").join(&job.id);
6173 std::fs::create_dir_all(&dir).unwrap();
6174 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6175 job
6176 }
6177
6178 #[test]
6179 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6180 let home = tempfile::tempdir().unwrap();
6181 let request = TaskWorker {
6182 opts: Opts::default(),
6183 queue: home.path().join("queue"),
6184 task: "startup".into(),
6185 lane: PermitKind::Ordinary,
6186 standalone: false,
6187 };
6188 let job = starting_task_reservation(home.path(), &request);
6189 let error = task_worker_spawn_error(
6190 home.path(),
6191 &request,
6192 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6193 )
6194 .expect_err("our own reservation must not turn a failed spawn into a resync");
6195 let active = crate::worker::active(home.path()).unwrap();
6196 assert_eq!(active.len(), 1);
6197 assert_eq!(active[0].id, job.id);
6198 assert_eq!(active[0].status, crate::worker::Status::Starting);
6199 assert!(
6200 crate::worker::control(home.path(), &job.key)
6201 .unwrap()
6202 .is_none()
6203 );
6204 let notices = notices::Notices::at(home.path().join("notifications")).list();
6205 assert_eq!(notices.len(), 1);
6206 assert_eq!(notices[0].key, "loop:worker-startup");
6207 assert!(!notices[0].message.contains("private-error-detail"));
6208 assert!(notices[0].message.contains(&request.task));
6209 assert!(
6210 notices[0]
6211 .message
6212 .contains("could not be started or acknowledged")
6213 );
6214 assert!(notices[0].message.contains("worker.log"));
6215 assert_eq!(
6216 notices[0].link,
6217 Some(Link::Task {
6218 id: request.task.clone()
6219 })
6220 );
6221 assert!(format!("{error:#}").contains("private-error-detail"));
6222 }
6223
6224 #[test]
6225 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6226 let home = tempfile::tempdir().unwrap();
6227 let mut request = TaskWorker {
6228 opts: Opts::default(),
6229 queue: home.path().join("queue"),
6230 task: "handoff".into(),
6231 lane: PermitKind::Ordinary,
6232 standalone: true,
6233 };
6234 let job = starting_task_reservation(home.path(), &request);
6235 request.standalone = false;
6236 task_worker_spawn_error(
6237 home.path(),
6238 &request,
6239 anyhow::anyhow!("operation key is already reserved for a different request"),
6240 )
6241 .expect("a different standalone request is the handoff race, not our startup failure");
6242 let active = crate::worker::active(home.path()).unwrap();
6243 assert_eq!(active.len(), 1);
6244 assert_eq!(active[0].id, job.id);
6245 assert_eq!(active[0].payload, job.payload);
6246 assert!(
6247 notices::Notices::at(home.path().join("notifications"))
6248 .list()
6249 .is_empty()
6250 );
6251 }
6252
6253 #[test]
6254 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6255 let home = tempfile::tempdir().unwrap();
6256 let queue = Queue::at(home.path().join("queue"));
6257 let mut task = task();
6258 task.runs = vec!["previous".into()];
6259 queue.put(&mut task).unwrap();
6260 let request = TaskWorker {
6261 opts: Opts::default(),
6262 queue: home.path().join("queue"),
6263 task: task.id.clone(),
6264 lane: PermitKind::Ordinary,
6265 standalone: false,
6266 };
6267 let job = starting_task_reservation(home.path(), &request);
6268 let claims = current_work(home.path(), Timestamp::now());
6269 assert_eq!(claims.len(), 1);
6270 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6271 assert_eq!(claims[0].run, "previous");
6272 assert!(other_claims_run(&claims, "previous", None));
6273 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6276 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6277 }
6278
6279 #[test]
6280 fn admission_excludes_only_the_callers_exact_reservation() {
6281 let own = Current {
6282 worker: Some("own".into()),
6283 task: "same-task".into(),
6284 run: "previous".into(),
6285 };
6286 assert!(!other_claims_run(
6287 std::slice::from_ref(&own),
6288 "previous",
6289 Some("own")
6290 ));
6291 assert!(other_claims_run(
6292 std::slice::from_ref(&own),
6293 "previous",
6294 None
6295 ));
6296 for worker in [None, Some("other".into())] {
6297 let other = Current {
6298 worker,
6299 ..own.clone()
6300 };
6301 assert!(other_claims_run(
6302 &[own.clone(), other],
6303 "previous",
6304 Some("own")
6305 ));
6306 }
6307 let legacy: Current =
6308 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6309 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6310 }
6311
6312 #[test]
6313 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6314 let mut task = task();
6315 let mut run = run_state(RunStatus::Reviewing);
6316 run.task_instruction = Some(task.instruction.clone());
6317 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6318 let branch = "magi/test/B";
6319 run.candidates.push(crate::run::Candidate {
6320 branch: branch.into(),
6321 ..candidate('B', "preserved work", false, None)
6322 });
6323 task.runs = vec![run.id.clone()];
6324 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6325 assert_eq!(
6326 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6327 Starter::Resume(run.id.clone())
6328 );
6329 assert!(matches!(
6330 review_retry_starter(
6331 &task,
6332 Some(&run),
6333 Some("different"),
6334 true,
6335 unfinished.as_deref()
6336 ),
6337 Starter::Review(_)
6338 ));
6339 assert_eq!(
6340 review_retry_starter(
6341 &task,
6342 Some(&run),
6343 Some(branch),
6344 false,
6345 unfinished.as_deref()
6346 ),
6347 Starter::Start
6348 );
6349 task.instruction.push_str(" changed criteria");
6350 assert!(matches!(
6351 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6352 Starter::Review(_)
6353 ));
6354 assert!(matches!(
6355 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6356 Starter::Review(_)
6357 ));
6358 }
6359
6360 #[test]
6361 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6362 let dir = tempfile::tempdir().unwrap();
6363 let queue = Queue::at(dir.path().join("queue"));
6364 let mut task = task();
6365 task.status = TaskStatus::Running;
6366 task.attempts = 1;
6367 queue.put(&mut task).unwrap();
6368 let live = std::collections::BTreeSet::from([task.id.clone()]);
6369 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6370 drop(queue.claim(&task.id).unwrap());
6371 let saved = queue.get(&task.id).unwrap();
6372 assert_eq!(saved.status, TaskStatus::Running);
6373 assert_eq!(saved.attempts, 1);
6374 assert!(saved.last_error.is_none());
6375 }
6376
6377 #[test]
6378 fn standalone_workers_do_not_consume_scheduler_lanes() {
6379 let request = |task: &str, lane, standalone| TaskWorker {
6380 opts: Opts::default(),
6381 queue: PathBuf::from("/queue"),
6382 task: task.to_owned(),
6383 lane,
6384 standalone,
6385 };
6386 let mut jobs = vec![
6387 request("ordinary", PermitKind::Ordinary, false),
6388 request("standalone-urgent", PermitKind::Urgent, true),
6389 request("standalone-ordinary", PermitKind::Ordinary, true),
6390 request("decision", PermitKind::None, false),
6391 ];
6392 assert_eq!(available_slots(&jobs, 2), (1, 1));
6393 jobs.push(request("urgent", PermitKind::Urgent, false));
6394 assert_eq!(available_slots(&jobs, 2), (1, 0));
6395 assert_eq!(available_slots(&jobs, 0), (0, 0));
6396 }
6397
6398 #[test]
6399 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6400 let now = Timestamp::now();
6401 let mut reading = Reading {
6402 pid: Some(std::process::id()),
6403 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6404 ..Reading::default()
6405 };
6406 assert!(!reading.running(now));
6407 assert!(legacy_scheduler_running(&reading, now, 0));
6408 reading.worker_protocol = Some(1);
6409 assert!(!legacy_scheduler_running(&reading, now, 0));
6410 }
6411
6412 #[test]
6413 fn selected_public_head_overrides_historical_resume_and_pin() {
6414 let mut t = task();
6415 let mut old = run_state(RunStatus::Blocked);
6416 let selection = crate::pr_recovery::Selection {
6417 url: "url".into(),
6418 branch: format!("magi/task-{}", t.id),
6419 head: "a".repeat(40),
6420 source_head: None,
6421 source_run: None,
6422 source_proof: None,
6423 disposition: Some(crate::pr_recovery::Disposition::Review),
6424 instruction: t.instruction.clone(),
6425 approved_at: Timestamp::now(),
6426 };
6427 t.resume_override = Some(crate::queue::OperatorResume {
6428 pinned_run: Some(old.id.clone()),
6429 question_id: "choice".into(),
6430 at: Timestamp::now(),
6431 conductor_rehold: None,
6432 forced: false,
6433 });
6434 assert_eq!(
6435 selected_starter(&selection, Some(&old), Some(&old.id)),
6436 Starter::Review(selection.branch.clone())
6437 );
6438 old.pr_recovery = Some(selection.clone());
6439 assert_eq!(
6440 selected_starter(&selection, Some(&old), Some(&old.id)),
6441 Starter::Review(selection.branch.clone())
6442 );
6443 old.reviewed_commits = Some(vec!["implementation".into()]);
6444 assert_eq!(
6445 selected_starter(&selection, Some(&old), Some(&old.id)),
6446 Starter::Resume(old.id.clone())
6447 );
6448 let mut changed = selection.clone();
6449 changed.head = "b".repeat(40);
6450 assert_eq!(
6451 selected_starter(&changed, Some(&old), Some(&old.id)),
6452 Starter::Review(changed.branch)
6453 );
6454 }
6455
6456 #[test]
6457 fn candidate_transplant_failure_is_one_actionable_hold() {
6458 let dir = tempfile::tempdir().unwrap();
6459 let q = Queue::at(dir.path().join("queue"));
6460 let mut t = task();
6461 let mut state = run_state(RunStatus::Blocked);
6462 state.event("prep", "cannot apply referenced commit: conflict");
6463 state.prep_failure = Some("conflict".into());
6464 t.start(state.id.clone());
6465 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6466 assert_eq!(t.attempts, 1);
6467 assert_eq!(t.status, TaskStatus::Held);
6468 assert!(t.operator_held());
6469 assert!(!t.status.runnable());
6470 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6471 let first = t.chat_notice_key().unwrap();
6472 t.chat_notices.push(first.clone());
6473 t.release_operator("test", None);
6474 state.id = "another-unprepared-run".into();
6475 t.start(state.id.clone());
6476 finish_attempt(
6477 3,
6478 &q,
6479 &mut t,
6480 &state,
6481 &[],
6482 Err(anyhow::anyhow!("still conflicts")),
6483 );
6484 assert_eq!(
6485 t.chat_notice_key().as_ref(),
6486 Some(&first),
6487 "no work happened"
6488 );
6489 t.release_operator("test", None);
6490 state.prep_failure = None;
6491 state.status = RunStatus::Failed;
6492 state.begin_pass();
6493 t.start(state.id.clone());
6494 finish_attempt(
6495 1,
6496 &q,
6497 &mut t,
6498 &state,
6499 &[],
6500 Err(anyhow::anyhow!("new gate failure")),
6501 );
6502 let failure = t.chat_notice_key().unwrap();
6503 assert_ne!(failure, first, "approved recovery reached a real failure");
6504 t.release_operator("test", None);
6505 state.begin_pass();
6506 t.start(state.id.clone());
6507 finish_attempt(
6508 1,
6509 &q,
6510 &mut t,
6511 &state,
6512 &[],
6513 Err(anyhow::anyhow!("new failure on resume")),
6514 );
6515 assert_ne!(
6516 t.chat_notice_key().as_ref(),
6517 Some(&failure),
6518 "a new settled pass"
6519 );
6520 }
6521
6522 #[test]
6523 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6524 for source in ["cli", "web"] {
6525 let mut t = task();
6526 let mut run = run_state(RunStatus::Blocked);
6527 run.instruction = "Run A original criteria".into();
6528 run.task_instruction = Some(run.instruction.clone());
6529 t.start(run.id.clone());
6530 t.release_operator("choice", Some("resume-question"));
6531 t.resume_override = Some(crate::queue::OperatorResume {
6532 question_id: "resume-question".into(),
6533 at: Timestamp::now(),
6534 conductor_rehold: None,
6535 forced: true,
6536 pinned_run: Some(run.id.clone()),
6537 });
6538 t.hold_machine(Some("not enough free space to start a run".into()));
6539 let mut unchanged = t.clone();
6540 unchanged
6541 .edit("title changed".into(), unchanged.instruction.clone())
6542 .unwrap();
6543 unchanged.release_operator(source, None);
6544 assert_eq!(
6545 unchanged
6546 .resume_override
6547 .as_ref()
6548 .unwrap()
6549 .pinned_run
6550 .as_deref(),
6551 Some(run.id.as_str())
6552 );
6553 assert!(!compatible_run_instruction(&run, &unchanged));
6554 assert!(may_resume_run(&run, &unchanged));
6555
6556 t.edit(
6557 "approved recovery".into(),
6558 "Reuse measured PR head B; fix Clippy; review".into(),
6559 )
6560 .unwrap();
6561 assert!(t.resume_override.is_none());
6562 t.release_operator(source, None);
6563 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6564 assert!(reloaded.resume_override.is_none());
6565 assert!(!may_resume_run(&run, &reloaded));
6566 let unfinished =
6567 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6568 .filter(|_| may_resume_run(&run, &reloaded));
6569 assert_eq!(
6570 choose_starter(None, false, unfinished.as_deref()),
6571 Starter::Start
6572 );
6573 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6574 assert_eq!(
6575 reloaded.resumption_history[0].question_id.as_deref(),
6576 Some("resume-question")
6577 );
6578
6579 unchanged.instruction = "Reuse measured PR head B".into();
6582 unchanged.hold_machine(Some("disk hold".into()));
6583 unchanged.release_operator(source, None);
6584 assert!(unchanged.resume_override.is_none());
6585 assert!(!may_resume_run(&run, &unchanged));
6586 }
6587 }
6588
6589 #[test]
6590 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6591 let mut t = task();
6592 let mut run = run_state(RunStatus::Blocked);
6593 t.review_of = Some("approved-head".into());
6594 t.instruction =
6595 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6596 .into();
6597 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6598 run.reviewed_commits = Some(vec!["Commit log".into()]);
6599 t.start(run.id.clone());
6600 t.hold_machine(Some("interrupted review".into()));
6601 t.release_operator("cli", None);
6602 assert!(
6603 compatible_run_instruction(&run, &t),
6604 "legacy explicit review remains resumable"
6605 );
6606 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6607 .filter(|_| compatible_run_instruction(&run, &t));
6608 assert_eq!(
6609 choose_starter(None, false, unfinished.as_deref()),
6610 Starter::Resume(run.id.clone())
6611 );
6612 run.task_instruction = Some(t.instruction.clone());
6613 assert!(compatible_run_instruction(&run, &t));
6614 let reloaded: RunState =
6615 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6616 assert!(compatible_run_instruction(&reloaded, &t));
6617 t.edit(
6618 "new approved head".into(),
6619 "Reuse measured PR head B".into(),
6620 )
6621 .unwrap();
6622 assert!(!compatible_run_instruction(&run, &t));
6623
6624 t.review_of = None;
6626 t.instruction = "Fix Clippy on approved head".into();
6627 run.task_instruction = None;
6628 run.instruction = format!(
6629 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6630 t.instruction
6631 );
6632 assert!(compatible_run_instruction(&run, &t));
6633 t.instruction.push('\n');
6634 assert!(
6635 compatible_run_instruction(&run, &t),
6636 "legacy generated review trimmed the original criteria"
6637 );
6638 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6639 let mut unchanged = t.clone();
6640 assert_eq!(
6641 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6642 Some("approved-head")
6643 );
6644 t.edit("new plan".into(), "Reuse measured PR head B".into())
6645 .unwrap();
6646 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6647 assert!(!compatible_run_instruction(&run, &t));
6648 t.review_branch = Some("approved-head".into());
6649 t.review_instruction = None;
6650 assert!(
6651 take_review_branch(&mut t, Some(&run)).is_none(),
6652 "legacy retained candidate cannot replace a revised approved head"
6653 );
6654 }
6655
6656 #[test]
6657 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6658 crate::run::pin_test_home();
6659 let dir = tempfile::tempdir().unwrap();
6660 let queue = Queue::at(dir.path().join("queue"));
6661 for resume in [true, false] {
6662 let state = run_state(RunStatus::Blocked);
6663 let mut t = task();
6664 t.start(state.id.clone());
6665 let action = if resume {
6666 ask::ChoiceAction::Resume {
6667 run: state.id.clone(),
6668 }
6669 } else {
6670 ask::ChoiceAction::Requeue
6671 };
6672 let mut q = action_question(&state.id, action);
6673 q.status = ask::QuestionStatus::Open;
6674 q.answer = None;
6675 q.answered_at = None;
6676 t.fail("run stopped with a question", 3);
6677 hold_if_runnable(&queue, &mut t);
6678 assert!(t.operator_decided_at.is_none());
6679 assert!(t.operator_held());
6680 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6681 let expected = if resume {
6682 ActionDecision::Resume(state.id.clone())
6683 } else {
6684 ActionDecision::Requeue
6685 };
6686 assert_eq!(
6687 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6688 expected
6689 );
6690 t.hold_manual(Some("later explicit hold".into()));
6691 assert_eq!(
6692 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6693 ActionDecision::Stale
6694 );
6695 }
6696 }
6697
6698 #[test]
6699 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6700 let mut t = task();
6701 t.answers.push(crate::queue::AnsweredQuestion {
6702 question: "Inspect the remote?".into(),
6703 answer: "Inspect then wait".into(),
6704 });
6705 let old = instruction_for(&t);
6706 assert!(compatible_instruction(&old, &t));
6707 t.instruction = "Reuse approved PR head and review".into();
6708 t.release_operator("cli", None);
6709 assert!(!compatible_instruction(&old, &t));
6710 let approved = instruction_for(&t);
6711 assert!(approved.contains("waiting premises are superseded"));
6712 assert!(compatible_instruction(&approved, &t));
6713 assert_eq!(resumed_instruction(&approved, &t), approved);
6714 t.release_operator("web", None);
6715 assert!(compatible_instruction(&approved, &t));
6716 assert_eq!(t.resumption_history.len(), 1);
6717 t.start("run".into());
6718 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6719 }
6720
6721 #[test]
6725 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
6726 let mut state = RunState::new(
6727 PathBuf::from("/repo"),
6728 "main".to_owned(),
6729 "abc1234".to_owned(),
6730 "task".to_owned(),
6731 Config::default(),
6732 );
6733 state.status = RunStatus::Blocked;
6734 state.resource_block = Some(crate::run::ResourceBlock {
6735 cause: crate::run::ResourceBlockCause::Other,
6736 node: "graft".to_owned(),
6737 kind: "shared build cache".to_owned(),
6738 detail: "timed out".to_owned(),
6739 at: jiff::Timestamp::now(),
6740 });
6741 let mut t = task();
6742 t.start("20260902-000000-aaaa".to_owned());
6743 assert_eq!(t.attempts, 1);
6744 let verdict = Verdict {
6745 status: RunStatus::Blocked,
6746 left_pr: false,
6747 quota_hit: false,
6748 parked: false,
6749 no_viable_candidates: false,
6750 };
6751 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6752 assert_eq!(t.status, TaskStatus::Held);
6753 assert_eq!(t.attempts, 0, "the attempt is refunded");
6754 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6755 assert!(
6756 !t.operator_held(),
6757 "a machine hold, released without a human"
6758 );
6759
6760 let mut selected = task();
6761 selected.start("20260902-000000-cccc".into());
6762 selected.replacement = Some(crate::merge_decision::Replacement {
6763 question: "fixture".into(),
6764 run: "earlier".into(),
6765 pr: "https://example.invalid/pr/1".into(),
6766 agent: "selected".into(),
6767 context: "fixture".into(),
6768 });
6769 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6770 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6771 assert_eq!(selected.attempts, 0);
6772 assert!(
6773 !selected.operator_held(),
6774 "a selected agent also retains a machine resource hold"
6775 );
6776 assert!(
6777 selected
6778 .hold_reason
6779 .as_deref()
6780 .unwrap()
6781 .starts_with("[resource] ")
6782 );
6783
6784 let mut p = task();
6786 p.start("20260902-000000-bbbb".to_owned());
6787 settle_and_diagnose(
6788 &mut p,
6789 Verdict {
6790 parked: true,
6791 ..verdict
6792 },
6793 "parked",
6794 2,
6795 &state,
6796 );
6797 assert_eq!(p.status, TaskStatus::Parked);
6798 assert_eq!(
6799 p.attempts, 0,
6800 "parking refunds once rather than also settling the resource hold"
6801 );
6802 }
6803
6804 #[test]
6805 fn record_failure_notice_links_the_task() {
6806 let dir = tempfile::tempdir().unwrap();
6807 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6810 let queue = Queue::at(dir.path().join("queue"));
6811 let mut t = interrupt_task("rec-fail-1");
6812 record(&queue, &mut t);
6813
6814 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6815 assert_eq!(found.len(), 1);
6816 assert_eq!(found[0].key, "loop:record");
6817 assert_eq!(
6818 found[0].link,
6819 Some(Link::Task {
6820 id: "rec-fail-1".to_owned()
6821 })
6822 );
6823 }
6824
6825 fn interrupt_task(id: &str) -> Task {
6828 let mut t = task();
6829 t.id = id.to_owned();
6830 t.interrupt = true;
6831 t
6832 }
6833
6834 fn task_with_id(id: &str) -> Task {
6836 let mut t = task();
6837 t.id = id.to_owned();
6838 t
6839 }
6840
6841 fn urgent_task(id: &str) -> Task {
6843 let mut t = task();
6844 t.id = id.to_owned();
6845 t.urgent = true;
6846 t
6847 }
6848
6849 #[test]
6850 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6851 let dir = tempfile::tempdir().unwrap();
6852 let queue = Queue::at(dir.path().join("queue"));
6853 let mut parked = task_with_id("parked");
6854 parked.start("run".into());
6855 parked.park("upgrade");
6856 parked.record_pass_at(
6857 "run",
6858 0,
6859 Some(Timestamp::now()),
6860 1,
6861 Some(crate::run::ParkCause::Upgrade {
6862 from: "old".into(),
6863 to: None,
6864 }),
6865 );
6866 let mut fresh = task_with_id("fresh");
6867 fresh.priority = 100;
6868 let mut urgent_task = urgent_task("urgent");
6869 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6870 queue.put(t).unwrap();
6871 }
6872 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6873 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6874 let candidates = runnable(&queue);
6875 assert_eq!(candidates[0].id, parked.id);
6876 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6877 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6878 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6879 drop(first);
6880 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6881 let next = runnable(&queue)
6882 .into_iter()
6883 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6884 .unwrap();
6885 assert_eq!(next.task.id, fresh.id);
6886 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6887 drop(claimed_elsewhere);
6888 drop(next);
6889 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6890 }
6891
6892 #[test]
6893 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6894 let dir = tempfile::tempdir().unwrap();
6895 let queue = Queue::at(dir.path().join("queue"));
6896 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6897 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6898 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6899 let mut first = task();
6900 queue.put(&mut first).unwrap();
6901 let stale_plain = queue.get(&first.id).unwrap();
6902 first.set_urgent(true).unwrap();
6903 queue.put(&mut first).unwrap();
6904 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
6905 let mut second = urgent_task("second");
6906 queue.put(&mut second).unwrap();
6907 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
6908 drop(queue.claim(&second.id).unwrap());
6909 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
6910 drop(dispatched);
6911 let stale_urgent = queue.get(&first.id).unwrap();
6912 first.set_urgent(false).unwrap();
6913 queue.put(&mut first).unwrap();
6914 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
6915 assert_eq!(urgent.available_permits(), 1);
6916 drop(queue.claim(&first.id).unwrap());
6917 drop(occupied);
6918 }
6919
6920 #[test]
6921 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
6922 let dir = tempfile::tempdir().unwrap();
6923 let queue = Queue::at(dir.path().join("queue"));
6924 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6925 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6926 let mut t = task();
6927 queue.put(&mut t).unwrap();
6928 for enabled in [true, false] {
6929 let stale = queue.get(&t.id).unwrap();
6930 t.set_urgent(enabled).unwrap();
6931 queue.put(&mut t).unwrap();
6932 assert_ne!(stale.urgent, enabled);
6933 let DispatchClaim {
6934 task: fresh,
6935 claim,
6936 permit,
6937 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
6938 assert_eq!(fresh.urgent, enabled);
6939 assert_eq!(ordinary.available_permits(), usize::from(enabled));
6940 assert_eq!(urgent.available_permits(), usize::from(!enabled));
6941 assert!(queue.claim(&t.id).is_err());
6942 assert_eq!(fresh.attempts, stale.attempts);
6943 drop((claim, permit));
6944 assert_eq!(ordinary.available_permits(), 1);
6945 assert_eq!(urgent.available_permits(), 1);
6946 }
6947 t.set_urgent(true).unwrap();
6948 queue.put(&mut t).unwrap();
6949 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
6950 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6951 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
6952 drop(queue.claim(&t.id).unwrap());
6953 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
6954 drop(occupied);
6955 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
6956 drop(queue.claim(&t.id).unwrap());
6957 let held = queue.claim(&t.id).unwrap();
6958 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6959 drop(held);
6960 t.status = TaskStatus::Held;
6961 queue.put(&mut t).unwrap();
6962 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6963 drop(queue.claim(&t.id).unwrap());
6964 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
6965 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
6966 drop(queue.claim(&t.id).unwrap());
6967 assert_eq!(urgent.available_permits(), 1);
6968 }
6969
6970 #[test]
6975 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
6976 assert_eq!(permit_kind(true, false), PermitKind::None);
6977 assert_eq!(permit_kind(true, true), PermitKind::None);
6978 }
6979
6980 #[test]
6985 fn permit_kind_separates_urgent_from_ordinary() {
6986 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
6987 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
6988 }
6989
6990 #[test]
6997 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
6998 let cfg = Config::default();
6999 let repo = Path::new("/any/repo/path");
7000
7001 let reason =
7002 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
7003 assert!(reason.contains("1024"), "{reason}");
7004 assert!(
7005 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7006 "{reason}"
7007 );
7008
7009 assert_eq!(
7010 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
7011 None,
7012 "exactly at the floor is open"
7013 );
7014 assert_eq!(
7015 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
7016 None,
7017 "comfortably above the floor is open"
7018 );
7019 }
7020
7021 #[test]
7022 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
7023 let mut cfg = Config::default();
7024 cfg.disk.min_free_bytes = 0;
7025 let repo = Path::new("/any/repo/path");
7026 assert_eq!(
7027 disk_gate_with(repo, &cfg, |_| Ok(0)),
7028 None,
7029 "a zero floor never measures at all"
7030 );
7031 }
7032
7033 #[test]
7034 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7035 let cfg = Config::default();
7036 let repo = Path::new("/any/repo/path");
7037 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7038 .expect("a measurement failure must close the gate, not open it");
7039 assert!(reason.contains("could not measure"), "{reason}");
7040 }
7041
7042 #[test]
7043 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7044 let ordinary = task();
7045 let next = advance_interrupt(
7046 Interrupt::Idle,
7047 std::slice::from_ref(&ordinary.id),
7048 std::slice::from_ref(&ordinary),
7049 );
7050 assert_eq!(next, Interrupt::Idle);
7051 }
7052
7053 #[test]
7054 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7055 let marked = interrupt_task("marked");
7058 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7059 assert_eq!(next, Interrupt::Idle);
7060 }
7061
7062 #[test]
7063 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7064 let marked = interrupt_task("marked");
7065 let next = advance_interrupt(
7066 Interrupt::Idle,
7067 &["running".to_owned()],
7068 std::slice::from_ref(&marked),
7069 );
7070 assert_eq!(
7071 next,
7072 Interrupt::Parking {
7073 parked: vec!["running".to_owned()],
7074 interrupt_task: "marked".to_owned(),
7075 }
7076 );
7077 }
7078
7079 #[test]
7088 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7089 let marked = interrupt_task("marked");
7090
7091 let two = advance_interrupt(
7092 Interrupt::Idle,
7093 &["a".to_owned(), "b".to_owned()],
7094 std::slice::from_ref(&marked),
7095 );
7096 assert_eq!(two, Interrupt::Idle);
7097
7098 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7099 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7100 }
7101
7102 #[test]
7103 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7104 let state = Interrupt::Parking {
7105 parked: vec!["running".to_owned()],
7106 interrupt_task: "marked".to_owned(),
7107 };
7108 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7110 assert_eq!(still_going, state);
7111
7112 let stopped_but_not_yet_dispatched =
7116 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7117 assert_eq!(stopped_but_not_yet_dispatched, state);
7118
7119 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7121 assert_eq!(
7122 dispatched,
7123 Interrupt::Running {
7124 parked: vec!["running".to_owned()],
7125 interrupt_task: "marked".to_owned(),
7126 }
7127 );
7128 }
7129
7130 #[test]
7131 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7132 let state = Interrupt::Running {
7133 parked: vec!["running".to_owned()],
7134 interrupt_task: "marked".to_owned(),
7135 };
7136 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7137 assert_eq!(still_running, state);
7138
7139 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7146 assert_eq!(
7147 ended,
7148 Interrupt::Resuming {
7149 parked: vec!["running".to_owned()]
7150 }
7151 );
7152 }
7153
7154 #[test]
7155 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7156 let state = Interrupt::Resuming {
7157 parked: vec!["running".to_owned()],
7158 };
7159 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7160 assert_eq!(still_waiting, state);
7161
7162 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7163 assert_eq!(dispatched, Interrupt::Idle);
7164 }
7165
7166 #[test]
7172 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7173 {
7174 let state = Interrupt::Parking {
7175 parked: vec!["running".to_owned()],
7176 interrupt_task: "marked".to_owned(),
7177 };
7178 let next = advance_interrupt(state, &[], &[]);
7181 assert_eq!(
7182 next,
7183 Interrupt::Resuming {
7184 parked: vec!["running".to_owned()]
7185 },
7186 "abandoning the interrupt must not abandon the resume it owes"
7187 );
7188 }
7189
7190 #[test]
7193 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7194 let state = Interrupt::Resuming {
7195 parked: vec!["running".to_owned()],
7196 };
7197 let next = advance_interrupt(state, &[], &[]);
7198 assert_eq!(
7199 next,
7200 Interrupt::Idle,
7201 "nothing is left to wait for; the loop must not stay wedged"
7202 );
7203 }
7204
7205 #[test]
7206 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7207 let marked = interrupt_task("marked");
7208 let next = advance_interrupt_tick(
7209 false,
7210 Interrupt::Idle,
7211 &["running".to_owned()],
7212 std::slice::from_ref(&marked),
7213 );
7214 assert_eq!(
7215 next,
7216 Interrupt::Idle,
7217 "an unmarked, unconfigured daemon must behave exactly as before"
7218 );
7219 }
7220
7221 #[test]
7222 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7223 let state = Interrupt::Parking {
7224 parked: vec!["running".to_owned()],
7225 interrupt_task: "marked".to_owned(),
7226 };
7227 let candidates = vec![interrupt_task("marked"), task()];
7228 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7229 assert!(
7230 allowed.is_empty(),
7231 "nothing may dispatch - not even the interrupt task itself - \
7232 until the parked run has actually stopped"
7233 );
7234 }
7235
7236 #[test]
7250 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7251 for state in [
7252 Interrupt::Parking {
7253 parked: vec!["running".to_owned()],
7254 interrupt_task: "marked".to_owned(),
7255 },
7256 Interrupt::Running {
7257 parked: vec!["running".to_owned()],
7258 interrupt_task: "marked".to_owned(),
7259 },
7260 Interrupt::Resuming {
7261 parked: vec!["running".to_owned()],
7262 },
7263 ] {
7264 let candidates = vec![
7265 interrupt_task("marked"),
7266 urgent_task("hot"),
7267 task_with_id("ordinary"),
7268 ];
7269 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7270 assert!(
7271 !allowed.iter().any(|t| t.id == "hot"),
7272 "an urgent candidate must wait out the same gate as anything \
7273 else while the run it would run alongside has not actually \
7274 left flight, for state {state:?}: {allowed:?}"
7275 );
7276 }
7277 }
7278
7279 #[test]
7285 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7286 let state = Interrupt::Resuming {
7287 parked: vec!["hot".to_owned()],
7288 };
7289 let candidates = vec![urgent_task("hot"), task()];
7290 let allowed = interrupt_gate(&state, &[], candidates);
7291 assert_eq!(
7292 allowed.iter().filter(|t| t.id == "hot").count(),
7293 1,
7294 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7295 );
7296 }
7297
7298 #[test]
7299 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7300 let state = Interrupt::Parking {
7301 parked: vec!["running".to_owned()],
7302 interrupt_task: "marked".to_owned(),
7303 };
7304 let other = task();
7305 let candidates = vec![interrupt_task("marked"), other.clone()];
7306 let allowed = interrupt_gate(&state, &[], candidates);
7307 assert_eq!(allowed.len(), 1);
7308 assert_eq!(allowed[0].id, "marked");
7309 }
7310
7311 #[test]
7312 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7313 let state = Interrupt::Running {
7314 parked: vec!["running".to_owned()],
7315 interrupt_task: "marked".to_owned(),
7316 };
7317 let candidates = vec![task(), task()];
7318 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7319 assert!(allowed.is_empty());
7320 }
7321
7322 #[test]
7329 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7330 let state = Interrupt::Resuming {
7331 parked: vec!["a".to_owned(), "c".to_owned()],
7332 };
7333 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7334 let allowed = interrupt_gate(&state, &[], candidates);
7335 assert_eq!(
7336 allowed.len(),
7337 1,
7338 "at most one candidate may be offered while resuming: {allowed:?}"
7339 );
7340 assert_eq!(allowed[0].id, "a");
7341 }
7342
7343 #[test]
7344 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7345 let state = Interrupt::Resuming {
7346 parked: vec!["a".to_owned()],
7347 };
7348 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7349 assert!(allowed.is_empty());
7350 }
7351
7352 #[test]
7358 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7359 let running = task(); let marked = interrupt_task("marked");
7361
7362 let mut state = Interrupt::Idle;
7363 let in_flight = vec![running.id.clone()];
7365 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7366 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7367 assert!(gated.is_empty(), "still waiting on `running` to park");
7368
7369 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7371 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7372 assert_eq!(
7373 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7374 vec!["marked"],
7375 "only the interrupt task may be offered to the dispatcher now"
7376 );
7377
7378 state = advance_interrupt_tick(
7380 true,
7381 state,
7382 &["marked".to_owned()],
7383 std::slice::from_ref(&running),
7384 );
7385 let gated = interrupt_gate(
7386 &state,
7387 &["marked".to_owned()],
7388 vec![marked.clone(), running.clone()],
7389 );
7390 assert!(
7391 gated.is_empty(),
7392 "the parked run must not be offered back while the interrupt \
7393 task is still running"
7394 );
7395
7396 let other = task_with_id("other");
7400 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7401 assert_eq!(
7402 state,
7403 Interrupt::Resuming {
7404 parked: vec![running.id.clone()]
7405 }
7406 );
7407 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7408 assert_eq!(
7409 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7410 vec![running.id.as_str()],
7411 "exactly the parked run resumes - not the unrelated task, even \
7412 though it was offered first"
7413 );
7414
7415 state = advance_interrupt_tick(
7419 true,
7420 state,
7421 std::slice::from_ref(&running.id),
7422 std::slice::from_ref(&other),
7423 );
7424 assert_eq!(state, Interrupt::Idle);
7425 let gated = interrupt_gate(
7426 &state,
7427 std::slice::from_ref(&running.id),
7428 vec![other.clone()],
7429 );
7430 assert_eq!(
7431 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7432 vec![other.id.as_str()],
7433 "ordinary dispatch is unrestricted again"
7434 );
7435 }
7436
7437 #[test]
7438 fn every_run_status_settles_the_task_it_came_from() {
7439 let table = [
7441 (RunStatus::Merged, TaskStatus::Done, 1),
7442 (RunStatus::Ready, TaskStatus::Done, 1),
7443 (RunStatus::Stalled, TaskStatus::Failed, 0),
7444 (RunStatus::Blocked, TaskStatus::Failed, 1),
7445 (RunStatus::Failed, TaskStatus::Failed, 1),
7446 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7447 (RunStatus::Prep, TaskStatus::Failed, 1),
7448 (RunStatus::Implementing, TaskStatus::Failed, 1),
7449 (RunStatus::Judging, TaskStatus::Failed, 1),
7450 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7451 (RunStatus::Voting, TaskStatus::Failed, 1),
7452 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7453 (RunStatus::Gating, TaskStatus::Failed, 1),
7454 ];
7455 for (run, want, attempts) in table {
7456 let mut t = task();
7457 t.start("20260902-000000-aaaa".to_owned());
7458 settle(
7459 &mut t,
7460 Verdict {
7461 status: run,
7462 left_pr: false,
7463 parked: false,
7464 quota_hit: matches!(run, RunStatus::Stalled),
7465 no_viable_candidates: false,
7466 },
7467 "why",
7468 2,
7469 );
7470 assert_eq!(t.status, want, "task status after {}", label(run));
7471 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7472 }
7473 }
7474
7475 #[test]
7476 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7477 let mut stalled = task();
7478 stalled.start("20260902-000000-aaaa".to_owned());
7479 settle(
7480 &mut stalled,
7481 Verdict {
7482 status: RunStatus::Stalled,
7483 left_pr: false,
7484 parked: false,
7485 quota_hit: true,
7486 no_viable_candidates: false,
7487 },
7488 "quota",
7489 1,
7490 );
7491 assert_eq!(stalled.attempts, 0);
7492 assert!(
7493 stalled.status.runnable(),
7494 "a machine problem must leave the task in line"
7495 );
7496
7497 let mut blocked = task();
7498 blocked.start("20260902-000000-aaaa".to_owned());
7499 settle(
7500 &mut blocked,
7501 Verdict {
7502 status: RunStatus::Blocked,
7503 left_pr: false,
7504 parked: false,
7505 quota_hit: false,
7506 no_viable_candidates: false,
7507 },
7508 "findings open",
7509 1,
7510 );
7511 assert_eq!(blocked.attempts, 1);
7512 assert_eq!(
7513 blocked.status,
7514 TaskStatus::Held,
7515 "the last attempt hands the task to a human"
7516 );
7517 }
7518
7519 #[test]
7520 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7521 let mut delivered = task();
7524 delivered.start("20260903-080619-01c2".to_owned());
7525 settle(
7526 &mut delivered,
7527 Verdict {
7528 status: RunStatus::Blocked,
7529 left_pr: true,
7530 parked: false,
7531 quota_hit: false,
7532 no_viable_candidates: false,
7533 },
7534 "no check status",
7535 4,
7536 );
7537 assert_eq!(
7538 delivered.status,
7539 TaskStatus::Held,
7540 "a pull request waiting on CI or a person is not a retryable failure"
7541 );
7542 assert!(
7543 !delivered.status.runnable(),
7544 "the loop must not pick this task up again"
7545 );
7546 assert_eq!(
7547 delivered.last_error.as_deref(),
7548 Some("no check status"),
7549 "the operator needs to be told what the gate was waiting for"
7550 );
7551
7552 let mut empty_handed = task();
7555 empty_handed.start("20260903-080619-01c2".to_owned());
7556 settle(
7557 &mut empty_handed,
7558 Verdict {
7559 status: RunStatus::Blocked,
7560 left_pr: false,
7561 parked: false,
7562 quota_hit: false,
7563 no_viable_candidates: false,
7564 },
7565 "findings open",
7566 4,
7567 );
7568 assert_eq!(empty_handed.status, TaskStatus::Failed);
7569 assert!(empty_handed.status.runnable());
7570 }
7571
7572 #[test]
7573 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7574 let mut noop = task();
7580 noop.start("20260912-131304-391f".to_owned());
7581 settle(
7582 &mut noop,
7583 Verdict {
7584 status: RunStatus::VerifiedNoop,
7585 left_pr: false,
7586 parked: false,
7587 quota_hit: false,
7588 no_viable_candidates: true,
7589 },
7590 "candidate A: already fixed by b32cfc4, on main",
7591 4,
7592 );
7593 assert_eq!(
7594 noop.status,
7595 TaskStatus::Held,
7596 "an unverified claim is a request for a human, not a failure"
7597 );
7598 assert!(
7599 !noop.status.runnable(),
7600 "the loop must not requeue this on the same unverified claim"
7601 );
7602 assert_eq!(noop.attempts, 1);
7607 }
7608
7609 #[test]
7610 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7611 let mut parked = task();
7616 parked.start("20260903-183634-2d98".to_owned());
7617 settle(
7618 &mut parked,
7619 Verdict {
7620 status: RunStatus::Implementing,
7621 left_pr: false,
7622 quota_hit: false,
7623 parked: true,
7624 no_viable_candidates: false,
7625 },
7626 "parked after `implementing`",
7627 2,
7628 );
7629 assert_eq!(parked.attempts, 0, "a park is refunded");
7630 assert!(
7631 parked.status.runnable(),
7632 "and the task stays in line so the next loop resumes its run"
7633 );
7634 assert_eq!(parked.status, TaskStatus::Parked);
7635 assert_eq!(
7636 parked.park_reason.as_deref(),
7637 Some("parked after `implementing`"),
7638 "the card says where it stopped"
7639 );
7640 assert_eq!(parked.last_error, None, "nothing failed");
7641
7642 let mut broken = task();
7646 broken.start("20260903-183634-2d98".to_owned());
7647 settle(
7648 &mut broken,
7649 Verdict {
7650 status: RunStatus::Implementing,
7651 left_pr: false,
7652 quota_hit: false,
7653 parked: false,
7654 no_viable_candidates: false,
7655 },
7656 "returned mid-flight",
7657 2,
7658 );
7659 assert_eq!(broken.attempts, 1);
7660 }
7661
7662 #[test]
7663 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7664 let mut flaky = task();
7669 flaky.start("20260903-123023-e633".to_owned());
7670 settle(
7671 &mut flaky,
7672 Verdict {
7673 status: RunStatus::Stalled,
7674 left_pr: false,
7675 parked: false,
7676 quota_hit: false,
7677 no_viable_candidates: false,
7678 },
7679 "verdict rests on 1 of 3 judges",
7680 2,
7681 );
7682 assert_eq!(
7683 flaky.attempts, 1,
7684 "flakiness spends an attempt, so `max_attempts` still bounds it"
7685 );
7686 assert!(flaky.status.runnable(), "and it is still worth retrying");
7687
7688 let mut limited = task();
7690 limited.start("20260903-123023-e633".to_owned());
7691 settle(
7692 &mut limited,
7693 Verdict {
7694 status: RunStatus::Stalled,
7695 left_pr: false,
7696 parked: false,
7697 quota_hit: true,
7698 no_viable_candidates: false,
7699 },
7700 "judge-2, judge-3 out of quota",
7701 2,
7702 );
7703 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7704 assert!(limited.status.runnable());
7705
7706 let mut worn = task();
7709 for _ in 0..2 {
7710 worn.release();
7711 }
7712 worn.start("20260903-123023-e633".to_owned());
7713 worn.attempts = 2;
7714 settle(
7715 &mut worn,
7716 Verdict {
7717 status: RunStatus::Stalled,
7718 left_pr: false,
7719 parked: false,
7720 quota_hit: false,
7721 no_viable_candidates: false,
7722 },
7723 "no quorum again",
7724 2,
7725 );
7726 assert_eq!(worn.status, TaskStatus::Held);
7727 assert!(!worn.status.runnable());
7728 }
7729
7730 #[test]
7731 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
7732 let mut wiped_out = task();
7739 wiped_out.start("20260907-025000-a1b2".to_owned());
7740 settle(
7741 &mut wiped_out,
7742 Verdict {
7743 status: RunStatus::Failed,
7744 left_pr: false,
7745 parked: false,
7746 quota_hit: true,
7747 no_viable_candidates: true,
7748 },
7749 "no candidate produced a change; nothing to judge",
7750 2,
7751 );
7752 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7753 assert!(
7754 wiped_out.status.runnable(),
7755 "a machine problem must leave the task in line"
7756 );
7757
7758 let mut partial_progress = task();
7764 partial_progress.start("20260907-025500-c3d4".to_owned());
7765 settle(
7766 &mut partial_progress,
7767 Verdict {
7768 status: RunStatus::Failed,
7769 left_pr: false,
7770 parked: false,
7771 quota_hit: true,
7772 no_viable_candidates: false,
7773 },
7774 "gate failed on the winning candidate",
7775 2,
7776 );
7777 assert_eq!(
7778 partial_progress.attempts, 1,
7779 "a candidate that actually produced a change spends the attempt \
7780 even though some other seat hit its quota"
7781 );
7782 assert!(partial_progress.status.runnable());
7783 }
7784
7785 #[test]
7786 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7787 let mut t = task();
7793 t.start("20260907-025000-a1b2".to_owned());
7794 let mut state = run_state(RunStatus::Failed);
7795 state.quota.push(QuotaLoss {
7796 seat: "cand-a".to_owned(),
7797 node: "implement".to_owned(),
7798 at: Timestamp::now(),
7799 reset: None,
7800 });
7801 assert!(
7802 state.viable().is_empty(),
7803 "no candidate was added, so nothing is viable"
7804 );
7805 reclaim(&mut t, Some(state), 2, "en");
7806 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7807 assert!(t.status.runnable());
7808 }
7809
7810 #[test]
7811 fn a_held_task_is_never_offered_to_the_loop() {
7812 let dir = tempfile::tempdir().unwrap();
7813 let queue = Queue::at(dir.path().to_path_buf());
7814 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7815 let mut t = task();
7816 t.id = format!("2026090{n}-000000-000{n}");
7817 t.priority = priority;
7818 queue.put(&mut t).unwrap();
7819 }
7820 let mut held = task();
7821 held.id = "20260909-000000-9999".to_owned();
7822 held.priority = 99;
7823 held.hold_machine(None);
7824 queue.put(&mut held).unwrap();
7825
7826 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7827 assert_eq!(order.len(), 3);
7828 assert!(!order.contains(&held.id));
7829 assert_eq!(
7830 order.first().cloned(),
7831 queue.next_runnable().map(|t| t.id),
7832 "the loop's first candidate is exactly what the queue offers"
7833 );
7834 assert_eq!(
7835 order,
7836 vec![
7837 "20260902-000000-0002".to_owned(),
7838 "20260903-000000-0003".to_owned(),
7839 "20260901-000000-0001".to_owned(),
7840 ],
7841 "priority first, then oldest, so nothing starves"
7842 );
7843 }
7844
7845 #[test]
7846 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7847 let dir = tempfile::tempdir().unwrap();
7848 let queue = Queue::at(dir.path().to_path_buf());
7849 let mut old = task();
7850 old.id = "20260101-000000-old0".to_owned();
7851 queue.put(&mut old).unwrap();
7852 let mut fresh = task();
7853 fresh.id = "20260101-000000-new0".to_owned();
7854 queue.put(&mut fresh).unwrap();
7855
7856 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7860 std::thread::sleep(Duration::from_millis(60));
7861 let live = queue.claim(&fresh.id).unwrap();
7862
7863 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7864 assert_eq!(swept, vec![old.id.clone()]);
7865 assert!(
7866 queue.claim(&old.id).is_ok(),
7867 "an unparseable lock older than the threshold is swept"
7868 );
7869 assert!(
7870 queue.claim(&fresh.id).is_err(),
7871 "a live pid protects its lock regardless of age"
7872 );
7873 drop(live);
7874 }
7875
7876 #[test]
7877 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7878 let dir = tempfile::tempdir().unwrap();
7881 let queue = Queue::at(dir.path().to_path_buf());
7882 let mut t = task();
7883 t.id = "20260101-000000-live".to_owned();
7884 queue.put(&mut t).unwrap();
7885
7886 let claim = queue.claim(&t.id).unwrap();
7887 std::thread::sleep(Duration::from_millis(60));
7888
7889 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7890 assert!(
7891 swept.is_empty(),
7892 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7893 );
7894 assert!(
7895 queue.claim(&t.id).is_err(),
7896 "the lock still protects its task"
7897 );
7898 drop(claim);
7899 }
7900
7901 fn injected_dead_pid() -> u32 {
7904 std::process::id().checked_add(1).unwrap_or(1)
7905 }
7906
7907 #[test]
7908 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
7909 let dir = tempfile::tempdir().unwrap();
7910 let queue = Queue::at(dir.path().to_path_buf());
7911 let mut t = task();
7912 t.id = "20260101-000000-dead".to_owned();
7913 queue.put(&mut t).unwrap();
7914 let dead_pid = injected_dead_pid();
7915
7916 std::fs::write(
7921 dir.path().join(format!("{}.lock", t.id)),
7922 dead_pid.to_string(),
7923 )
7924 .unwrap();
7925
7926 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7927 pid != dead_pid
7928 });
7929 assert_eq!(
7930 swept,
7931 vec![t.id.clone()],
7932 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
7933 );
7934 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
7935 }
7936
7937 #[test]
7938 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
7939 let dir = tempfile::tempdir().unwrap();
7940 let queue = Queue::at(dir.path().to_path_buf());
7941 let mut t = task();
7942 t.id = "20260101-000000-late".to_owned();
7943 queue.put(&mut t).unwrap();
7944 let dead_pid = injected_dead_pid();
7945
7946 assert!(
7949 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
7950 "nothing has claimed the task yet"
7951 );
7952
7953 std::fs::write(
7956 dir.path().join(format!("{}.lock", t.id)),
7957 dead_pid.to_string(),
7958 )
7959 .unwrap();
7960
7961 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
7965 pid != dead_pid
7966 });
7967 assert_eq!(swept, vec![t.id.clone()]);
7968 }
7969
7970 #[test]
7971 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
7972 crate::run::pin_test_home();
7977 let dir = tempfile::tempdir().unwrap();
7978 let queue = Queue::at(dir.path().to_path_buf());
7979 let mut t = task();
7980 t.id = "20260101-000000-crsh".to_owned();
7981 t.status = TaskStatus::Running;
7982 t.attempts = 1;
7983 t.runs.push("20260904-000000-4043".to_owned());
7987 queue.put(&mut t).unwrap();
7988 let dead_pid = injected_dead_pid();
7989
7990 std::fs::write(
7993 dir.path().join(format!("{}.lock", t.id)),
7994 dead_pid.to_string(),
7995 )
7996 .unwrap();
7997
7998 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
8004 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8005
8006 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8007 pid != dead_pid
8008 });
8009 assert_eq!(swept, vec![t.id.clone()]);
8010
8011 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8012 assert_eq!(reclaimed, vec![t.id.clone()]);
8013 let after = queue.get(&t.id).unwrap();
8014 assert_eq!(
8015 after.status,
8016 TaskStatus::Held,
8017 "no run.json to recover from, so a human is asked"
8018 );
8019 assert_eq!(
8020 after.runs,
8021 vec!["20260904-000000-4043".to_owned()],
8022 "the crashed run's id is kept as evidence, not discarded"
8023 );
8024 }
8025
8026 #[test]
8027 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
8028 let dir = tempfile::tempdir().unwrap();
8029 let queue = Queue::at(dir.path().to_path_buf());
8030 let mut t = task();
8031 t.id = "20260101-000000-unknown".to_owned();
8032 queue.put(&mut t).unwrap();
8033 let dead_pid = injected_dead_pid();
8034 std::fs::write(
8035 dir.path().join(format!("{}.lock", t.id)),
8036 dead_pid.to_string(),
8037 )
8038 .unwrap();
8039
8040 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8041 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8042 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8043 }
8044
8045 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8046 let mut s = run_state(status);
8047 s.config.graph.language = language.to_owned();
8048 s
8049 }
8050
8051 fn unstarted_verdict(status: RunStatus) -> Verdict {
8052 Verdict {
8053 status,
8054 left_pr: false,
8055 quota_hit: false,
8056 parked: false,
8057 no_viable_candidates: false,
8058 }
8059 }
8060
8061 #[test]
8062 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8063 let mut t = task();
8064 t.attempts = 1;
8065 settle_in(
8066 &mut t,
8067 unstarted_verdict(RunStatus::AlreadyInBase),
8068 "already in main",
8069 1,
8070 phrases("en"),
8071 );
8072 assert_eq!(t.status, TaskStatus::Done);
8073 assert_eq!(t.attempts, 0);
8074 }
8075
8076 #[test]
8077 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8078 let reason = |language: &str| {
8079 let mut t = task();
8080 settle_in(
8081 &mut t,
8082 unstarted_verdict(RunStatus::Judging),
8083 "boom",
8084 1,
8085 phrases(language),
8086 );
8087 t.last_error.or(t.hold_reason).unwrap_or_default()
8088 };
8089 assert!(
8090 reason("en").starts_with("the graph stopped at `"),
8091 "{}",
8092 reason("en")
8093 );
8094 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8095 assert!(reason("日本語").contains("boom"));
8096 assert_eq!(reason("fr"), reason("en"));
8097 }
8098
8099 #[test]
8100 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8101 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8102 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8103 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8104 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8105 assert!(ja.contains("[run "), "{ja}");
8106 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8107 assert!(ended.starts_with("run 終了: "), "{ended}");
8108 let mut de = run_state_in(RunStatus::Failed, "de");
8109 let mut en = run_state_in(RunStatus::Failed, "en");
8110 de.id = "same".to_owned();
8111 en.id = "same".to_owned();
8112 assert_eq!(describe(&de), describe(&en));
8113 }
8114
8115 #[test]
8116 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8117 use crate::handover::Refused;
8118 let cases = [
8119 Refused::Foreign {
8120 branch: "b".into(),
8121 path: "/w/x".into(),
8122 why: "made by hand".into(),
8123 },
8124 Refused::Unsafe {
8125 branch: "b".into(),
8126 path: "/w/x".into(),
8127 why: "its worktree has uncommitted changes (a.rs)".into(),
8128 },
8129 Refused::ReleaseFailed {
8130 branch: "b".into(),
8131 path: "/w/x".into(),
8132 run: "ab12".into(),
8133 },
8134 ];
8135 for r in &cases {
8136 let en = (phrases("en").handover_refused)(r);
8137 assert_eq!(en, r.to_string());
8138 let ja = (phrases("ja").handover_refused)(r);
8139 assert!(
8140 !ja.contains("is checked out") && !ja.contains("try again"),
8141 "{ja}"
8142 );
8143 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8144 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8145 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8146 }
8147 }
8148 assert!(phrases("en").handover_hint.contains("release the task"));
8149 assert!(phrases("ja").handover_hint.contains("解放"));
8150 }
8151
8152 #[test]
8153 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8154 let build = |lang: &str| {
8155 let mut s = run_state_in(RunStatus::Blocked, lang);
8156 s.id = "same".to_owned();
8157 s.config.graph.review_rounds = 3;
8158 let mut r = review_round(3);
8159 r.expected = 3;
8160 r.answered = 1;
8161 s.reviews.push(r);
8162 s.event(
8163 "review",
8164 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8165 );
8166 s
8167 };
8168 let en = describe(&build("en"));
8169 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8170 let ja = describe(&build("ja"));
8171 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8172 assert!(!ja.contains("never answered"), "{ja}");
8173 assert!(ja.contains("[run "), "{ja}");
8174
8175 let mut failed = build("ja");
8178 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8179 command: "cargo test".to_owned(),
8180 code: Some(1),
8181 output_tail: String::new(),
8182 duration_ms: 0,
8183 resource_blocked: false,
8184 });
8185 failed.event("review", "stopped; e2e failed: cargo test");
8186 let ja = describe(&failed);
8187 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8188 assert!(!ja.contains("席のレビュアー"), "{ja}");
8189 }
8190
8191 #[test]
8192 fn refusals_and_recovery_prose_follow_the_language() {
8193 let t = held_task_with("r1");
8194 let q = action_question(
8195 "r1",
8196 ask::ChoiceAction::Resume {
8197 run: "r1".to_owned(),
8198 },
8199 );
8200 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
8201 ActionDecision::Refuse(s) => s,
8202 other => panic!("{other:?}"),
8203 };
8204 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8205 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8206 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8207
8208 let mut held = task();
8209 reclaim(&mut held, None, 2, "ja");
8210 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8211 let mut held = task();
8212 reclaim(&mut held, None, 2, "xx");
8213 assert!(held.hold_reason.unwrap().contains("held for a human"));
8214 }
8215
8216 fn run_state(status: RunStatus) -> RunState {
8217 let mut state = RunState::new(
8218 PathBuf::from("/repo"),
8219 "main".to_owned(),
8220 "abc1234def".to_owned(),
8221 "add retries".to_owned(),
8222 Config::default(),
8223 );
8224 state.status = status;
8225 state
8226 }
8227
8228 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8229 Candidate {
8230 index: 0,
8231 label,
8232 agent: "claude".to_owned(),
8233 branch: format!("magi/x/{label}"),
8234 worktree: PathBuf::from("/repo"),
8235 summary: summary.to_owned(),
8236 stat: String::new(),
8237 files: 0,
8238 commits: usize::from(!empty),
8239 empty,
8240 failed: failed.map(str::to_owned),
8241 verified_noop: None,
8242 duration_ms: 0,
8243 folded: false,
8244 }
8245 }
8246
8247 #[test]
8248 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8249 let mut state = run_state(RunStatus::Blocked);
8250 state.gate = vec![
8251 CommandOutcome {
8252 command: "cargo make check".to_owned(),
8253 code: Some(0),
8254 output_tail: "ok".to_owned(),
8255 duration_ms: 0,
8256 resource_blocked: false,
8257 },
8258 CommandOutcome {
8259 command: "cargo test".to_owned(),
8260 code: Some(101),
8261 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8262 duration_ms: 0,
8263 resource_blocked: false,
8264 },
8265 ];
8266 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8267 assert!(d.contains("cargo test"), "{d}");
8268 assert!(
8269 !d.contains("cargo make check"),
8270 "a passing check is not a diagnostic: {d}"
8271 );
8272 assert!(d.contains("assertion failed"), "{d}");
8273 }
8274
8275 #[test]
8276 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8277 let mut state = run_state(RunStatus::Blocked);
8278 state.event(
8279 "land",
8280 "stopped: the fixer produced no commit while 2 check(s) were failing \
8281 (build, lint); stopping instead of looping on an unchanged tree",
8282 );
8283 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8284 assert!(d.contains("build"), "{d}");
8285 assert!(d.contains("lint"), "{d}");
8286 assert!(d.contains("fixer produced no commit"), "{d}");
8287 }
8288
8289 #[test]
8290 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8291 let state = run_state(RunStatus::VerifiedNoop);
8296 let d = describe(&state);
8297 assert!(
8298 d.contains("agent-verified no-op"),
8299 "expected the display label, not the wire spelling: {d}"
8300 );
8301 assert!(!d.contains("verified_noop"), "{d}");
8302 }
8303
8304 #[test]
8305 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8306 let mut state = run_state(RunStatus::Failed);
8312 state.candidates = vec![candidate(
8313 'A',
8314 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8315 true,
8316 None,
8317 )];
8318 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8319 assert!(d.contains("candidate A"), "{d}");
8320 assert!(d.contains("tagged v1.2.3"), "{d}");
8321 }
8322
8323 #[test]
8324 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8325 let mut state = run_state(RunStatus::Failed);
8326 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8327 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8328 assert!(d.contains("candidate A"), "{d}");
8329 assert!(d.contains("agent timed out"), "{d}");
8330 }
8331
8332 #[test]
8333 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8334 let mut state = run_state(RunStatus::Failed);
8337 state.candidates = vec![candidate('A', "did the work", false, None)];
8338 assert!(diagnostic(&state).is_none());
8339 }
8340
8341 #[test]
8342 fn diagnostic_is_bounded_however_much_a_run_printed() {
8343 let mut state = run_state(RunStatus::Blocked);
8344 state.gate = vec![
8345 CommandOutcome {
8346 command: "cargo test".to_owned(),
8347 code: Some(101),
8348 output_tail: "x".repeat(50_000),
8349 duration_ms: 0,
8350 resource_blocked: false,
8351 },
8352 CommandOutcome {
8353 command: "cargo clippy".to_owned(),
8354 code: Some(1),
8355 output_tail: "y".repeat(50_000),
8356 duration_ms: 0,
8357 resource_blocked: false,
8358 },
8359 ];
8360 state.candidates = vec![
8361 candidate('A', &"z".repeat(50_000), true, None),
8362 candidate('B', &"w".repeat(50_000), true, None),
8363 ];
8364 let d = diagnostic(&state).expect("plenty here to diagnose");
8365 assert!(
8366 d.len() <= DIAGNOSTIC_MAX,
8367 "diagnostic grew to {} bytes, unbounded",
8368 d.len()
8369 );
8370 }
8371
8372 #[test]
8373 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8374 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8375 let mut t = task();
8376 let mut state = run_state(status);
8377 t.pr_recovery = Some(crate::pr_recovery::Selection {
8378 url: "url".into(),
8379 branch: format!("magi/task-{}", t.id),
8380 head: "a".repeat(40),
8381 source_head: None,
8382 source_run: None,
8383 source_proof: None,
8384 disposition: Some(crate::pr_recovery::Disposition::Land),
8385 instruction: t.instruction.clone(),
8386 approved_at: Timestamp::now(),
8387 });
8388 state.pr_recovery = t.pr_recovery.clone();
8389 t.start(state.id.clone());
8390 settle_and_diagnose(
8391 &mut t,
8392 Verdict {
8393 status,
8394 left_pr: false,
8395 quota_hit: false,
8396 parked: false,
8397 no_viable_candidates: false,
8398 },
8399 "delivery stopped",
8400 5,
8401 &state,
8402 );
8403 assert_eq!(t.status, TaskStatus::Held);
8404 let reason = t.hold_reason.as_deref().unwrap();
8405 if status == RunStatus::Ready {
8406 assert!(reason.contains("Selected PR delivery is incomplete"));
8407 assert!(t.recovery_budget.is_none());
8408 } else {
8409 assert!(reason.contains("Selected recovery stopped"));
8410 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8411 }
8412 assert!(t.pr_recovery.is_some());
8413 }
8414 }
8415
8416 #[test]
8417 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8418 let mut state = run_state(RunStatus::Blocked);
8419 state.gate = vec![CommandOutcome {
8420 command: "cargo test".to_owned(),
8421 code: Some(101),
8422 output_tail: "assertion failed".to_owned(),
8423 duration_ms: 0,
8424 resource_blocked: false,
8425 }];
8426 let verdict = Verdict {
8427 status: RunStatus::Blocked,
8428 left_pr: false,
8429 quota_hit: false,
8430 parked: false,
8431 no_viable_candidates: false,
8432 };
8433
8434 let mut t = task();
8437 t.start("run-1".to_owned());
8438 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8439 assert_eq!(t.status, TaskStatus::Failed);
8440 assert!(t.diagnostic.is_none());
8441
8442 t.start("run-2".to_owned());
8445 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8446 assert_eq!(t.status, TaskStatus::Held);
8447 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8448 assert!(d.contains("cargo test"), "{d}");
8449 }
8450
8451 #[test]
8452 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8453 crate::run::pin_test_home();
8458 let home = crate::run::home();
8459 let state = run_state(RunStatus::VerifiedNoop);
8460 let mut q = ask::Question::new(
8461 state.id.clone(),
8462 "implement".to_owned(),
8463 "impl-A".to_owned(),
8464 "is this really a no-op?".to_owned(),
8465 String::new(),
8466 Vec::new(),
8467 );
8468 Questions::at(home.join("questions")).put(&mut q).unwrap();
8469
8470 let verdict = Verdict {
8471 status: RunStatus::VerifiedNoop,
8472 left_pr: false,
8473 quota_hit: false,
8474 parked: false,
8475 no_viable_candidates: false,
8476 };
8477 let mut t = task();
8478 t.start(state.id.clone());
8479 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8480
8481 assert_eq!(t.status, TaskStatus::Held);
8482 let reason = t.hold_reason.expect("a held task must record why");
8483 assert!(
8484 reason.starts_with("run ended agent-verified no-op"),
8485 "the original settle reason must survive unchanged: {reason}"
8486 );
8487 assert!(
8488 reason.contains(q.short()),
8489 "the open question's id must be named so the notice is actionable: {reason}"
8490 );
8491 }
8492
8493 #[test]
8494 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8495 crate::run::pin_test_home();
8496 let state = run_state(RunStatus::VerifiedNoop);
8497
8498 let verdict = Verdict {
8499 status: RunStatus::VerifiedNoop,
8500 left_pr: false,
8501 quota_hit: false,
8502 parked: false,
8503 no_viable_candidates: false,
8504 };
8505 let mut t = task();
8506 t.start(state.id.clone());
8507 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8508
8509 assert_eq!(t.status, TaskStatus::Held);
8510 assert_eq!(
8511 t.hold_reason.as_deref(),
8512 Some("run ended agent-verified no-op"),
8513 "nothing to append when the question was already answered or never asked"
8514 );
8515 }
8516
8517 #[test]
8518 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8519 crate::run::pin_test_home();
8520 let mut first = run_state(RunStatus::Blocked);
8521 first.id = "20260101-000000-sup1".to_owned();
8522 first.save().unwrap();
8523 let mut second = run_state(RunStatus::Merged);
8524 second.id = "20260101-000000-sup2".to_owned();
8525 second.save().unwrap();
8526
8527 let mut t = task();
8528 t.runs = vec![first.id.clone(), second.id.clone()];
8529 t.status = TaskStatus::Done;
8530
8531 supersede_prior_runs(&t, &crate::run::home());
8532
8533 assert_eq!(
8534 RunState::load(&first.id).unwrap().status,
8535 RunStatus::Superseded,
8536 "the first attempt's Blocked no longer needs anyone's attention"
8537 );
8538 assert_eq!(
8539 RunState::load(&second.id).unwrap().status,
8540 RunStatus::Merged,
8541 "the run that actually succeeded is left exactly as it was"
8542 );
8543 }
8544
8545 #[test]
8546 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8547 crate::run::pin_test_home();
8553 let mut first = run_state(RunStatus::Blocked);
8554 first.id = "20260101-000000-sup9".to_owned();
8555 first.driver_pid = Some(std::process::id());
8558 first.driver_started_at = Some(
8559 crate::proc::process_started_at(std::process::id())
8560 .expect("this test process's own start time must be queryable"),
8561 );
8562 first.save().unwrap();
8563 let mut second = run_state(RunStatus::Merged);
8564 second.id = "20260101-000000-supa".to_owned();
8565 second.save().unwrap();
8566
8567 let mut t = task();
8568 t.runs = vec![first.id.clone(), second.id.clone()];
8569 t.status = TaskStatus::Done;
8570
8571 supersede_prior_runs(&t, &crate::run::home());
8572
8573 assert_eq!(
8574 RunState::load(&first.id).unwrap().status,
8575 RunStatus::Blocked,
8576 "a live driver_pid means something is still actually working this run, \
8577 even though no daemon claims it - rewriting under it would just be \
8578 undone the next time that process saves"
8579 );
8580 }
8581
8582 #[test]
8583 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8584 let dir = tempfile::tempdir().unwrap();
8590 let home = dir.path().to_path_buf();
8591 let queue = Queue::at(dir.path().join("queue"));
8592
8593 let mut first = run_state(RunStatus::Blocked);
8594 first.id = "20260101-000000-supd".to_owned();
8595 first.driver_pid = Some(std::process::id());
8596 first.driver_started_at = Some(
8597 crate::proc::process_started_at(std::process::id())
8598 .expect("this test process's own start time must be queryable"),
8599 );
8600 first.save_under(&home).unwrap();
8601 let mut second = run_state(RunStatus::Merged);
8602 second.id = "20260101-000000-supe".to_owned();
8603 second.save_under(&home).unwrap();
8604
8605 let mut t = task();
8606 t.runs = vec![first.id.clone(), second.id.clone()];
8607 t.status = TaskStatus::Done;
8608 queue.put(&mut t).unwrap();
8609
8610 resweep_superseded_attempts(&queue, &home);
8611 assert_eq!(
8612 RunState::load_under(&first.id, &home).unwrap().status,
8613 RunStatus::Blocked,
8614 "still live on the first pass, so still untouched"
8615 );
8616
8617 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8623 stale.driver_started_at = Some("1".to_owned());
8624 stale.save_under(&home).unwrap();
8625
8626 resweep_superseded_attempts(&queue, &home);
8627 assert_eq!(
8628 RunState::load_under(&first.id, &home).unwrap().status,
8629 RunStatus::Superseded,
8630 "the second pass catches up what the first one correctly skipped"
8631 );
8632 }
8633
8634 #[test]
8635 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8636 crate::run::pin_test_home();
8637 let mut first = run_state(RunStatus::Blocked);
8638 first.id = "20260101-000000-sup3".to_owned();
8639 first.save().unwrap();
8640 let mut second = run_state(RunStatus::Blocked);
8641 second.id = "20260101-000000-sup4".to_owned();
8642 second.save().unwrap();
8643
8644 let mut t = task();
8645 t.runs = vec![first.id.clone(), second.id.clone()];
8646 t.status = TaskStatus::Failed;
8650
8651 supersede_prior_runs(&t, &crate::run::home());
8652
8653 assert_eq!(
8654 RunState::load(&first.id).unwrap().status,
8655 RunStatus::Blocked
8656 );
8657 assert_eq!(
8658 RunState::load(&second.id).unwrap().status,
8659 RunStatus::Blocked
8660 );
8661 }
8662
8663 #[test]
8664 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
8665 crate::run::pin_test_home();
8669 let mut first = run_state(RunStatus::Blocked);
8670 first.id = "20260101-000000-sup5".to_owned();
8671 first.save().unwrap();
8672
8673 let mut t = task();
8674 t.runs = vec![first.id.clone()];
8675 t.status = TaskStatus::Done;
8676
8677 supersede_prior_runs(&t, &crate::run::home());
8678
8679 assert_eq!(
8680 RunState::load(&first.id).unwrap().status,
8681 RunStatus::Blocked,
8682 "a single-attempt task has no earlier run to supersede"
8683 );
8684 }
8685
8686 #[test]
8687 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
8688 crate::run::pin_test_home();
8695 let mut first = run_state(RunStatus::Blocked);
8696 first.id = "20260101-000000-supb".to_owned();
8697 first.save().unwrap();
8698 let mut second = run_state(RunStatus::Failed);
8699 second.id = "20260101-000000-supc".to_owned();
8700 second.save().unwrap();
8701
8702 let mut t = task();
8703 t.runs = vec![first.id.clone(), second.id.clone()];
8704 t.status = TaskStatus::Done;
8705
8706 supersede_prior_runs(&t, &crate::run::home());
8707
8708 assert_eq!(
8709 RunState::load(&first.id).unwrap().status,
8710 RunStatus::Blocked,
8711 "the task's last attempt never landed, so there is nothing here \
8712 actually superseding it"
8713 );
8714 }
8715
8716 #[test]
8717 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
8718 crate::run::pin_test_home();
8722 let mut failed = run_state(RunStatus::Failed);
8723 failed.id = "20260101-000000-sup6".to_owned();
8724 failed.save().unwrap();
8725 let mut noop = run_state(RunStatus::VerifiedNoop);
8726 noop.id = "20260101-000000-sup7".to_owned();
8727 noop.save().unwrap();
8728 let mut winner = run_state(RunStatus::Ready);
8729 winner.id = "20260101-000000-sup8".to_owned();
8730 winner.save().unwrap();
8731
8732 let mut t = task();
8733 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8734 t.status = TaskStatus::Done;
8735
8736 supersede_prior_runs(&t, &crate::run::home());
8737
8738 assert_eq!(
8739 RunState::load(&failed.id).unwrap().status,
8740 RunStatus::Failed
8741 );
8742 assert_eq!(
8743 RunState::load(&noop.id).unwrap().status,
8744 RunStatus::VerifiedNoop
8745 );
8746 }
8747
8748 fn approval_question(run: &str) -> ask::Question {
8749 ask::Question::new(
8750 run.to_owned(),
8751 land::APPROVAL_NODE.to_owned(),
8752 "land".to_owned(),
8753 "merge?".to_owned(),
8754 String::new(),
8755 vec!["merge".to_owned(), "hold".to_owned()],
8756 )
8757 }
8758
8759 #[test]
8760 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8761 crate::run::pin_test_home();
8762 let mut state = run_state(RunStatus::Landing);
8763 state.id = "20260101-000000-fre1".to_owned();
8764 state.parked = true;
8765 state.save().unwrap();
8766 ask::Questions::open()
8767 .put(&mut approval_question(&state.id))
8768 .unwrap();
8769
8770 let mut t = task();
8771 t.runs.push(state.id.clone());
8772 assert_eq!(
8773 land_resume_state(&t),
8774 LandResume::StillWaiting,
8775 "nobody has answered and the timeout has not passed"
8776 );
8777 }
8778
8779 #[test]
8780 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8781 crate::run::pin_test_home();
8782 let mut state = run_state(RunStatus::Gating);
8783 state.id = "20260101-000000-gt01".to_owned();
8784 state.parked = true;
8785 state.config.graph.answer_timeout = 60;
8786 state.github_text = Some(crate::run::GithubTextAsk {
8787 rewrite: None,
8788 fingerprint: "f".into(),
8789 title: true,
8790 body: false,
8791 categories: vec!["title-language".into()],
8792 question: None,
8793 asks: 1,
8794 resolved: false,
8795 chosen_title: None,
8796 choice: None,
8797 });
8798 state.save().unwrap();
8799 let store = ask::Questions::open();
8800 let mut q = ask::Question::new(
8801 state.id.clone(),
8802 crate::github_text::ASK_NODE.to_owned(),
8803 crate::github_text::ASK_SEAT.to_owned(),
8804 "s".into(),
8805 String::new(),
8806 vec!["use fallback".into()],
8807 );
8808 store.put(&mut q).unwrap();
8809 let mut t = task();
8810 t.runs.push(state.id.clone());
8811 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8812 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8813 store.put(&mut q).unwrap();
8814 assert_eq!(land_resume_state(&t), LandResume::Ready);
8815 assert!(!store.get(&q.id).unwrap().status.open());
8816 state.github_text.as_mut().unwrap().resolved = true;
8818 state.save().unwrap();
8819 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8820 }
8821
8822 #[test]
8823 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8824 crate::run::pin_test_home();
8829 let mut state = run_state(RunStatus::Landing);
8830 state.id = "20260101-000000-exp1".to_owned();
8831 state.parked = true;
8832 state.config.graph.answer_timeout = 60;
8833 state.save().unwrap();
8834
8835 let store = ask::Questions::open();
8836 let mut q = approval_question(&state.id);
8837 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8838 store.put(&mut q).unwrap();
8839
8840 let mut t = task();
8841 t.runs.push(state.id.clone());
8842 assert_eq!(
8843 land_resume_state(&t),
8844 LandResume::Ready,
8845 "an expired question must not be waited on forever"
8846 );
8847
8848 let after = store.get(&q.id).unwrap();
8849 assert!(
8850 !after.status.open(),
8851 "the question is abandoned, not silently ignored"
8852 );
8853 assert!(
8854 after.resolution().is_none(),
8855 "an abandoned question is not read as a decision"
8856 );
8857 }
8858
8859 #[test]
8860 fn reclaim_settles_a_running_task_against_its_last_run() {
8861 let mut t = task();
8862 t.start("20260904-000000-4043".to_owned());
8863 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8864 assert_eq!(
8865 t.status,
8866 TaskStatus::Done,
8867 "a run that actually finished must not stay `running` forever"
8868 );
8869 }
8870
8871 #[test]
8872 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8873 let mut t = task();
8874 t.start("20260904-000000-4043".to_owned());
8875 t.last_error = Some("earlier trouble".to_owned());
8876 let mut state = run_state(RunStatus::Judging);
8877 state.parked = true;
8878 reclaim(&mut t, Some(state), 1, "en");
8879 assert_eq!(t.status, TaskStatus::Parked);
8880 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8881 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8882 assert!(t.park_reason.is_some());
8883 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8884 assert!(!t.fresh_start);
8885 assert!(t.status.runnable());
8886 }
8887
8888 #[test]
8889 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8890 let mut t = task();
8891 t.start("20260904-000000-4043".to_owned());
8892 t.last_error = Some("earlier trouble".to_owned());
8893 settle(
8894 &mut t,
8895 Verdict {
8896 status: RunStatus::Judging,
8897 left_pr: false,
8898 quota_hit: false,
8899 parked: true,
8900 no_viable_candidates: false,
8901 },
8902 "parked after `judging`",
8903 1,
8904 );
8905 assert_eq!(t.status, TaskStatus::Parked);
8906 assert_eq!(t.attempts, 0);
8907 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8908 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
8909 assert_eq!(t.schema, crate::queue::SCHEMA);
8910 assert!(
8911 crate::notices::task_held(&t).is_none(),
8912 "a park is not news"
8913 );
8914 t.start("20260904-000000-4043".to_owned());
8915 assert_eq!(t.park_reason, None);
8916 }
8917
8918 #[test]
8919 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
8920 let mut t = task();
8924 t.start("20260904-000000-4043".to_owned());
8925 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
8926 assert_eq!(t.status, TaskStatus::Failed);
8927 assert!(t.status.runnable());
8928 }
8929
8930 #[test]
8931 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
8932 let mut t = task();
8933 t.start("20260904-000000-4043".to_owned());
8934 assert_eq!(t.attempts, 1);
8935 let mut run = run_state(RunStatus::Implementing);
8936 run.id = "20260904-000000-4043".to_owned();
8937 run.begin_pass();
8938 let cause = crate::run::ParkCause::Upgrade {
8939 from: "0.121.0".to_owned(),
8940 to: Some("0.121.1".to_owned()),
8941 };
8942 run.end_pass_parked(cause.clone());
8943 run.parked = true;
8944 let verdict = Verdict {
8945 status: run.status,
8946 left_pr: false,
8947 quota_hit: false,
8948 parked: true,
8949 no_viable_candidates: false,
8950 };
8951 let mut orphan = t.clone();
8954 reclaim(&mut orphan, Some(run.clone()), 2, "en");
8955 assert_eq!(orphan.status, TaskStatus::Parked);
8956 assert_eq!(orphan.attempts, 0);
8957 assert_eq!(
8958 orphan.upgrade_resume_key(),
8959 Some((run.passes[0].started_at, orphan.id.as_str()))
8960 );
8961 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
8962 assert_eq!(t.attempts, 0);
8963 assert_eq!(t.passes.len(), 1);
8964 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8965 assert_eq!(t.passes[0].parked, Some(cause));
8966 assert_eq!(t.passes[0].pass_index, 0);
8967 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
8968 assert!(t.upgrade_resume_key().is_some());
8969
8970 t.start("20260904-000000-4043".to_owned());
8973 run.begin_pass();
8974 run.status = RunStatus::Failed;
8975 let verdict = Verdict {
8976 status: RunStatus::Failed,
8977 left_pr: false,
8978 quota_hit: false,
8979 parked: false,
8980 no_viable_candidates: false,
8981 };
8982 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
8983 assert_eq!(t.passes.len(), 2);
8984 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
8985 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
8986 assert_eq!(t.passes[1].pass_index, 1);
8987 assert_eq!(t.attempts, 1);
8988 }
8989
8990 #[test]
8991 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
8992 let mut t = task();
8993 t.start("20260904-000000-4043".to_owned());
8994 reclaim(&mut t, None, 2, "en");
8995 assert_eq!(t.status, TaskStatus::Held);
8996 assert!(
8997 t.last_error
8998 .as_deref()
8999 .is_some_and(|e| e.contains("running")),
9000 "the operator needs to know why this task was held"
9001 );
9002 }
9003
9004 #[test]
9005 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
9006 let dir = tempfile::tempdir().unwrap();
9007 let queue = Queue::at(dir.path().to_path_buf());
9008
9009 let mut orphaned = task();
9011 orphaned.id = "20260904-000000-orph".to_owned();
9012 orphaned.status = TaskStatus::Running;
9013 orphaned.attempts = 1;
9014 queue.put(&mut orphaned).unwrap();
9015
9016 let mut alive = task();
9017 alive.id = "20260904-000000-live".to_owned();
9018 alive.status = TaskStatus::Running;
9019 alive.attempts = 1;
9020 queue.put(&mut alive).unwrap();
9021 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
9022
9023 let mut queued = task();
9024 queued.id = "20260904-000000-wait".to_owned();
9025 queue.put(&mut queued).unwrap();
9026
9027 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9028 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
9029
9030 assert_eq!(
9031 queue.get(&orphaned.id).unwrap().status,
9032 TaskStatus::Held,
9033 "nothing was driving it and there was no run to recover"
9034 );
9035 assert_eq!(
9036 queue.get(&alive.id).unwrap().status,
9037 TaskStatus::Running,
9038 "a live claim must protect the task it belongs to"
9039 );
9040 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9041 }
9042
9043 fn read_run_under(home: &Path, id: &str) -> RunState {
9049 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9050 serde_json::from_str(&body).unwrap()
9051 }
9052
9053 #[test]
9054 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9055 let dir = tempfile::tempdir().unwrap();
9056 let home = dir.path().to_path_buf();
9057 let now = Timestamp::now();
9058 let overrun_seat = || crate::run::ActiveSeat {
9059 node: "implement".to_owned(),
9060 started_at: now - jiff::SignedDuration::new(21_000, 0),
9061 timeout_secs: 3_600,
9062 attempt: 0,
9063 task: None,
9064 command: None,
9065 index: None,
9066 total: None,
9067 };
9068
9069 let mut dead = run_state(RunStatus::Implementing);
9070 dead.id = "20260101-000000-dead".to_owned();
9071 dead.active.insert("impl-A".to_owned(), overrun_seat());
9072 dead.driver_pid = Some(4242);
9075 dead.save_under(&home).unwrap();
9076
9077 let mut alive = run_state(RunStatus::Implementing);
9080 alive.id = "20260101-000000-aliv".to_owned();
9081 alive.active.insert("impl-A".to_owned(), overrun_seat());
9082 alive.save_under(&home).unwrap();
9083 let mut status = Status::new();
9084 status.current = vec![Current {
9085 worker: None,
9086 task: "20260101-000000-task".to_owned(),
9087 run: alive.id.clone(),
9088 }];
9089 write_status_to(&home.join("daemon.json"), &status).unwrap();
9090
9091 let questions = Questions::at(home.join("questions"));
9095 let mut q = ask::Question::new(
9096 dead.id.clone(),
9097 "implement".to_owned(),
9098 "impl-A".to_owned(),
9099 "Which storage backend?".to_owned(),
9100 String::new(),
9101 vec!["SQLite".to_owned(), "Redis".to_owned()],
9102 );
9103 questions.put(&mut q).unwrap();
9104
9105 let abandoned = reclaim_abandoned_runs_with(
9106 &home,
9107 now,
9108 |pid| if pid == 4242 { Some(false) } else { None },
9109 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9110 );
9111 assert_eq!(abandoned, vec![dead.id.clone()]);
9112
9113 let reloaded = read_run_under(&home, &dead.id);
9114 assert_eq!(reloaded.status, RunStatus::Failed);
9115 assert!(reloaded.active.is_empty());
9116 assert!(
9117 !questions.get(&q.id).unwrap().status.open(),
9118 "the failed run's own open question must be settled in the same pass"
9119 );
9120
9121 let still_alive = read_run_under(&home, &alive.id);
9122 assert_eq!(
9123 still_alive.status,
9124 RunStatus::Implementing,
9125 "a live daemon's claim protects it"
9126 );
9127 assert!(!still_alive.active.is_empty());
9128 }
9129
9130 #[test]
9140 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9141 let dir = tempfile::tempdir().unwrap();
9142 let home = dir.path().to_path_buf();
9143 let now = Timestamp::now();
9144
9145 let mut manual = run_state(RunStatus::Reviewing);
9146 manual.id = "20260101-000000-manl".to_owned();
9147 manual.active.insert(
9148 "review-1".to_owned(),
9149 crate::run::ActiveSeat {
9150 node: "review".to_owned(),
9151 started_at: now - jiff::SignedDuration::new(21_000, 0),
9152 timeout_secs: 3_600,
9153 attempt: 0,
9154 task: None,
9155 command: None,
9156 index: None,
9157 total: None,
9158 },
9159 );
9160 manual.driver_pid = Some(4242);
9164 manual.driver_started_at = Some("1790000000".to_owned());
9165 manual.save_under(&home).unwrap();
9166
9167 let abandoned = reclaim_abandoned_runs_with(
9168 &home,
9169 now,
9170 |pid| if pid == 4242 { Some(true) } else { None },
9171 |pid| {
9172 if pid == 4242 {
9173 Some("1790000000".to_owned())
9174 } else {
9175 None
9176 }
9177 },
9178 );
9179 assert!(
9180 abandoned.is_empty(),
9181 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9182 );
9183
9184 let reloaded = read_run_under(&home, &manual.id);
9185 assert_eq!(reloaded.status, RunStatus::Reviewing);
9186 assert!(!reloaded.active.is_empty());
9187 }
9188
9189 #[test]
9190 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9191 let dir = tempfile::tempdir().unwrap();
9192 let queue = Queue::at(dir.path().to_path_buf());
9193 let mut only = task();
9194 queue.put(&mut only).unwrap();
9195
9196 let _elsewhere = queue.claim(&only.id).unwrap();
9197 let candidates = runnable(&queue);
9198 assert_eq!(candidates.len(), 1, "the task is still runnable");
9199 assert!(
9200 queue.claim(&candidates[0].id).is_err(),
9201 "the loop cannot take a claim somebody else holds"
9202 );
9203
9204 let after = queue.get(&only.id).unwrap();
9205 assert_eq!(after.status, TaskStatus::Queued);
9206 assert_eq!(
9207 after.attempts, 0,
9208 "losing the race is not an attempt at the task"
9209 );
9210 assert_eq!(after.last_error, None);
9211 }
9212
9213 #[test]
9214 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9215 let dir = tempfile::tempdir().unwrap();
9216 let path = dir.path().join("daemon.json");
9217
9218 let mut status = Status::new();
9219 status.idle = false;
9220 status.completed = 7;
9221 status.current = vec![Current {
9222 worker: None,
9223 task: "20260902-000000-t111".to_owned(),
9224 run: "20260902-000001-r111".to_owned(),
9225 }];
9226 write_status_to(&path, &status).unwrap();
9227 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9228 assert_eq!(first.schema, SCHEMA);
9229 assert_eq!(first.pid, std::process::id());
9230 assert!(!first.idle);
9231 assert_eq!(first.completed, 7);
9232 assert_eq!(first.current, status.current);
9233 assert!(
9234 !path.with_extension("json.tmp").exists(),
9235 "the temp file is renamed, not left behind"
9236 );
9237
9238 std::thread::sleep(Duration::from_millis(5));
9239 status.updated_at = Timestamp::now();
9240 status.polls = 3;
9241 write_status_to(&path, &status).unwrap();
9242 let second: Status =
9243 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9244 assert!(
9245 second.updated_at > first.updated_at,
9246 "a reader can only detect staleness if the heartbeat moves"
9247 );
9248 assert_eq!(
9249 second.started_at, first.started_at,
9250 "the start time is not a heartbeat"
9251 );
9252 assert_eq!(second.polls, 3);
9253 }
9254
9255 #[test]
9256 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9257 let dir = tempfile::tempdir().unwrap();
9258
9259 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9260
9261 let mut status = Status::new();
9262 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9263 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9264 let stale = read_status(dir.path()).unwrap();
9265 assert!(
9266 !stale.running(Timestamp::now()),
9267 "a minute without a heartbeat is a dead daemon, not a busy one"
9268 );
9269 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9270
9271 status.updated_at = Timestamp::now();
9272 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9273 let fresh = read_status(dir.path()).unwrap();
9274 assert!(fresh.running(Timestamp::now()));
9275 }
9276
9277 #[test]
9278 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9279 let dir = tempfile::tempdir().unwrap();
9280 let now = Timestamp::now();
9281 let mine = "20260903-080619-01c2";
9282
9283 assert!(
9284 !is_working_on(dir.path(), mine, now),
9285 "no status file means nobody is working on anything"
9286 );
9287
9288 let mut status = Status::new();
9289 status.current = vec![Current {
9290 worker: None,
9291 task: "20260903-080340-0167".to_owned(),
9292 run: mine.to_owned(),
9293 }];
9294 status.updated_at = now;
9295 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9296 assert!(is_working_on(dir.path(), mine, now));
9297 assert!(
9298 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9299 "a daemon busy with one run is not working on another"
9300 );
9301
9302 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9305 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9306 assert!(
9307 !is_working_on(dir.path(), mine, now),
9308 "a stale heartbeat is a dead daemon, so its run is a leftover"
9309 );
9310 }
9311
9312 #[test]
9313 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9314 let dir = tempfile::tempdir().unwrap();
9315 let now = Timestamp::now();
9316
9317 assert!(
9318 !is_working_on_short(dir.path(), "01c2", now),
9319 "no status file means nobody is working on anything"
9320 );
9321
9322 let mut status = Status::new();
9323 status.current = vec![Current {
9324 worker: None,
9325 task: "20260903-080340-0167".to_owned(),
9326 run: "20260903-080619-01c2".to_owned(),
9327 }];
9328 status.updated_at = now;
9329 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9330 assert!(
9331 is_working_on_short(dir.path(), "01c2", now),
9332 "the run's short id is the last block of its full id"
9333 );
9334 assert!(
9335 !is_working_on_short(dir.path(), "3cbf", now),
9336 "a daemon busy with one worktree bay is not working on another"
9337 );
9338 }
9339
9340 #[test]
9341 fn a_newer_status_file_still_yields_a_reading() {
9342 let dir = tempfile::tempdir().unwrap();
9343 std::fs::write(
9346 dir.path().join("daemon.json"),
9347 serde_json::json!({
9348 "schema": 2,
9349 "updated_at": Timestamp::now().to_string(),
9350 "idle": true,
9351 "surprise": { "nested": [1, 2, 3] },
9352 })
9353 .to_string(),
9354 )
9355 .unwrap();
9356
9357 let reading = read_status(dir.path()).expect("a forward-compatible read");
9358 assert!(reading.running(Timestamp::now()));
9359 assert!(reading.idle);
9360 assert!(reading.current.is_empty());
9361 }
9362
9363 #[test]
9364 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9365 let dir = tempfile::tempdir().unwrap();
9371 std::fs::write(
9372 dir.path().join("daemon.json"),
9373 serde_json::json!({
9374 "schema": 1,
9375 "pid": 4242,
9376 "updated_at": Timestamp::now().to_string(),
9377 "idle": false,
9378 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9379 "completed": 3,
9380 "polls": 9,
9381 })
9382 .to_string(),
9383 )
9384 .unwrap();
9385
9386 let reading = read_status(dir.path()).expect("an older shape must still parse");
9387 assert!(reading.running(Timestamp::now()));
9388 assert_eq!(
9389 reading.current,
9390 vec![Current {
9391 worker: None,
9392 task: "20260902-140501-aaaa".to_owned(),
9393 run: "20260902-140502-bbbb".to_owned(),
9394 }]
9395 );
9396 }
9397
9398 #[test]
9399 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9400 let dir = tempfile::tempdir().unwrap();
9401 std::fs::write(
9402 dir.path().join("daemon.json"),
9403 serde_json::json!({
9404 "schema": 1,
9405 "updated_at": Timestamp::now().to_string(),
9406 "idle": true,
9407 "current": null,
9408 })
9409 .to_string(),
9410 )
9411 .unwrap();
9412 let with_null = read_status(dir.path()).expect("null must still parse");
9413 assert!(with_null.current.is_empty());
9414
9415 std::fs::write(
9416 dir.path().join("daemon.json"),
9417 serde_json::json!({
9418 "schema": 1,
9419 "updated_at": Timestamp::now().to_string(),
9420 "idle": true,
9421 })
9422 .to_string(),
9423 )
9424 .unwrap();
9425 let absent = read_status(dir.path()).expect("a missing field must still parse");
9426 assert!(absent.current.is_empty());
9427 }
9428
9429 #[test]
9430 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9431 let fallback = Path::new("/default");
9432 let mut blank = task();
9433 blank.repo = PathBuf::new();
9434 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9435 let mut dot = task();
9436 dot.repo = PathBuf::from(".");
9437 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9438 assert_eq!(
9439 repo_for(&task(), fallback),
9440 PathBuf::from("/repo"),
9441 "a task that names a repository keeps it"
9442 );
9443 }
9444
9445 #[test]
9446 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9447 let dir = tempfile::tempdir().unwrap();
9448 let path = dir.path().join("explicit.toml");
9449 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9450 let opts = Opts {
9451 config: Some(path),
9452 merge: Some("none".into()),
9453 ..Default::default()
9454 };
9455 let mut t = task();
9456 t.set_implementers(
9457 2,
9458 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9459 .unwrap(),
9460 )
9461 .unwrap();
9462 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9463 assert_eq!(cfg.graph.implementers, 2);
9464 assert_eq!(cfg.graph.judges, 1);
9465 assert_eq!(cfg.merge.mode, MergeMode::None);
9466 t.set_implementers(1, &cfg).unwrap();
9467 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9468 apply_task_implementers(&mut cfg, &t);
9469 assert_eq!(cfg.graph.implementers, 1);
9470 assert_eq!(cfg.graph.judges, 1);
9471 assert_eq!(cfg.merge.mode, MergeMode::None);
9472 }
9473
9474 #[test]
9475 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9476 let dir = tempfile::tempdir().unwrap();
9477 let repo = dir.path().join("repo");
9478 std::fs::create_dir(&repo).unwrap();
9479 std::fs::write(
9480 repo.join("magi.toml"),
9481 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9482 )
9483 .unwrap();
9484 let daemon_config = dir.path().join("daemon.toml");
9485 std::fs::write(
9486 &daemon_config,
9487 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9488 )
9489 .unwrap();
9490 let opts = Opts {
9491 config: Some(daemon_config),
9492 merge: Some("pr".into()),
9493 ..Default::default()
9494 };
9495 let q = Queue::at(dir.path().join("queue"));
9496 let validation: Config =
9497 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9498 .unwrap();
9499 for record in ["{}", r#"{"candidates":2}"#] {
9501 let overrides = serde_json::from_str(record).unwrap();
9502 let filing = crate::direct::Filing {
9503 instruction: "work".into(),
9504 title: "direct".into(),
9505 repo: repo.clone(),
9506 source: crate::queue::Source::Human,
9507 solo: false,
9508 overrides,
9509 review_of: None,
9510 sources: Default::default(),
9511 };
9512 let filed =
9513 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9514 let crate::direct::Filed::Standalone {
9515 mut task,
9516 claim: _claim,
9517 } = filed
9518 else {
9519 panic!("local filing")
9520 };
9521 let before = prepare_for(&repo, &opts, &task).unwrap();
9522 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9523 assert_eq!(
9524 before.merge.mode,
9525 MergeMode::None,
9526 "daemon --merge must not leak"
9527 );
9528 for n in [2, 1, 2] {
9529 task.set_implementers(n, &validation).unwrap();
9530 q.put(&mut task).unwrap();
9531 task = q.get(&task.id).unwrap();
9532 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9533 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9534 apply_task_implementers(&mut cfg, &task);
9535 assert_eq!(cfg.graph.implementers, usize::from(n));
9536 assert_eq!(cfg.graph.judges, 2);
9537 assert_eq!(cfg.merge.mode, MergeMode::None);
9538 }
9539 }
9540 }
9541
9542 #[test]
9543 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9544 let cfg: Config =
9545 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9546 .unwrap();
9547 let mut t = task();
9548 t.review_of = Some("review-target".into());
9549 t.hold_manual(Some("retry".into()));
9550 let mut previous = RunState::new(
9551 PathBuf::from("/repo"),
9552 "main".into(),
9553 "abc1234".into(),
9554 "review".into(),
9555 cfg.clone(),
9556 );
9557 previous.config.graph.implementers = 2;
9558 for count in [1, 3] {
9559 t.set_implementers(count, &cfg).unwrap();
9560 t.release();
9561 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9562 for proposed in [
9563 Starter::Resume(previous.id.clone()),
9564 Starter::Review("recovery-target".into()),
9565 ] {
9566 assert_eq!(
9567 starter_with_implementers(&t, Some(&previous), proposed),
9568 Starter::Review("review-target".into())
9569 );
9570 }
9571 let mut configured = cfg.clone();
9572 apply_task_implementers(&mut configured, &t);
9573 assert_eq!(configured.graph.implementers, usize::from(count));
9574 assert_eq!(previous.config.graph.implementers, 2);
9575 }
9576 }
9577
9578 #[test]
9579 fn task_implementers_dispatch_and_resume_agree() {
9580 let mut t = task();
9581 let cfg: crate::config::Config =
9582 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9583 .unwrap();
9584 let mut run = RunState::new(
9585 PathBuf::from("/repo"),
9586 "main".into(),
9587 "abc1234".into(),
9588 "work".into(),
9589 cfg.clone(),
9590 );
9591 run.config.graph.implementers = 2;
9592 for n in [2, 1, 3, 2] {
9593 t.set_implementers(n, &cfg).unwrap();
9594 let mut dispatched = cfg.clone();
9595 apply_task_implementers(&mut dispatched, &t);
9596 assert_eq!(dispatched.graph.implementers, usize::from(n));
9597 assert_eq!(implementers_match(&t, &run), n == 2);
9598 for proposed in [
9599 Starter::Resume(run.id.clone()),
9600 Starter::Review("old-branch".into()),
9601 ] {
9602 let expected = if n == 2 {
9603 proposed.clone()
9604 } else {
9605 Starter::Start
9606 };
9607 assert_eq!(
9608 starter_with_implementers(&t, Some(&run), proposed),
9609 expected
9610 );
9611 }
9612 assert_eq!(
9613 run.config.graph.implementers, 2,
9614 "history must not be rewritten"
9615 );
9616 }
9617 t.solo = false;
9618 t.overrides = None;
9619 assert!(implementers_match(&t, &run));
9620 }
9621
9622 #[test]
9623 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9624 let mut solo_cfg = Config::default();
9630 solo_cfg.graph.implementers = 3;
9631 let mut solo_task = task();
9632 solo_task.solo = true;
9633 apply_task_implementers(&mut solo_cfg, &solo_task);
9634 assert_eq!(solo_cfg.graph.implementers, 1);
9635
9636 let mut plain_cfg = Config::default();
9637 plain_cfg.graph.implementers = 3;
9638 let plain_task = task();
9639 assert!(!plain_task.solo);
9640 apply_task_implementers(&mut plain_cfg, &plain_task);
9641 assert_eq!(
9642 plain_cfg.graph.implementers, 3,
9643 "a task that did not ask to run alone keeps the config's candidates"
9644 );
9645 }
9646
9647 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9648 QuotaLoss {
9649 seat: seat.into(),
9650 node: "judge".into(),
9651 at: at.parse().unwrap(),
9652 reset: reset.map(str::to_string),
9653 }
9654 }
9655
9656 #[test]
9657 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
9658 let old: Vec<QuotaLoss> = (1..=4)
9659 .map(|i| {
9660 loss(
9661 &format!("judge-{i}"),
9662 "2026-09-23T05:23:00Z",
9663 Some("2:40pm (Asia/Tokyo)"),
9664 )
9665 })
9666 .collect();
9667 let fresh = losses_this_attempt(&old, &old);
9668 assert!(fresh.is_empty());
9669 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
9670 }
9672
9673 #[test]
9674 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
9675 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
9676 let now = Timestamp::now();
9677 let mut after = old.clone();
9678 after.push(loss("judge-2", &now.to_string(), None));
9679 let fresh = losses_this_attempt(&old, &after);
9680 assert_eq!(fresh, vec![after[1].clone()]);
9681 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
9682 assert_eq!(
9683 until,
9684 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
9685 );
9686 }
9687
9688 #[test]
9689 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
9690 let before = vec![
9693 loss("judge-1", "2026-09-23T05:23:00Z", None),
9694 loss("judge-2", "2026-09-23T05:24:00Z", None),
9695 ];
9696 let after = vec![
9697 loss("judge-2", "2026-09-23T05:24:00Z", None),
9698 loss("judge-1", "2026-09-24T01:00:00Z", None),
9699 ];
9700 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
9701 }
9702
9703 #[test]
9704 fn merge_overrides_are_parsed_or_refused() {
9705 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
9706 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
9707 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
9708 assert!(merge_mode("squash").is_err());
9709 }
9710
9711 #[test]
9712 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
9713 let now = Timestamp::now();
9714 let fallback = Duration::from_secs(300);
9715 let cap = Duration::from_secs(1800);
9716
9717 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
9719
9720 let soon = now + jiff::SignedDuration::from_secs(600);
9722 assert_eq!(
9723 quota_wait(Some(soon), now, fallback, cap),
9724 Duration::from_secs(600)
9725 );
9726
9727 let past = now - jiff::SignedDuration::from_secs(60);
9730 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
9731
9732 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
9735 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
9736 }
9737
9738 #[test]
9739 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9740 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9741
9742 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9743 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9744
9745 let already_past =
9749 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9750 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9751
9752 assert!(
9753 parse_reset_hint("session limit reached", now, now).is_none(),
9754 "free text with no recognised shape is not guessed at"
9755 );
9756 assert!(
9757 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9758 "an unresolvable zone name is not guessed at either"
9759 );
9760 }
9761
9762 #[test]
9763 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9764 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9765
9766 let at = parse_reset_hint(
9767 "You've hit your usage limit. Visit \
9768 https://chatgpt.com/codex/settings/usage to purchase more \
9769 credits or try again at Sep 19th, 2026 5:10 PM.",
9770 now,
9771 now,
9772 )
9773 .expect("the codex reset wording is a recognised shape");
9774 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9775
9776 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9781 .expect("an explicit year needs no rollover");
9782 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9783
9784 assert!(
9785 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9786 "a two-digit year is not the documented shape and is not guessed at"
9787 );
9788 assert!(
9789 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9790 "a four-letter month name is not the documented three-letter abbreviation"
9791 );
9792 assert!(
9793 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9794 "an explicit zone on the dated shape is a format nobody has \
9795 documented, and is refused rather than guessed at as UTC"
9796 );
9797 }
9798
9799 #[test]
9800 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9801 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9802 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9803
9804 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9805 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9806
9807 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9808 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9809
9810 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9811 assert!(
9812 parse_reset_hint(bad, now, recorded).is_none(),
9813 "{bad:?} must not be guessed at"
9814 );
9815 }
9816 }
9817
9818 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9822 let config = dir.join("magi.toml");
9823 std::fs::write(
9824 &config,
9825 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9826 )
9827 .unwrap();
9828 let opts = Opts {
9829 poll: Duration::from_secs(30),
9830 config: Some(config),
9831 repo: dir.join("repo"),
9835 ..Opts::default()
9836 };
9837 let home = dir.join("home");
9846 let stamps = home.join("service-ticks");
9849 std::fs::create_dir_all(&stamps).unwrap();
9850 for service in ["waiter", "deputies", "release-watch"] {
9851 std::fs::write(
9852 stamps.join(service),
9853 Timestamp::now().as_second().to_string(),
9854 )
9855 .unwrap();
9856 }
9857 for operation in [
9858 "conductor",
9859 "maintenance",
9860 "housekeeping",
9861 "disk-pressure",
9862 "cache",
9863 "fetch",
9864 ] {
9865 std::fs::write(
9866 stamps.join(format!("daemon-{operation}")),
9867 serde_json::to_vec(&Timestamp::now()).unwrap(),
9868 )
9869 .unwrap();
9870 }
9871 let worktrees = dir.join("wt");
9872 (
9873 opts,
9874 Queue::at(dir.join("queue")),
9875 home.join("daemon.json"),
9876 home,
9877 worktrees,
9878 )
9879 }
9880
9881 #[test]
9882 fn a_stop_is_idempotent_and_once_set_stays_set() {
9883 let stop = Stop::new();
9884 assert!(!stop.stopped());
9885
9886 stop.stop();
9887 assert!(stop.stopped());
9888 stop.stop();
9889 assert!(stop.stopped(), "a second stop is not a toggle");
9890
9891 let shared = stop.clone();
9892 assert!(
9893 shared.stopped(),
9894 "a clone is the same stop; that is how the loop and its caller share one"
9895 );
9896 }
9897
9898 #[test]
9899 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9900 let stop = Stop::new();
9901 stop.busy.store(1, Ordering::SeqCst);
9902 assert!(
9903 !stop.finishing(),
9904 "a busy loop nobody has asked to stop is just running"
9905 );
9906
9907 stop.stop();
9908 assert!(
9909 stop.finishing(),
9910 "a stop asked for mid-run has not landed until the run is settled"
9911 );
9912
9913 stop.busy.store(0, Ordering::SeqCst);
9914 assert!(
9915 !stop.finishing(),
9916 "once the run is settled the stop has landed and there is nothing to finish"
9917 );
9918 }
9919
9920 #[test]
9921 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
9922 let stop = Stop::new();
9923 stop.busy.store(2, Ordering::SeqCst);
9924 stop.stop();
9925 assert!(stop.finishing(), "two runs still in flight");
9926
9927 stop.busy.store(1, Ordering::SeqCst);
9928 assert!(
9929 stop.finishing(),
9930 "one run finished, but a sibling is still working"
9931 );
9932
9933 stop.busy.store(0, Ordering::SeqCst);
9934 assert!(
9935 !stop.finishing(),
9936 "the last run out is what actually lands the stop"
9937 );
9938 }
9939
9940 #[tokio::test]
9941 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
9942 let dir = tempfile::tempdir().unwrap();
9943 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9944 let stop = Stop::new();
9945 stop.stop();
9946
9947 let began = std::time::Instant::now();
9948 tokio::time::timeout(
9949 Duration::from_secs(2),
9950 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9951 )
9952 .await
9953 .expect("a stopped loop must return, not sit out its poll interval")
9954 .expect("the loop's own setup and teardown must not fail");
9955 assert!(
9956 began.elapsed() < opts.poll,
9957 "returned only after {:?}, which is a poll interval, not a stop",
9958 began.elapsed()
9959 );
9960 }
9961
9962 #[tokio::test]
9963 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
9964 let dir = tempfile::tempdir().unwrap();
9965 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9966 let stop = Stop::new();
9967
9968 let asker = {
9971 let stop = stop.clone();
9972 tokio::spawn(async move {
9973 tokio::time::sleep(Duration::from_millis(20)).await;
9974 stop.stop();
9975 })
9976 };
9977
9978 let began = std::time::Instant::now();
9979 tokio::time::timeout(
9980 Duration::from_secs(2),
9981 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
9982 )
9983 .await
9984 .expect("a stop asked for while idle must wake the wait")
9985 .expect("the loop's own setup and teardown must not fail");
9986 asker.await.unwrap();
9987 assert!(
9988 began.elapsed() < opts.poll,
9989 "returned only after {:?}, so the stop waited on the sleep",
9990 began.elapsed()
9991 );
9992 }
9993
9994 #[tokio::test]
9995 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
9996 let dir = tempfile::tempdir().unwrap();
9997 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
9998 let stop = Stop::new();
9999 stop.stop();
10000
10001 tokio::time::timeout(
10002 Duration::from_secs(2),
10003 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10004 )
10005 .await
10006 .expect("a stopped loop must return")
10007 .expect("the loop's own setup and teardown must not fail");
10008
10009 assert!(
10010 home.is_dir(),
10011 "the loop did publish a status file, so its removal is the teardown and not an absence"
10012 );
10013 assert!(
10014 !status_file.exists(),
10015 "a stopped loop clears its status file"
10016 );
10017 assert!(
10018 read_status(&home).is_none(),
10019 "a reader must see no daemon at all, not a heartbeat that merely stopped"
10020 );
10021 }
10022
10023 #[tokio::test]
10024 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
10025 let dir = tempfile::tempdir().unwrap();
10026 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10027
10028 let mut settled = RunState::new(
10029 dir.path().join("repo"),
10030 "main".to_owned(),
10031 "abc1234".to_owned(),
10032 "fixture".to_owned(),
10033 Config::default(),
10034 );
10035 settled.status = RunStatus::Ready;
10036 let run_dir = home.join("runs").join(&settled.id);
10037 std::fs::create_dir_all(&run_dir).unwrap();
10038 std::fs::write(
10039 run_dir.join("run.json"),
10040 serde_json::to_string_pretty(&settled).unwrap(),
10041 )
10042 .unwrap();
10043 let questions = Questions::at(home.join("questions"));
10044 let mut question = ask::Question::new(
10045 settled.id.clone(),
10046 "review".to_owned(),
10047 "reviewer-1".to_owned(),
10048 "Continue?".to_owned(),
10049 String::new(),
10050 Vec::new(),
10051 );
10052 questions.put(&mut question).unwrap();
10053
10054 run_service_worker(
10055 &home,
10056 serde_json::to_value(ServiceWorker {
10057 opts,
10058 queue: queue.root().to_path_buf(),
10059 worktrees_root: worktrees,
10060 operation: "housekeeping".to_owned(),
10061 })
10062 .unwrap(),
10063 )
10064 .await
10065 .unwrap();
10066
10067 assert_eq!(
10068 questions.get(&question.id).unwrap().status,
10069 ask::QuestionStatus::Abandoned,
10070 "the independent housekeeping worker settles ended-run questions"
10071 );
10072 }
10073
10074 #[test]
10075 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10076 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10077
10078 assert!(
10079 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10080 "never checked before: due at once"
10081 );
10082
10083 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10084 assert!(
10085 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10086 "well inside the interval: not due yet"
10087 );
10088
10089 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10090 assert!(
10091 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10092 "exactly at the edge: not yet due, same convention as `clean::due`"
10093 );
10094
10095 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10096 assert!(
10097 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10098 "past the interval: due again"
10099 );
10100 }
10101
10102 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10107 let config = dir.join("magi.toml");
10108 std::fs::write(
10114 &config,
10115 format!(
10116 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10117 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10118 cache_dir.display()
10119 ),
10120 )
10121 .unwrap();
10122 Opts {
10123 config: Some(config),
10124 repo: dir.join("repo"),
10125 ..Opts::default()
10126 }
10127 }
10128
10129 #[tokio::test]
10130 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10131 let dir = tempfile::tempdir().unwrap();
10132 let home = dir.path().join("home");
10133 let cache_dir = dir.path().join("cache");
10134 std::fs::create_dir_all(&cache_dir).unwrap();
10135 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10136 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10137
10138 let running = Stop::new();
10141 let mut last_checked = None;
10142 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10143 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10144 .await;
10145 assert_eq!(
10146 crate::disk::dir_size(&cache_dir),
10147 0,
10148 "over the cap on the first check ever: pruned at once, no idle queue required"
10149 );
10150 assert_eq!(last_checked, Some(t0));
10151
10152 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10154 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10155 maybe_prune_cache_between_runs(
10156 &opts.repo,
10157 &opts,
10158 &home,
10159 &running,
10160 &mut last_checked,
10161 too_soon,
10162 )
10163 .await;
10164 assert_eq!(
10165 crate::disk::dir_size(&cache_dir),
10166 10,
10167 "too soon since the last check: left alone rather than rescanned every call"
10168 );
10169 assert_eq!(
10170 last_checked,
10171 Some(t0),
10172 "an idle check does not reset the clock"
10173 );
10174
10175 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10177 maybe_prune_cache_between_runs(
10178 &opts.repo,
10179 &opts,
10180 &home,
10181 &running,
10182 &mut last_checked,
10183 due_again,
10184 )
10185 .await;
10186 assert_eq!(
10187 crate::disk::dir_size(&cache_dir),
10188 0,
10189 "due again: pruned back under the cap"
10190 );
10191 }
10192
10193 #[tokio::test]
10201 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
10202 let dir = tempfile::tempdir().unwrap();
10203 let home = dir.path().join("home");
10204 let cache_dir = dir.path().join("cache");
10205 std::fs::create_dir_all(&cache_dir).unwrap();
10206 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10207 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10208
10209 let stop = Stop::new();
10210 stop.stop();
10211 assert!(
10212 !stop.finishing(),
10213 "no run is in flight at a between-runs boundary, so nothing else \
10214 would tell the operator this stop had not taken effect yet"
10215 );
10216
10217 let mut last_checked = None;
10218 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10219 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10220 .await;
10221 assert_eq!(
10222 crate::disk::dir_size(&cache_dir),
10223 10,
10224 "over its cap, and due for the first check ever, but a stop outranks \
10225 it: the cap is a standing policy the next start measures again"
10226 );
10227 assert_eq!(
10228 last_checked, None,
10229 "a check that never happened must not claim the interval"
10230 );
10231 }
10232
10233 #[tokio::test]
10236 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10237 let dir = tempfile::tempdir().unwrap();
10238 let cache_dir = dir.path().join("cache");
10239 std::fs::create_dir_all(&cache_dir).unwrap();
10240 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10241
10242 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10243 opts.poll = Duration::from_millis(20);
10244 opts.max_attempts = 1_000;
10245
10246 let queue = Queue::at(dir.path().join("queue"));
10247 for i in 0..128 {
10248 let mut task = Task::new(
10249 format!("queued work {i}"),
10250 format!("queued work {i}"),
10251 opts.repo.clone(),
10252 Source::Human,
10253 );
10254 queue.put(&mut task).unwrap();
10255 }
10256
10257 let home = dir.path().join("home");
10258 let worktrees = dir.path().join("wt");
10259 run_service_worker(
10260 &home,
10261 serde_json::to_value(ServiceWorker {
10262 opts,
10263 queue: queue.root().to_path_buf(),
10264 worktrees_root: worktrees,
10265 operation: "cache".to_owned(),
10266 })
10267 .unwrap(),
10268 )
10269 .await
10270 .unwrap();
10271 let after = queue.list();
10272 assert_eq!(after.len(), 128);
10273 assert!(
10274 after
10275 .iter()
10276 .all(|task| task.status.runnable() && task.attempts == 0)
10277 );
10278
10279 assert_eq!(
10280 crate::disk::dir_size(&cache_dir),
10281 0,
10282 "an oversized cache must not be left to grow unboundedly just because the \
10283 queue kept the loop busy the whole time"
10284 );
10285 }
10286
10287 #[test]
10288 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10289 let dir = tempfile::tempdir().unwrap();
10290 let queue = Queue::at(dir.path().join("queue"));
10291 let questions = Questions::at(dir.path().join("questions"));
10292 let mut task = task();
10293 queue.put(&mut task).unwrap();
10294
10295 let mut task_question = ask::Question::new(
10296 task.id.clone(),
10297 crate::conduct::NODE.to_owned(),
10298 "conduct".to_owned(),
10299 "Which backend?".to_owned(),
10300 String::new(),
10301 Vec::new(),
10302 );
10303 questions.put(&mut task_question).unwrap();
10304 task.block(vec![task_question.id.clone()], None);
10305 queue.put(&mut task).unwrap();
10306
10307 let mut run_question = ask::Question::new(
10308 "20260101-000000-run1".to_owned(),
10309 "review".to_owned(),
10310 "reviewer-1".to_owned(),
10311 "Run question".to_owned(),
10312 String::new(),
10313 Vec::new(),
10314 );
10315 questions.put(&mut run_question).unwrap();
10316
10317 let mut coincidental = ask::Question::new(
10322 task.id.clone(),
10323 "review".to_owned(),
10324 "reviewer-1".to_owned(),
10325 "Unrelated review question".to_owned(),
10326 String::new(),
10327 Vec::new(),
10328 );
10329 questions.put(&mut coincidental).unwrap();
10330
10331 reconcile_task_questions(&queue, &questions);
10332 assert!(questions.get(&task_question.id).unwrap().status.open());
10333 assert!(questions.get(&run_question.id).unwrap().status.open());
10334 assert!(questions.get(&coincidental.id).unwrap().status.open());
10335
10336 task.release();
10337 queue.put(&mut task).unwrap();
10338 reconcile_task_questions(&queue, &questions);
10339 assert_eq!(
10340 questions.get(&task_question.id).unwrap().status,
10341 ask::QuestionStatus::Abandoned
10342 );
10343 assert!(
10344 questions.get(&run_question.id).unwrap().status.open(),
10345 "run questions remain the run janitor's responsibility"
10346 );
10347 assert!(
10348 questions.get(&coincidental.id).unwrap().status.open(),
10349 "a non-conductor question must not be abandoned just because its \
10350 run id coincides with a task id"
10351 );
10352 }
10353
10354 #[test]
10355 fn a_freshly_started_running_task_is_never_stalled() {
10356 let dir = tempfile::tempdir().unwrap();
10357 let mut t = task();
10358 t.start("run-1".to_owned());
10359 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10362 }
10363
10364 #[test]
10365 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10366 let dir = tempfile::tempdir().unwrap();
10367 let mut t = task();
10368 t.start("run-1".to_owned());
10369 t.updated_at = Timestamp::now()
10370 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10371 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10372 assert_eq!(
10373 stalled_tasks(
10374 &Queue::at(dir.path().join("q")),
10375 dir.path(),
10376 Timestamp::now()
10377 )
10378 .len(),
10379 0,
10380 "the task was never written to this queue"
10381 );
10382 }
10383
10384 #[test]
10385 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10386 let dir = tempfile::tempdir().unwrap();
10387 let mut t = task();
10388 t.id = "20260903-080340-0167".to_owned();
10389 t.start("20260903-080619-01c2".to_owned());
10390 t.updated_at = Timestamp::now()
10391 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10392
10393 let mut status = Status::new();
10394 status.current = vec![Current {
10395 worker: None,
10396 task: t.id.clone(),
10397 run: "20260903-080619-01c2".to_owned(),
10398 }];
10399 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10400
10401 assert!(
10402 !is_stalled(&t, dir.path(), Timestamp::now()),
10403 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10404 );
10405 }
10406
10407 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10411 let path = queue.path_of(id);
10412 let body = std::fs::read_to_string(&path).unwrap();
10413 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10414 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10415 v["updated_at"] = serde_json::Value::String(old.to_string());
10416 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10417 }
10418
10419 #[test]
10420 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10421 let dir = tempfile::tempdir().unwrap();
10434 let queue = Queue::at(dir.path().join("queue"));
10435 let home = dir.path().join("home");
10436
10437 let mut t = task();
10438 t.id = "20260101-000001-lock".to_owned();
10439 t.start("run-1".to_owned());
10440 queue.put(&mut t).unwrap();
10441 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10442 std::fs::write(
10443 dir.path().join("queue").join(format!("{}.lock", t.id)),
10444 "not a pid",
10445 )
10446 .unwrap();
10447
10448 let now = Timestamp::now();
10449 assert!(
10450 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10451 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10452 and reclaim must leave the task alone"
10453 );
10454 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10455
10456 let stalled = stalled_tasks(&queue, &home, now);
10457 assert_eq!(
10458 stalled.len(),
10459 1,
10460 "reclaim's inability to claim it yet must not hide it from the conductor"
10461 );
10462 assert_eq!(stalled[0].id, t.id);
10463 }
10464
10465 #[test]
10466 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10467 let dir = tempfile::tempdir().unwrap();
10468 crate::run::set_home(dir.path().join("run-home"));
10469 let queue = Queue::at(dir.path().join("queue"));
10470 let home = dir.path().join("home");
10471 let questions = Questions::at(dir.path().join("questions"));
10472
10473 let mut t = task();
10474 t.id = "20260101-000003-dead".to_owned();
10475 t.start("missing-run".to_owned());
10476 queue.put(&mut t).unwrap();
10477 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10478
10479 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10482 assert_eq!(
10483 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10484 [&t.id]
10485 );
10486 assert_eq!(
10487 reclaim_orphaned_running(&queue, 2, &Default::default()),
10488 [t.id.clone()]
10489 );
10490 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10491
10492 crate::conduct::apply(
10495 &queue,
10496 &questions,
10497 &crate::conduct::Verdict {
10498 decisions: vec![crate::conduct::Decision {
10499 id: t.id.clone(),
10500 recovery: Some(crate::conduct::Recovery::Requeue),
10501 ..crate::conduct::Decision::default()
10502 }],
10503 },
10504 )
10505 .unwrap();
10506 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10507 }
10508
10509 #[test]
10510 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10511 let dir = tempfile::tempdir().unwrap();
10512 let queue = Queue::at(dir.path().join("queue"));
10513 let home = dir.path().join("home");
10514
10515 let mut fresh = task();
10516 fresh.id = "20260101-000001-aaaa".to_owned();
10517 fresh.start("run-1".to_owned());
10518 queue.put(&mut fresh).unwrap();
10519
10520 let mut old = task();
10521 old.id = "20260101-000002-bbbb".to_owned();
10522 old.start("run-2".to_owned());
10523 queue.put(&mut old).unwrap();
10524 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10525
10526 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10527 assert_eq!(stalled.len(), 1);
10528 assert_eq!(stalled[0].id, old.id);
10529 }
10530
10531 #[test]
10532 fn queued_and_finished_task_views_partition_by_status() {
10533 let dir = tempfile::tempdir().unwrap();
10534 let queue = Queue::at(dir.path().join("queue"));
10535
10536 let mut queued = task();
10537 queued.id = "20260101-000001-aaaa".to_owned();
10538 queue.put(&mut queued).unwrap();
10539
10540 let mut failed = task();
10541 failed.id = "20260101-000002-bbbb".to_owned();
10542 failed.start("run-1".to_owned());
10543 failed.fail("gate red", 5);
10544 queue.put(&mut failed).unwrap();
10545
10546 let mut held = task();
10547 held.id = "20260101-000003-cccc".to_owned();
10548 held.hold_machine(None);
10549 queue.put(&mut held).unwrap();
10550
10551 let mut running = task();
10552 running.id = "20260101-000004-dddd".to_owned();
10553 running.start("run-2".to_owned());
10554 queue.put(&mut running).unwrap();
10555
10556 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
10557 assert_eq!(queued_ids, [queued.id.clone()]);
10558
10559 let mut finished_ids: Vec<String> =
10560 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
10561 finished_ids.sort_unstable();
10562 let mut want = vec![failed.id.clone(), held.id.clone()];
10563 want.sort_unstable();
10564 assert_eq!(finished_ids, want);
10565 }
10566
10567 #[test]
10568 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
10569 for short in [false, true] {
10570 let dir = tempfile::tempdir().unwrap();
10571 let queue = Queue::at(dir.path().join("queue"));
10572 let questions = ask::Questions::at(dir.path().join("questions"));
10573 let mut running = task();
10574 running.id = "20261013-192213-feb2".into();
10575 running.start("existing-run".into());
10576 queue.put(&mut running).unwrap();
10577 let mut middle = task();
10578 middle.id = "20261013-193611-ca5b".into();
10579 middle.block(
10580 vec![if short {
10581 "feb2".into()
10582 } else {
10583 running.id.clone()
10584 }],
10585 None,
10586 );
10587 queue.put(&mut middle).unwrap();
10588 let mut waiting = task();
10589 waiting.id = "20261013-202849-ee37".into();
10590 waiting.block(
10591 vec![if short {
10592 "ca5b".into()
10593 } else {
10594 middle.id.clone()
10595 }],
10596 Some("wait for implementation".into()),
10597 );
10598 queue.put(&mut waiting).unwrap();
10599 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
10600 let inv = crate::blockers::Inventory::load(&queue, &questions);
10601 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
10602 assert!(inv.stuck().is_empty());
10603 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10604 assert!(report.quarantined.is_empty());
10605 assert!(report.asked.is_empty());
10606 resolve_blockers(&queue, &questions);
10607 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10608 assert!(questions.list().is_empty());
10609 running.succeed();
10612 queue.put(&mut running).unwrap();
10613 resolve_blockers(&queue, &questions);
10614 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
10615 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10616 middle = queue.get(&middle.id).unwrap();
10617 middle.succeed();
10618 queue.put(&mut middle).unwrap();
10619 resolve_blockers(&queue, &questions);
10620 let after = queue.get(&waiting.id).unwrap();
10621 assert_eq!(after.status, TaskStatus::Queued);
10622 assert!(after.blocked_by.is_empty());
10623 assert_eq!(after.instruction, waiting.instruction);
10624 }
10625 }
10626
10627 #[test]
10628 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
10629 for future_schema in [false, true] {
10630 let name = "20261013-193611-ca5b";
10631 let mut future = task();
10632 future.id = name.into();
10633 future.schema = crate::queue::SCHEMA + 1;
10634 let contents = if future_schema {
10635 serde_json::to_string(&future).unwrap()
10636 } else {
10637 "corrupt".into()
10638 };
10639 let dir = tempfile::tempdir().unwrap();
10640 let queue = Queue::at(dir.path().join("queue"));
10641 let questions = ask::Questions::at(dir.path().join("questions"));
10642 let mut waiting = task();
10643 waiting.id = "20261013-202849-ee37".into();
10644 waiting.block(vec!["ca5b".into()], None);
10645 queue.put(&mut waiting).unwrap();
10646 std::fs::write(queue.path_of(name), contents).unwrap();
10647 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
10648 assert!(
10649 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
10650 );
10651 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10652 assert!(report.asked.is_empty());
10653 resolve_blockers(&queue, &questions);
10654 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10655 assert!(questions.list().is_empty());
10656 }
10657 let dir = tempfile::tempdir().unwrap();
10658 let queue = Queue::at(dir.path().join("queue"));
10659 let questions = ask::Questions::at(dir.path().join("questions"));
10660 let mut waiting = task();
10661 waiting.id = "20261013-202849-ee37".into();
10662 waiting.block(vec!["ca5b".into()], None);
10663 queue.put(&mut waiting).unwrap();
10664 let mut done = task();
10665 done.id = "20261013-193611-ca5b".into();
10666 done.succeed();
10667 queue.put(&mut done).unwrap();
10668 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
10671 assert!(queue.get("ca5b").is_err());
10672 assert_eq!(
10673 crate::queue::dependency(&queue, &questions, "ca5b"),
10674 crate::queue::Dependency::Ambiguous
10675 );
10676 resolve_blockers(&queue, &questions);
10677 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10678 assert!(report.asked.is_empty());
10679 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10680 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
10681 }
10682
10683 #[test]
10684 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
10685 for held in [false, true] {
10686 for question in [false, true] {
10687 for reference in ["ca5b", "20261013-193611-ca5b"] {
10688 let dir = tempfile::tempdir().unwrap();
10689 let queue = Queue::at(dir.path().join("queue"));
10690 let questions = ask::Questions::at(dir.path().join("questions"));
10691 let mut prerequisite = task();
10692 prerequisite.id = "20261013-193611-ca5b".into();
10693 prerequisite.start("existing-run".into());
10694 queue.put(&mut prerequisite).unwrap();
10695 let mut waiting = task();
10696 waiting.id = "20261013-202849-ee37".into();
10697 if held {
10698 waiting.hold_machine(Some("waiting for separate consent".into()));
10699 }
10700 queue.put(&mut waiting).unwrap();
10701 let reason = "Deleting saved data requires separate owner consent";
10702 crate::conduct::apply(
10703 &queue,
10704 &questions,
10705 &crate::conduct::Verdict {
10706 decisions: vec![crate::conduct::Decision {
10707 id: waiting.id.clone(),
10708 blocked_by: vec![reference.into()],
10709 question: question.then(|| "May the saved data be deleted?".into()),
10710 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
10711 reason: Some(reason.into()),
10712 ..crate::conduct::Decision::default()
10713 }],
10714 },
10715 )
10716 .unwrap();
10717 let before = queue.get(&waiting.id).unwrap();
10718 if question {
10719 assert_eq!(before.status, TaskStatus::Blocked);
10720 let q = questions.get(&before.blocked_by[0]).unwrap();
10721 assert_eq!(q.summary, "May the saved data be deleted?");
10722 assert!(q.status.open());
10723 } else {
10724 assert_eq!(before.status, TaskStatus::Held);
10725 let recorded_reason = before.hold_reason.as_deref().unwrap();
10726 assert!(recorded_reason.starts_with(reason));
10727 if held {
10728 assert!(recorded_reason.contains("waiting for separate consent"));
10729 }
10730 }
10731 prerequisite.succeed();
10732 queue.put(&mut prerequisite).unwrap();
10733 resolve_blockers(&queue, &questions);
10734 let after = queue.get(&waiting.id).unwrap();
10735 assert_eq!(after.status, before.status);
10736 assert_eq!(after.blocked_by, before.blocked_by);
10737 assert_eq!(after.hold_reason, before.hold_reason);
10738 assert_eq!(after.instruction, waiting.instruction);
10739 if question {
10740 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10741 }
10742 assert!(
10743 queue.next_runnable().is_none(),
10744 "dependency completion is not consent"
10745 );
10746 }
10747 }
10748 }
10749 }
10750
10751 #[test]
10752 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10753 let dir = tempfile::tempdir().unwrap();
10754 let queue = Queue::at(dir.path().join("queue"));
10755 let questions = ask::Questions::at(dir.path().join("questions"));
10756
10757 let mut dep = task();
10758 dep.id = "20260101-000001-dep0".to_owned();
10759 dep.succeed();
10760 queue.put(&mut dep).unwrap();
10761
10762 let mut still_going = task();
10763 still_going.id = "20260101-000002-dep1".to_owned();
10764 queue.put(&mut still_going).unwrap();
10765
10766 let mut blocked = task();
10767 blocked.id = "20260101-000003-main".to_owned();
10768 blocked.block(
10769 vec![dep.id.clone(), still_going.id.clone()],
10770 Some("waits on both".to_owned()),
10771 );
10772 queue.put(&mut blocked).unwrap();
10773
10774 resolve_blockers(&queue, &questions);
10775
10776 let after = queue.get(&blocked.id).unwrap();
10777 assert_eq!(
10778 after.status,
10779 TaskStatus::Blocked,
10780 "one dependency is still outstanding"
10781 );
10782 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10783 }
10784
10785 #[test]
10786 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10787 let dir = tempfile::tempdir().unwrap();
10788 let queue = Queue::at(dir.path().join("queue"));
10789 let questions = ask::Questions::at(dir.path().join("questions"));
10790
10791 let mut q = crate::ask::Question::new(
10792 "20260101-000001-main".to_owned(),
10793 crate::conduct::NODE.to_owned(),
10794 "conduct".to_owned(),
10795 "Which backend?".to_owned(),
10796 String::new(),
10797 Vec::new(),
10798 );
10799 questions.put(&mut q).unwrap();
10800 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10801 .unwrap();
10802 questions.put(&mut q).unwrap();
10803
10804 let mut blocked = task();
10805 blocked.id = "20260101-000001-main".to_owned();
10806 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10807 queue.put(&mut blocked).unwrap();
10808
10809 resolve_blockers(&queue, &questions);
10810
10811 let after = queue.get(&blocked.id).unwrap();
10812 assert_eq!(
10813 after.status,
10814 TaskStatus::Queued,
10815 "the only blocker resolved"
10816 );
10817 assert_eq!(after.answers.len(), 1);
10818 assert_eq!(after.answers[0].question, "Which backend?");
10819 assert_eq!(after.answers[0].answer, "SQLite");
10820
10821 let instruction = instruction_for(&after);
10823 assert!(instruction.contains("Which backend?"));
10824 assert!(instruction.contains("SQLite"));
10825 }
10826
10827 #[test]
10828 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10829 let dir = tempfile::tempdir().unwrap();
10830 let queue = Queue::at(dir.path().join("queue"));
10831 let questions = ask::Questions::at(dir.path().join("questions"));
10832
10833 let mut q = crate::ask::Question::new(
10834 "20260101-000001-main".to_owned(),
10835 crate::conduct::NODE.to_owned(),
10836 "conduct".to_owned(),
10837 "Is the setup done?".to_owned(),
10838 String::new(),
10839 Vec::new(),
10840 );
10841 q.abandon("no answer within 60s of asking");
10842 questions.put(&mut q).unwrap();
10843
10844 let mut blocked = task();
10845 blocked.id = "20260101-000001-main".to_owned();
10846 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10847 queue.put(&mut blocked).unwrap();
10848
10849 resolve_blockers(&queue, &questions);
10850
10851 let after = queue.get(&blocked.id).unwrap();
10852 assert_eq!(
10853 after.status,
10854 TaskStatus::Held,
10855 "never left blocked on nothing"
10856 );
10857 assert!(!after.operator_held(), "a machine hold, for triage");
10858 assert!(
10859 after
10860 .hold_reason
10861 .as_deref()
10862 .unwrap_or_default()
10863 .contains("went unanswered")
10864 );
10865 }
10866
10867 #[test]
10868 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10869 let dir = tempfile::tempdir().unwrap();
10875 let queue = Queue::at(dir.path().join("queue"));
10876 let questions = ask::Questions::at(dir.path().join("questions"));
10877
10878 let mut q = crate::ask::Question::new(
10879 "20260101-000001-main".to_owned(),
10880 crate::conduct::NODE.to_owned(),
10881 "conduct".to_owned(),
10882 "How should this be handled?".to_owned(),
10883 String::new(),
10884 Vec::new(),
10885 );
10886 questions.put(&mut q).unwrap();
10887 q.answer(crate::ask::Answer::Text(
10888 "leave it held, a human will look at it later".to_owned(),
10889 ))
10890 .unwrap();
10891 questions.put(&mut q).unwrap();
10892
10893 let mut held = task();
10894 held.id = "20260101-000001-main".to_owned();
10895 held.hold_machine(Some("out of attempts".to_owned()));
10896 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10897 queue.put(&mut held).unwrap();
10898
10899 resolve_blockers(&queue, &questions);
10900
10901 let after = queue.get(&held.id).unwrap();
10902 assert_eq!(after.status, TaskStatus::Held);
10903 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
10904 assert_eq!(
10905 after.answers[0].answer,
10906 "leave it held, a human will look at it later"
10907 );
10908 }
10909
10910 #[test]
10911 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
10912 let dir = tempfile::tempdir().unwrap();
10913 let queue = Queue::at(dir.path().join("queue"));
10914 let questions = ask::Questions::at(dir.path().join("questions"));
10915 let mut dep = task();
10916 dep.id = "20260101-000001-gone".to_owned();
10917 queue.put(&mut dep).unwrap();
10918 let mut blocked = task();
10919 blocked.id = "20260101-000003-main".to_owned();
10920 blocked.block(vec![dep.id.clone()], None);
10921 queue.put(&mut blocked).unwrap();
10922
10923 let claim = queue.claim(&blocked.id).unwrap();
10925 queue.remove(&dep.id, false, &questions).unwrap();
10926 drop(claim);
10927 resolve_blockers(&queue, &questions);
10928
10929 let after = queue.get(&blocked.id).unwrap();
10930 assert_eq!(after.status, TaskStatus::Queued);
10931 assert!(after.blocked_by.is_empty());
10932 }
10933
10934 #[test]
10935 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
10936 let dir = tempfile::tempdir().unwrap();
10942 let queue = Queue::at(dir.path().join("queue"));
10943 let questions = ask::Questions::at(dir.path().join("questions"));
10944
10945 let mut still_going = task();
10946 still_going.id = "20260101-000002-dep1".to_owned();
10947 queue.put(&mut still_going).unwrap();
10948
10949 let mut blocked = task();
10950 blocked.id = "20260101-000003-main".to_owned();
10951 blocked.block(
10952 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
10953 Some("waits on both".to_owned()),
10954 );
10955 queue.put(&mut blocked).unwrap();
10956
10957 resolve_blockers(&queue, &questions);
10958
10959 let after = queue.get(&blocked.id).unwrap();
10960 assert_eq!(
10961 after.status,
10962 TaskStatus::Held,
10963 "a missing dependency must not leave the task blocked forever"
10964 );
10965 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
10966 assert!(after.blocked_by.is_empty());
10967 let reason = after.hold_reason.as_deref().unwrap_or_default();
10968 assert!(
10969 reason.contains("20260101-000001-gone"),
10970 "the missing id must be named so an operator can tell what happened: {reason}"
10971 );
10972 assert!(
10973 reason.contains(&still_going.id),
10974 "the still-valid dependency must not silently vanish from the record: {reason}"
10975 );
10976 }
10977
10978 #[test]
10979 fn instruction_for_is_unchanged_without_any_answers() {
10980 let t = task();
10981 assert_eq!(instruction_for(&t), t.instruction);
10982 }
10983
10984 #[test]
10985 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
10986 let dir = tempfile::tempdir().unwrap();
10987 let q = Queue::at(dir.path().join("queue"));
10988 let src = dir.path().join("shot.png");
10989 std::fs::write(&src, "x").unwrap();
10990 let mut t = task();
10991 q.attach(&mut t, &[src]).unwrap();
10992 let paths = task_attachments(&q, &t).unwrap();
10993 assert_eq!(paths.len(), 1);
10994 assert!(paths[0].is_absolute() && paths[0].is_file());
10995 std::fs::remove_file(&paths[0]).unwrap();
10996 let err = task_attachments(&q, &t).unwrap_err().to_string();
10997 assert!(err.contains("shot.png"), "{err}");
10998 }
10999
11000 #[tokio::test]
11001 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
11002 let dir = tempfile::tempdir().unwrap();
11003 let queue_path = dir.path().join("queue");
11004 let queue = Queue::at(queue_path.clone());
11005 let mut t = task();
11006 t.source = Source::Agent {
11007 run: "originating-chat".into(),
11008 node: crate::queue::CHAT_NODE.into(),
11009 };
11010 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
11011 stage: "config-fetch",
11012 target: "origin/main".into(),
11013 });
11014 record_config_failure(&mut t, &fetch);
11015 t.hold_machine(Some(format!("{fetch:#}")));
11016 let fetch_key = t.chat_notice_due().unwrap();
11017 t.chat_notices.push(fetch_key.clone());
11018 queue.put(&mut t).unwrap();
11019
11020 let queue = Queue::at(queue_path);
11021 let mut t = queue.get(&t.id).unwrap();
11022 let a = dir.path().join("a.toml");
11023 let b = dir.path().join("b.toml");
11024 for (path, text, new_cause) in [
11025 (&a, "[graph", true),
11026 (&a, "[verify", false),
11027 (&a, "[graph]\nimplementers = 'invalid-type'", true),
11028 (&b, "[graph]\nimplementers = 'invalid-type'", true),
11029 ] {
11030 t.release_operator("test", None);
11031 std::fs::write(path, text).unwrap();
11032 let error = Config::discover_fetched(dir.path(), Some(path))
11033 .await
11034 .unwrap_err();
11035 record_config_failure(&mut t, &error);
11036 t.hold_machine(Some(format!("{error:#}")));
11037 let due = t.chat_notice_due();
11038 assert_eq!(due.is_some(), new_cause, "{error:#}");
11039 if let Some(key) = due {
11040 assert_ne!(key, fetch_key);
11041 t.chat_notices.push(key);
11042 }
11043 }
11044 t.succeed();
11045 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11046 }
11047
11048 #[test]
11049 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11050 let dir = tempfile::tempdir().unwrap();
11051 let queue_path = dir.path().join("queue");
11052 let queue = Queue::at(queue_path.clone());
11053 let mut t = task();
11054 t.source = crate::queue::Source::Agent {
11055 run: "originating-chat".into(),
11056 node: crate::queue::CHAT_NODE.into(),
11057 };
11058 let a = dir.path().join("a.png");
11059 let b = dir.path().join("b.png");
11060 std::fs::write(&a, "a").unwrap();
11061 std::fs::write(&b, "b").unwrap();
11062 queue.attach(&mut t, &[a, b]).unwrap();
11063 let paths = task_attachments(&queue, &t).unwrap();
11064 for path in &paths {
11065 std::fs::remove_file(path).unwrap();
11066 }
11067 let missing = task_attachments(&queue, &t).unwrap_err();
11068 t.attempts += 1;
11069 let reason = missing.to_string();
11070 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11071 let first = t.chat_notice_due().expect("first attachment notice");
11072 t.chat_notices.push(first.clone());
11073 queue.put(&mut t).unwrap();
11074
11075 let queue = Queue::at(queue_path.clone());
11076 let mut t = queue.get(&t.id).unwrap();
11077 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11078 t.release_operator("test", None);
11079 t.attempts += 1;
11080 let missing = task_attachments(&queue, &t).unwrap_err();
11081 t.fail_startup_target(
11082 "startup-attachments",
11083 missing.name,
11084 "same missing attachment with regenerated wording",
11085 1,
11086 );
11087 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11088
11089 std::fs::write(&paths[0], "restored").unwrap();
11090 t.release_operator("test", None);
11091 t.attempts += 1;
11092 let missing = task_attachments(&queue, &t).unwrap_err();
11093 let reason = missing.to_string();
11094 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11095 let second = t
11096 .chat_notice_due()
11097 .expect("different attachment must notify");
11098 assert_ne!(first, second);
11099 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11100 }
11101
11102 #[test]
11103 fn resumed_instruction_is_unchanged_without_any_answers() {
11104 let t = task();
11105 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11106 }
11107
11108 #[test]
11109 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11110 let mut t = task();
11111 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11112 let old = t.instruction.clone();
11116
11117 let refreshed = resumed_instruction(&old, &t);
11118 assert!(refreshed.starts_with(&old), "the original text is kept");
11119 assert!(refreshed.contains("Which backend?"));
11120 assert!(refreshed.contains("SQLite"));
11121 }
11122
11123 #[test]
11124 fn resumed_instruction_keeps_an_original_answers_heading() {
11125 let mut t = task();
11126 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11127 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11128
11129 let refreshed = resumed_instruction(&t.instruction, &t);
11130
11131 assert!(
11132 refreshed.starts_with(&t.instruction),
11133 "an answers heading in the original instruction is not the appended block"
11134 );
11135 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11136 assert!(refreshed.contains("Which backend?"));
11137 assert!(refreshed.contains("SQLite"));
11138
11139 let repeated = resumed_instruction(&refreshed, &t);
11140 assert_eq!(
11141 repeated, refreshed,
11142 "only the final appended block is refreshed"
11143 );
11144 }
11145
11146 #[test]
11147 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11148 let mut t = task();
11149 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11150
11151 let once = resumed_instruction(&t.instruction, &t);
11155 let twice = resumed_instruction(&once, &t);
11156 assert_eq!(once, twice);
11157 assert_eq!(once.matches("Which backend?").count(), 1);
11158
11159 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
11161 let refreshed = resumed_instruction(&once, &t);
11162 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
11163 assert!(refreshed.contains("Which backend?"));
11164 assert!(refreshed.contains("Which cache?"));
11165 }
11166
11167 #[test]
11168 fn prepare_instruction_covers_all_three_starters() {
11169 let mut t = task();
11170 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11171
11172 assert_eq!(
11175 prepare_instruction(&Starter::Start, None, &t),
11176 Some(instruction_for(&t))
11177 );
11178
11179 let old = t.instruction.clone();
11182 assert_eq!(
11183 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
11184 Some(resumed_instruction(&old, &t))
11185 );
11186
11187 assert_eq!(
11191 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
11192 None
11193 );
11194 }
11195
11196 #[test]
11197 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
11198 assert_eq!(
11199 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
11200 Starter::Review("magi/eba2/A".to_owned())
11201 );
11202 }
11203
11204 #[test]
11205 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
11206 assert_eq!(
11207 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
11208 Starter::Start,
11209 "a vanished review branch must not fall back to resuming the old run either"
11210 );
11211 }
11212
11213 #[test]
11214 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11215 let mut t = task();
11216 t.start("old-run".to_owned());
11217 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11218 t.release();
11219 let branch = t.review_branch.take();
11220 assert_eq!(
11221 choose_starter(branch.as_deref(), true, Some("old-run")),
11222 Starter::Review("magi/eba2/A".to_owned()),
11223 "a review wins over resuming the old run"
11224 );
11225 }
11226
11227 #[test]
11228 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11229 assert_eq!(
11230 choose_starter(None, false, Some("some-run")),
11231 Starter::Resume("some-run".to_owned())
11232 );
11233 assert_eq!(choose_starter(None, false, None), Starter::Start);
11234 }
11235
11236 #[test]
11237 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11238 let mut released = task();
11239 released.start("stalled-run".to_owned());
11240 released.requeue();
11241 let unfinished = (!released.fresh_start)
11242 .then(|| Some("stalled-run".to_owned()))
11243 .flatten();
11244 assert_eq!(
11245 choose_starter(None, false, unfinished.as_deref()),
11246 Starter::Start,
11247 "release keeps run history but must not resume it"
11248 );
11249 assert_eq!(released.runs, ["stalled-run"]);
11250 }
11251
11252 fn stopped_public_run(status: RunStatus) -> RunState {
11253 let mut state = run_state(status);
11254 state.driver_exited = true;
11255 state.config.graph.land = true;
11256 state.config.graph.review_rounds = 1;
11257 state.reviews = vec![review_round(1)];
11258 state.merge = Some(crate::run::MergeOutcome {
11259 mode: MergeMode::Pr,
11260 ok: false,
11261 detail: "landing stopped".into(),
11262 empty: false,
11263 });
11264 state.pr = Some(crate::run::PrRecord {
11265 url: "https://example.test/pull/1".into(),
11266 number: 1,
11267 state: "open".into(),
11268 checks: "red".into(),
11269 round: 1,
11270 rounds: 1,
11271 red_at_merge: Vec::new(),
11272 });
11273 state
11274 }
11275
11276 #[test]
11277 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11278 for status in [RunStatus::Blocked, RunStatus::Gating] {
11279 let home = tempfile::tempdir().unwrap();
11280 let mut state = stopped_public_run(status);
11281 state.save_under(home.path()).unwrap();
11282 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11283 assert!(!exhausted_review_budget(&state));
11284 let mut task = held_task_with(&state.id);
11285 task.release();
11286 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11287 assert_eq!(
11288 choose_starter(None, false, unfinished.as_deref()),
11289 Starter::Resume(state.id.clone())
11290 );
11291 let question = action_question(&state.id, resume_action(&state.id));
11292 assert_eq!(
11293 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11294 ActionDecision::Resume(state.id.clone())
11295 );
11296 assert_eq!(
11298 crate::resume::validate(home.path(), &state.id).unwrap().id,
11299 state.id
11300 );
11301 let queue = Queue::at(home.path().join("queue"));
11302 let mut controlled = Task::new(
11303 "resume land".into(),
11304 state.instruction.clone(),
11305 state.repo.clone(),
11306 Source::Agent {
11307 run: "talk".into(),
11308 node: crate::queue::CHAT_NODE.into(),
11309 },
11310 );
11311 controlled.runs.push(state.id.clone());
11312 controlled.hold_machine(Some("land stopped".into()));
11313 queue.put(&mut controlled).unwrap();
11314 let resumed =
11315 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11316 assert_eq!(resumed.status, TaskStatus::Queued);
11317 assert_eq!(
11318 resumed.resume_override.unwrap().pinned_run,
11319 Some(state.id.clone())
11320 );
11321 assert_eq!(resumed.attempts, 0);
11322 }
11323 }
11324
11325 #[test]
11326 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11327 for review_rounds in [0, 1] {
11328 let home = tempfile::tempdir().unwrap();
11329 let mut state = run_state(RunStatus::Blocked);
11330 state.config.graph.review_rounds = review_rounds;
11331 if review_rounds > 0 {
11332 let mut review = review_round(1);
11333 review.clean = true;
11334 state.reviews = vec![review];
11335 }
11336 state.gate_ran = true;
11337 state.gate = vec![crate::run::CommandOutcome {
11338 command: "verify".into(),
11339 code: Some(1),
11340 output_tail: "gate failed after repair budget exhausted".into(),
11341 duration_ms: 0,
11342 resource_blocked: false,
11343 }];
11344 state.save_under(home.path()).unwrap();
11345 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11346 assert!(exhausted_review_budget(&state));
11347 let mut task = held_task_with(&state.id);
11348 task.release();
11349 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11350 let question = action_question(&state.id, resume_action(&state.id));
11351 assert_ne!(
11352 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11353 ActionDecision::Resume(state.id.clone())
11354 );
11355 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11356
11357 state.gate_ran = false;
11359 state.gate.clear();
11360 assert!(!exhausted_review_budget(&state));
11361 assert_eq!(
11362 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11363 Some(state.id)
11364 );
11365 }
11366 }
11367
11368 #[test]
11369 fn resource_blocked_verification_resumes_the_same_run() {
11370 for (kind, stage) in [
11371 ("cache", "clean"),
11372 ("coordination", "clean"),
11373 ("cache", "review-e2e"),
11374 ("coordination", "no-reviews"),
11375 ("coordination", "handoff"),
11376 ] {
11377 let home = tempfile::tempdir().unwrap();
11378 let mut state = run_state(RunStatus::Blocked);
11379 state.config.graph.review_rounds = 1;
11380 let mut review = review_round(1);
11381 review.clean = true;
11382 state.reviews = vec![review];
11383 if stage == "review-e2e" {
11384 let review = &mut state.reviews[0];
11385 review.clean = false;
11386 review.blocking = 1;
11387 review.e2e = vec![crate::run::CommandOutcome {
11388 command: "verify".into(),
11389 code: None,
11390 output_tail: "resource wait expired".into(),
11391 duration_ms: 0,
11392 resource_blocked: true,
11393 }];
11394 } else if stage == "no-reviews" {
11395 state.config.graph.review_rounds = 0;
11396 state.reviews.clear();
11397 } else if stage == "handoff" {
11398 let review = &mut state.reviews[0];
11401 review.clean = false;
11402 review.blocking = 1;
11403 review.e2e = vec![crate::run::CommandOutcome {
11404 command: "verify".into(),
11405 code: Some(0),
11406 output_tail: String::new(),
11407 duration_ms: 0,
11408 resource_blocked: false,
11409 }];
11410 }
11411 state.gate_ran = false;
11412 state.resource_block = Some(crate::run::ResourceBlock {
11413 cause: crate::run::ResourceBlockCause::Other,
11414 node: if stage == "review-e2e" {
11415 "review"
11416 } else {
11417 "gate"
11418 }
11419 .into(),
11420 kind: kind.into(),
11421 detail: "resource wait expired".into(),
11422 at: jiff::Timestamp::now(),
11423 });
11424 state.save_under(home.path()).unwrap();
11425 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11426 assert!(!exhausted_review_budget(&state));
11427 let mut task = held_task_with(&state.id);
11428 task.release();
11429 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11430 assert_eq!(
11431 choose_starter(None, false, unfinished.as_deref()),
11432 Starter::Resume(state.id.clone())
11433 );
11434 let question = action_question(&state.id, resume_action(&state.id));
11435 assert_eq!(
11436 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11437 ActionDecision::Resume(state.id.clone())
11438 );
11439 assert_eq!(
11440 crate::resume::validate(home.path(), &state.id).unwrap().id,
11441 state.id
11442 );
11443 let queue = Queue::at(home.path().join("queue"));
11444 let mut controlled = Task::new(
11445 "resume verification".into(),
11446 state.instruction.clone(),
11447 state.repo.clone(),
11448 Source::Agent {
11449 run: "talk".into(),
11450 node: crate::queue::CHAT_NODE.into(),
11451 },
11452 );
11453 controlled.runs.push(state.id.clone());
11454 controlled.hold_machine(Some("verification resource unavailable".into()));
11455 queue.put(&mut controlled).unwrap();
11456 let resumed =
11457 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11458 assert_eq!(resumed.status, TaskStatus::Queued);
11459 assert_eq!(
11460 resumed.resume_override.unwrap().pinned_run,
11461 Some(state.id.clone())
11462 );
11463 assert_eq!(resumed.attempts, 0);
11464
11465 state.config.graph.review_rounds = 1;
11467 let mut unresolved = review_round(1);
11468 unresolved.blocking = 1;
11469 state.reviews = vec![unresolved];
11470 state.resource_block = None;
11471 state.save_under(home.path()).unwrap();
11472 assert!(exhausted_review_budget(&state));
11473 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11474 }
11475 }
11476
11477 #[tokio::test]
11478 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11479 if !crate::standalone_test(
11480 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
11481 ) {
11482 return;
11483 }
11484 let home = tempfile::tempdir().unwrap();
11485 let mut owner = stopped_public_run(RunStatus::Blocked);
11486 owner.save_under(home.path()).unwrap();
11487 let history = vec![owner.id.clone()];
11488 let repo = owner.repo.clone();
11489 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11490 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11491 let initial =
11492 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11493 assert!(
11494 initial.is_pending(),
11495 "a busy record must not hold or resume the task"
11496 );
11497 owner.retired_to = Some("20260101-000001-bbbb".into());
11498 owner.save_under(home.path()).unwrap();
11499 drop(lock);
11500 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11501 .await
11502 .expect("ownership lock released")
11503 .expect("read settled owner");
11504 assert!(
11505 result.is_none(),
11506 "the retired owner must not be resumed from a stale read"
11507 );
11508 }
11509
11510 #[tokio::test]
11511 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11512 if !crate::standalone_test(
11513 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
11514 ) {
11515 return;
11516 }
11517 let home = tempfile::tempdir().unwrap();
11518 let mut owner = stopped_public_run(RunStatus::Blocked);
11519 let repo = tempfile::tempdir().unwrap();
11520 crate::git::git(repo.path(), &["init"]).await.unwrap();
11521 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11522 owner.id = "20260101-000000-aaaa".into();
11523 owner.save_under(home.path()).unwrap();
11524 let mut successor = run_state(RunStatus::Blocked);
11525 successor.id = "20260101-000001-bbbb".into();
11526 successor.save_under(home.path()).unwrap();
11527 let runs = vec![owner.id.clone(), successor.id.clone()];
11528 assert_eq!(
11529 public_resume_owner(home.path(), &runs, &owner.repo)
11530 .await
11531 .unwrap()
11532 .unwrap()
11533 .id,
11534 owner.id
11535 );
11536 let subdir = owner.repo.join("nested");
11537 std::fs::create_dir(&subdir).unwrap();
11538 assert_eq!(
11539 public_resume_owner(home.path(), &runs, &subdir)
11540 .await
11541 .unwrap()
11542 .unwrap()
11543 .id,
11544 owner.id
11545 );
11546 #[cfg(unix)]
11547 {
11548 let alias = home.path().join("repo-alias");
11549 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
11550 assert_eq!(
11551 public_resume_owner(home.path(), &runs, &alias.join("nested"))
11552 .await
11553 .unwrap()
11554 .unwrap()
11555 .id,
11556 owner.id
11557 );
11558 }
11559 let different = tempfile::tempdir().unwrap();
11560 crate::git::git(different.path(), &["init"]).await.unwrap();
11561 assert!(
11562 public_resume_owner(home.path(), &runs, different.path())
11563 .await
11564 .is_err()
11565 );
11566 std::fs::remove_file(
11567 home.path()
11568 .join("runs")
11569 .join(&successor.id)
11570 .join("run.json"),
11571 )
11572 .unwrap();
11573 assert!(
11574 public_resume_owner(home.path(), &runs, &owner.repo)
11575 .await
11576 .is_err()
11577 );
11578 }
11579
11580 #[test]
11581 fn an_ordinary_release_keeps_a_resumable_run_available() {
11582 let mut released = task();
11583 released.start("stalled-run".to_owned());
11584 released.release();
11585 let unfinished = (!released.fresh_start)
11586 .then(|| Some("stalled-run".to_owned()))
11587 .flatten();
11588 assert_eq!(
11589 choose_starter(None, false, unfinished.as_deref()),
11590 Starter::Resume("stalled-run".to_owned()),
11591 "manual release must preserve the normal resume path"
11592 );
11593 }
11594
11595 #[test]
11596 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
11597 let mut state = run_state(RunStatus::Blocked);
11598 state.config.graph.review_rounds = 3;
11599 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
11600 assert!(exhausted_review_budget(&state));
11601
11602 state.reviews.pop();
11604 assert!(!exhausted_review_budget(&state));
11605
11606 let mut stalled = run_state(RunStatus::Stalled);
11609 stalled.config.graph.review_rounds = 1;
11610 stalled.reviews = vec![review_round(1)];
11611 assert!(!exhausted_review_budget(&stalled));
11612 }
11613
11614 fn review_round(round: usize) -> crate::run::ReviewRound {
11615 crate::run::ReviewRound {
11616 round,
11617 head: "deadbeef".to_owned(),
11618 verified_head: None,
11619 verified_at: None,
11620 reviews: Vec::new(),
11621 e2e: Vec::new(),
11622 verify_retried: false,
11623 e2e_deferred: false,
11624 e2e_defer_reason: None,
11625 fix: None,
11626 blocking: 0,
11627 answered: 1,
11628 expected: 1,
11629 clean: false,
11630 progressed: true,
11631 vote_split: false,
11632 reconsideration: Vec::new(),
11633 verdict: None,
11634 }
11635 }
11636
11637 #[test]
11638 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
11639 let mut run = RunState::new(
11640 PathBuf::from("/repo"),
11641 "main".to_owned(),
11642 "abc1234def".to_owned(),
11643 "add retries".to_owned(),
11644 Config::default(),
11645 );
11646 run.status = RunStatus::Judging;
11647 run.parked = true;
11648 let mut task = Task::new(
11649 "add retries".to_owned(),
11650 "add retries".to_owned(),
11651 PathBuf::from("/repo"),
11652 crate::queue::Source::Human,
11653 );
11654 task.status = TaskStatus::Failed;
11655 task.runs = vec![run.id.clone()];
11656 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
11657 assert!(with(&task, &run), "parked after judging is the case");
11658 let mut parked_task = task.clone();
11659 parked_task.status = TaskStatus::Parked;
11660 assert!(with(&parked_task, &run), "the Parked status is the case");
11661
11662 let mut not_parked = run.clone();
11663 not_parked.parked = false;
11664 not_parked.status = RunStatus::Stalled;
11665 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
11666
11667 let mut fresh = task.clone();
11668 fresh.fresh_start = true;
11669 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
11670
11671 let mut review = task.clone();
11672 review.review_branch = Some("magi/x/A".to_owned());
11673 assert!(!with(&review, &run), "review is ranked before resume");
11674
11675 let mut held = task.clone();
11676 held.status = TaskStatus::Held;
11677 assert!(!with(&held, &run), "a hold stays visible to the conductor");
11678
11679 let mut released = run.clone();
11680 released.released_to = Some("20260901-000000-new1".to_owned());
11681 assert!(!with(&task, &released), "nothing left to resume into");
11682
11683 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
11684 "unreadable"
11685 )));
11686 }
11687
11688 #[test]
11689 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
11690 let mut released = RunState::new(
11691 PathBuf::from("/repo"),
11692 "main".to_owned(),
11693 "abc1234def".to_owned(),
11694 "add retries".to_owned(),
11695 Config::default(),
11696 );
11697 released.status = RunStatus::Blocked;
11698 assert_eq!(
11699 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11700 Some(released.id.clone())
11701 );
11702 released.released_to = Some("20260901-000000-new1".to_owned());
11703 assert_eq!(
11704 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11705 None,
11706 "there is nothing left to resume it into"
11707 );
11708 }
11709
11710 #[test]
11711 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
11712 let mut exhausted = RunState::new(
11721 PathBuf::from("/repo"),
11722 "main".to_owned(),
11723 "abc1234def".to_owned(),
11724 "add retries".to_owned(),
11725 Config::default(),
11726 );
11727 exhausted.status = RunStatus::Blocked;
11728 exhausted.config.graph.review_rounds = 1;
11729 exhausted.reviews = vec![review_round(1)];
11730
11731 assert_eq!(
11732 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
11733 None,
11734 "an exhausted `Blocked` run must not be offered as resumable"
11735 );
11736
11737 let mut has_budget_left = RunState::new(
11740 PathBuf::from("/repo"),
11741 "main".to_owned(),
11742 "abc1234def".to_owned(),
11743 "add retries".to_owned(),
11744 Config::default(),
11745 );
11746 has_budget_left.status = RunStatus::Blocked;
11747 has_budget_left.config.graph.review_rounds = 3;
11748 has_budget_left.reviews = vec![review_round(1)];
11749
11750 assert_eq!(
11751 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
11752 Ok(has_budget_left.clone())
11753 }),
11754 Some(has_budget_left.id.clone())
11755 );
11756 }
11757
11758 #[test]
11759 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
11760 let mut run = RunState::new(
11761 PathBuf::from("/repo"),
11762 "main".to_owned(),
11763 "abc1234def".to_owned(),
11764 "add retries".to_owned(),
11765 Config::default(),
11766 );
11767 run.status = RunStatus::Blocked;
11768 run.config.graph.review_rounds = 1;
11769 run.reviews = vec![review_round(1)];
11770 run.pr_publication_pending = true;
11771 assert_eq!(
11772 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11773 Some(run.id.clone()),
11774 "a spent review budget must not turn a failed publication into a re-competition"
11775 );
11776 }
11777
11778 #[test]
11779 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11780 let mut task = Task::new(
11781 "recover work".into(),
11782 "recover work".into(),
11783 PathBuf::from("/repo"),
11784 crate::queue::Source::Human,
11785 );
11786 let mut history = std::collections::BTreeMap::new();
11787 for index in 0..8 {
11788 let mut run = run_state(RunStatus::Blocked);
11789 run.id = format!("20260101-000000-{index:04}");
11790 run.base_commit = format!("upstream-{index}");
11791 if index > 0 {
11793 run.candidates.clear();
11794 }
11795 task.runs.extend([run.id.clone(), run.id.clone()]);
11796 history.insert(run.id.clone(), run);
11797 task.requeue();
11798 }
11799 let check = |task: &Task, starter: &Starter, limit| {
11800 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11801 };
11802 assert!(
11803 check(&task, &Starter::Start, 8)
11804 .unwrap()
11805 .contains("8 unretired")
11806 );
11807 assert!(check(&task, &Starter::Start, 9).is_none());
11808 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11809 task.release_operator("owner", None);
11810 assert_eq!(task.attempts, 0);
11811 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11812 assert!(check(&restored, &Starter::Start, 8).is_some());
11813 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11814 assert!(check(&restored, &Starter::Start, 0).is_none());
11815 }
11816
11817 #[test]
11818 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11819 let mut task = Task::new(
11820 "recover work".into(),
11821 "recover work".into(),
11822 PathBuf::from("/repo"),
11823 crate::queue::Source::Human,
11824 );
11825 let mut run = run_state(RunStatus::Superseded);
11826 run.candidates.clear();
11827 task.runs.push(run.id.clone());
11828 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11829 run.retired_to = Some("authority".into());
11830 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11831 run.retired_to = None;
11832 run.released_to = Some("authority".into());
11833 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11834 assert!(
11835 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11836 .unwrap()
11837 .contains("Cannot verify")
11838 );
11839 assert!(
11841 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11842 panic!("resume must not load historical attempts")
11843 })
11844 .is_none()
11845 );
11846 }
11847
11848 #[test]
11849 fn retired_resume_requires_a_valid_same_repository_authority() {
11850 let mut source = run_state(RunStatus::Blocked);
11851 let mut empty = run_state(RunStatus::Blocked);
11852 empty.retired_to = Some(source.id.clone());
11853 let runs = vec![source.id.clone(), empty.id.clone()];
11854 let resolve = |source: &RunState, empty: &RunState| {
11855 unfinished_run_with(&runs, "fixture", |id| {
11856 if id == source.id {
11857 Ok(source.clone())
11858 } else if id == empty.id {
11859 Ok(empty.clone())
11860 } else {
11861 bail!("unreadable authority")
11862 }
11863 })
11864 };
11865 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11866 assert_eq!(
11867 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11868 Starter::Resume(source.id.clone())
11869 );
11870 source.retired_to = Some(empty.id.clone());
11871 assert!(
11872 resolve(&source, &empty).is_none(),
11873 "cycles do not revive obsolete runs"
11874 );
11875 source.retired_to = None;
11876 source.repo = PathBuf::from("/another-repository");
11877 assert!(resolve(&source, &empty).is_none());
11878 source.repo = empty.repo.clone();
11879 source.released_to = Some("later".into());
11880 assert!(resolve(&source, &empty).is_none());
11881 empty.retired_to = Some("unlisted-run".into());
11882 assert!(resolve(&source, &empty).is_none());
11883 }
11884
11885 #[test]
11886 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11887 let mut older_stalled = RunState::new(
11895 PathBuf::from("/repo"),
11896 "main".to_owned(),
11897 "abc1234def".to_owned(),
11898 "add retries".to_owned(),
11899 Config::default(),
11900 );
11901 older_stalled.status = RunStatus::Stalled;
11902
11903 let mut newest_exhausted = RunState::new(
11904 PathBuf::from("/repo"),
11905 "main".to_owned(),
11906 "abc1234def".to_owned(),
11907 "add retries".to_owned(),
11908 Config::default(),
11909 );
11910 newest_exhausted.status = RunStatus::Blocked;
11911 newest_exhausted.config.graph.review_rounds = 1;
11912 newest_exhausted.reviews = vec![review_round(1)];
11913
11914 assert_eq!(
11915 unfinished_run_with(
11916 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
11917 "t",
11918 |_| Ok(newest_exhausted.clone())
11919 ),
11920 None,
11921 "the newest run is exhausted, so nothing here is worth resuming - \
11922 least of all the older, already-superseded run"
11923 );
11924 }
11925
11926 #[test]
11927 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
11928 assert_eq!(
11929 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
11930 Err(anyhow::anyhow!("fixture is absent"))
11931 }),
11932 None
11933 );
11934 }
11935
11936 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
11937 let mut q = ask::Question::new(
11938 run.to_owned(),
11939 "implement".to_owned(),
11940 "impl-A".to_owned(),
11941 "continue?".to_owned(),
11942 String::new(),
11943 vec!["resume で続行する".to_owned(), "other".to_owned()],
11944 );
11945 q.actions.insert("resume で続行する".to_owned(), action);
11946 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
11947 .unwrap();
11948 q
11949 }
11950
11951 fn held_task_with(run: &str) -> Task {
11952 let mut t = task();
11953 t.runs = vec![run.to_owned()];
11954 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
11955 t
11956 }
11957
11958 fn resume_action(run: &str) -> ask::ChoiceAction {
11959 ask::ChoiceAction::Resume { run: run.into() }
11960 }
11961
11962 #[test]
11963 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
11964 for parked in [false, true] {
11965 let mut t = task();
11966 t.start("A".into());
11967 t.start("B".into());
11968 assert_eq!(t.runs, vec!["A", "B"]);
11969 t.start("A".into());
11970 let mut resumed = run_state(RunStatus::Blocked);
11971 resumed.id = "A".into();
11972 resumed.parked = parked;
11973 resumed.driver_exited = true;
11974 t.hold_machine(Some("resumed run stopped".into()));
11975 let q = action_question("A", resume_action("A"));
11976 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
11977 assert_eq!(
11978 decide_action(&t, &q, &PHRASES_EN, |id| {
11979 assert_eq!(id, "A");
11980 Ok(resumed.clone())
11981 }),
11982 ActionDecision::Resume("A".into())
11983 );
11984 assert_eq!(
11985 action_standing(&t, &action_question("B", resume_action("B"))),
11986 ActionStanding::Stale
11987 );
11988 t.release_operator("cli", None);
11989 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
11990 assert_eq!(id, "A");
11991 Ok(resumed.clone())
11992 });
11993 assert_eq!(
11994 choose_starter(None, false, unfinished.as_deref()),
11995 Starter::Resume("A".into())
11996 );
11997 assert_eq!(t.runs, vec!["B", "A"]);
11998 }
11999 }
12000
12001 #[test]
12002 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
12003 let mut state = crate::land::tests::repair_state(true, true);
12004 state.status = RunStatus::Landing;
12005 state.parked = true;
12006 state.driver_exited = true;
12007 let mut q = ask::Question::new(
12008 state.id.clone(),
12009 land::APPROVAL_NODE.into(),
12010 "land".into(),
12011 "Merge?".into(),
12012 String::new(),
12013 vec![land::APPROVE.into(), land::HOLD.into()],
12014 );
12015 q.id = state.id.clone();
12016 state.land_approval = Some(crate::run::LandApproval {
12017 question: q.id.clone(),
12018 head: "current-head".into(),
12019 });
12020 assert!(land::upgrade_repair_choice(&mut q, &state));
12021 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12022 let mut task = held_task_with(&state.id);
12023 task.status = TaskStatus::Parked;
12024 assert_eq!(
12025 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12026 ActionDecision::Resume(state.id.clone())
12027 );
12028 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12029 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
12030 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
12031 assert_eq!(
12032 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12033 ActionDecision::Resume(state.id.clone()),
12034 "a repair answer newer than the manual hold resumes the existing owner"
12035 );
12036 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12037 for answered_at in [q.answered_at, None] {
12038 let mut older = q.clone();
12039 older.answered_at = answered_at;
12040 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
12041 panic!("a later manual hold must prevent dispatch")
12042 });
12043 let ActionDecision::Refuse(reason) = refused else {
12044 panic!("expected an actionable refusal, got {refused:?}");
12045 };
12046 assert!(reason.contains("later operator decision"));
12047 assert!(reason.contains(&format!("magi task release {}", task.id)));
12048 }
12049 task.operator_decided_at = None;
12050 let mut superseded = state.clone();
12051 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
12052 assert_eq!(
12053 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
12054 ActionDecision::Stale
12055 );
12056 for case in 1..4 {
12057 let mut changed = state.clone();
12058 match case {
12059 1 => changed.status = RunStatus::Blocked,
12060 2 => changed.released_to = Some("new-owner".into()),
12061 _ => changed.land_approval.as_mut().unwrap().head.clear(),
12062 }
12063 assert!(matches!(
12064 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
12065 ActionDecision::Refuse(_)
12066 ));
12067 }
12068 assert!(matches!(
12069 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
12070 ActionDecision::Refuse(_)
12071 ));
12072 task.runs.push("replacement".into());
12073 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12074 assert_eq!(
12075 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
12076 "old owner must not load"
12077 )),
12078 ActionDecision::Stale
12079 );
12080 }
12081
12082 #[test]
12083 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
12084 let mut state = crate::land::tests::repair_state(true, true);
12085 state.status = RunStatus::Landing;
12086 state.parked = true;
12087 state.driver_exited = true;
12088 let mut q = ask::Question::new(
12089 state.id.clone(),
12090 land::APPROVAL_NODE.into(),
12091 "land".into(),
12092 "Merge?".into(),
12093 String::new(),
12094 vec![land::APPROVE.into(), land::HOLD.into()],
12095 );
12096 q.id = state.id.clone();
12097 state.land_approval = Some(crate::run::LandApproval {
12098 question: q.id.clone(),
12099 head: "current-head".into(),
12100 });
12101 state.save().unwrap();
12102 land::upgrade_repair_choice(&mut q, &state);
12103 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12104 Questions::open().put(&mut q).unwrap();
12105 let tmp = tempfile::tempdir().unwrap();
12106 let queue = Queue::at(tmp.path().join("queue"));
12107 let mut task = held_task_with(&state.id);
12108 task.status = TaskStatus::Parked;
12109 queue.put(&mut task).unwrap();
12110 let empty = Arc::new(tokio::sync::Semaphore::new(0));
12111 let free = Arc::new(tokio::sync::Semaphore::new(1));
12112 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
12113 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
12114 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
12115 assert!(claim.permit.is_some());
12116 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
12117 drop(claim);
12118 Questions::open()
12119 .update(&q.id, |q| {
12120 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
12121 Ok(())
12122 })
12123 .unwrap();
12124 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
12125 assert!(claim.permit.is_none());
12126 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
12127 }
12128
12129 #[test]
12130 fn decide_action_resumes_only_the_latest_resumable_run() {
12131 let t = held_task_with("r1");
12132 let q = action_question("r1", resume_action("r1"));
12133 let load = |s: RunState| move |_: &str| Ok(s);
12134 assert_eq!(
12135 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
12136 ActionDecision::Resume("r1".into())
12137 );
12138 let q_other = action_question("r1", resume_action("r0"));
12140 assert!(matches!(
12141 decide_action(
12142 &t,
12143 &q_other,
12144 &PHRASES_EN,
12145 load(run_state(RunStatus::Blocked))
12146 ),
12147 ActionDecision::Refuse(_)
12148 ));
12149 assert!(matches!(
12151 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
12152 ActionDecision::Refuse(_)
12153 ));
12154 let mut released = run_state(RunStatus::Blocked);
12156 released.released_to = Some("elsewhere".into());
12157 assert!(matches!(
12158 decide_action(&t, &q, &PHRASES_EN, load(released)),
12159 ActionDecision::Refuse(_)
12160 ));
12161 assert!(matches!(
12163 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
12164 ActionDecision::Refuse(_)
12165 ));
12166 }
12167
12168 #[test]
12169 fn decide_action_ignores_a_question_about_an_earlier_run() {
12170 let mut t = held_task_with("r1");
12171 t.runs.push("r2".to_owned());
12172 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12173 assert_eq!(
12174 decide_action(
12175 &t,
12176 &action_question("r1", ask::ChoiceAction::Done),
12177 &PHRASES_EN,
12178 never
12179 ),
12180 ActionDecision::Stale
12181 );
12182 }
12183
12184 #[test]
12185 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
12186 let mut t = held_task_with("r1");
12187 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12188 assert_eq!(
12189 decide_action(
12190 &t,
12191 &action_question("r1", ask::ChoiceAction::Requeue),
12192 &PHRASES_EN,
12193 never
12194 ),
12195 ActionDecision::Requeue
12196 );
12197 let done_q = action_question("r1", ask::ChoiceAction::Done);
12198 assert_eq!(
12199 decide_action(&t, &done_q, &PHRASES_EN, never),
12200 ActionDecision::Done
12201 );
12202 t.mark_action_applied(&done_q.id);
12203 assert_eq!(
12204 decide_action(&t, &done_q, &PHRASES_EN, never),
12205 ActionDecision::Skip
12206 );
12207
12208 let mut plain = action_question("r1", ask::ChoiceAction::Done);
12210 plain.actions.clear();
12211 assert_eq!(
12212 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
12213 ActionDecision::Skip
12214 );
12215 let mut running = held_task_with("r1");
12217 running.status = TaskStatus::Running;
12218 assert_eq!(
12219 decide_action(
12220 &running,
12221 &action_question("r1", ask::ChoiceAction::Done),
12222 &PHRASES_EN,
12223 never
12224 ),
12225 ActionDecision::Skip
12226 );
12227 }
12228
12229 #[test]
12230 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12231 crate::run::pin_test_home();
12232 for legacy in [false, true] {
12233 let dir = tempfile::tempdir().unwrap();
12234 let queue = Queue::at(dir.path().join("queue"));
12235 let questions = Questions::at(dir.path().join("questions"));
12236 let run = "20260101-000000-hold";
12237 let mut task = held_task_with(run);
12238 task.status = TaskStatus::Parked;
12239 let mut q = ask::Question::new(
12240 run.into(),
12241 land::APPROVAL_NODE.into(),
12242 "land".into(),
12243 "Repair?".into(),
12244 String::new(),
12245 vec![land::REPAIR.into()],
12246 );
12247 q.actions.insert(land::REPAIR.into(), resume_action(run));
12248 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12249 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12251 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12252 questions.put(&mut q).unwrap();
12253 task.hold_manual(Some("Keep this PR paused".into()));
12254 if legacy {
12255 task.hold_source = None;
12256 }
12257 queue.put(&mut task).unwrap();
12258
12259 apply_task_choice_actions(&queue, &questions, dir.path());
12260 let after = queue.get(&task.id).unwrap();
12261 assert!(after.operator_held());
12262 assert_eq!(after.hold_source, task.hold_source);
12263 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12264 assert_eq!(after.runs, task.runs);
12265 assert_eq!(after.instruction, task.instruction);
12266 assert!(after.resume_override.is_none());
12267 assert!(after.action_applied(&q.id));
12268 let reason = after.hold_reason.as_deref().unwrap();
12269 assert!(reason.starts_with("Keep this PR paused"));
12270 assert!(reason.contains("later operator decision"));
12271 assert!(reason.contains(&format!("magi task release {}", task.id)));
12272
12273 for recovery in [
12274 crate::conduct::Recovery::Review,
12275 crate::conduct::Recovery::Requeue,
12276 ] {
12277 crate::conduct::apply(
12278 &queue,
12279 &questions,
12280 &crate::conduct::Verdict {
12281 decisions: vec![crate::conduct::Decision {
12282 id: task.id.clone(),
12283 recovery: Some(recovery),
12284 ..Default::default()
12285 }],
12286 },
12287 )
12288 .unwrap();
12289 }
12290 apply_task_choice_actions(&queue, &questions, dir.path());
12293 let still_held = queue.get(&task.id).unwrap();
12294 assert!(still_held.operator_held());
12295 assert_eq!(still_held.hold_reason, after.hold_reason);
12296 assert_eq!(still_held.runs, task.runs);
12297 assert!(still_held.review_branch.is_none());
12298 assert!(!still_held.fresh_start);
12299 assert!(queue.next_runnable().is_none());
12300 }
12301 }
12302
12303 #[test]
12304 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
12305 for held in [false, true] {
12306 for retry in [false, true] {
12307 let dir = tempfile::tempdir().unwrap();
12308 let queue = Queue::at(dir.path().join("queue"));
12309 let questions = Questions::at(dir.path().join("questions"));
12310 let home = dir.path().join("home");
12311 let mut t = task();
12312 if held {
12313 t.hold_machine(Some("driver state unknown".into()));
12314 }
12315 queue.put(&mut t).unwrap();
12316 let verdict = serde_json::from_value(serde_json::json!({
12317 "decisions": [{
12318 "id": t.id,
12319 "question": "Driver state unknown; retry the checks or hold?",
12320 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
12321 "actions": {
12322 "daemon: recheck and retry": {"do": "requeue"},
12323 "daemon: hold the task": {"do": "hold"}
12324 }
12325 }]
12326 }))
12327 .unwrap();
12328 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
12329 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12330 let mut q = questions.list().pop().unwrap();
12331 let choice = if retry {
12332 "daemon: recheck and retry"
12333 } else {
12334 "daemon: hold the task"
12335 };
12336 assert_eq!(q.actions.len(), 2);
12337 q.answer(ask::Answer::Choice(choice.into())).unwrap();
12338 questions.put(&mut q).unwrap();
12339
12340 questions.beat(&q.id, ask::WaiterKind::Deputy);
12342 for _ in 0..2 {
12343 resolve_blockers(&queue, &questions);
12344 apply_task_choice_actions(&queue, &questions, &home);
12345 let waiting = queue.get(&t.id).unwrap();
12346 assert_eq!(waiting.status, TaskStatus::Blocked);
12347 assert_eq!(waiting.blocked_by, [q.id.clone()]);
12348 assert!(waiting.answers.is_empty());
12349 assert!(!waiting.action_applied(&q.id));
12350 assert!(runnable(&queue).is_empty());
12351 let free = Arc::new(tokio::sync::Semaphore::new(1));
12352 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
12353 }
12354 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12355
12356 resolve_blockers(&queue, &questions);
12358 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12359 apply_task_choice_actions(&queue, &questions, &home);
12360 let after = queue.get(&t.id).unwrap();
12361 assert_eq!(
12362 after.status,
12363 if retry {
12364 TaskStatus::Queued
12365 } else {
12366 TaskStatus::Held
12367 }
12368 );
12369 assert!(after.action_applied(&q.id));
12370 assert_eq!(after.operator_held(), !retry);
12371 assert_eq!(after.answers.len(), 1);
12372
12373 let mut changed = after;
12375 if retry {
12376 changed.hold_manual(Some("later hold".into()));
12377 } else {
12378 changed.release_operator("test", None);
12379 }
12380 let expected = changed.status;
12381 queue.put(&mut changed).unwrap();
12382 apply_task_choice_actions(&queue, &questions, &home);
12383 assert_eq!(queue.get(&t.id).unwrap().status, expected);
12384 }
12385 }
12386 }
12387
12388 #[test]
12389 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
12390 let dir = tempfile::tempdir().unwrap();
12391 let queue = Queue::at(dir.path().join("queue"));
12392 let questions = Questions::at(dir.path().join("questions"));
12393 let home = dir.path().join("home");
12394 let mut state = run_state(RunStatus::Blocked);
12395 state.id = "20260101-000000-act1".to_owned();
12396 state.save_under(&home).unwrap();
12397
12398 let mut t = held_task_with(&state.id);
12399 queue.put(&mut t).unwrap();
12400 let mut q = action_question(&state.id, resume_action(&state.id));
12401 questions.put(&mut q).unwrap();
12402
12403 apply_task_choice_actions(&queue, &questions, &home);
12404 let after = queue.get(&t.id).unwrap();
12405 assert_eq!(after.status, TaskStatus::Queued);
12406 assert!(!after.fresh_start);
12407 assert!(after.action_applied(&q.id));
12408 let pin = after.resume_override.clone().unwrap();
12409 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
12410 assert!(pin.forced);
12411
12412 let mut again = queue.get(&t.id).unwrap();
12414 again.hold_machine(Some("later".into()));
12415 queue.put(&mut again).unwrap();
12416 apply_task_choice_actions(&queue, &questions, &home);
12417 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12418 }
12419
12420 #[test]
12421 fn a_delivered_answer_is_still_acted_on_exactly_once() {
12422 let dir = tempfile::tempdir().unwrap();
12423 let queue = Queue::at(dir.path().join("queue"));
12424 let questions = Questions::at(dir.path().join("questions"));
12425 let home = dir.path().join("home");
12426 let mut state = run_state(RunStatus::Blocked);
12427 state.id = "20260101-000000-act2".to_owned();
12428 state.save_under(&home).unwrap();
12429
12430 let mut t = held_task_with(&state.id);
12431 queue.put(&mut t).unwrap();
12432 let mut q = action_question(&state.id, resume_action(&state.id));
12433 q.answer_delivered = true;
12434 questions.put(&mut q).unwrap();
12435
12436 apply_task_choice_actions(&queue, &questions, &home);
12439 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12440 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
12441
12442 let mut again = queue.get(&t.id).unwrap();
12444 again.hold_machine(Some("later".into()));
12445 queue.put(&mut again).unwrap();
12446 apply_task_choice_actions(&queue, &questions, &home);
12447 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12448 }
12449
12450 #[test]
12451 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
12452 let dir = tempfile::tempdir().unwrap();
12453 let queue = Queue::at(dir.path().join("queue"));
12454 let questions = Questions::at(dir.path().join("questions"));
12455 let home = dir.path().join("home");
12456 let mut state = run_state(RunStatus::Blocked);
12457 state.id = "20260101-000000-act3".to_owned();
12458 state.save_under(&home).unwrap();
12459 let mut t = held_task_with(&state.id);
12460 queue.put(&mut t).unwrap();
12461 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
12462 questions.put(&mut q).unwrap();
12463
12464 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
12465 apply_task_choice_actions(&queue, &questions, &home);
12466 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12467
12468 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12469 apply_task_choice_actions(&queue, &questions, &home);
12470 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12471 }
12472
12473 #[test]
12474 fn action_standing_tells_the_reasons_apart() {
12475 let mut t = held_task_with("r1");
12476 let q = action_question("r1", ask::ChoiceAction::Requeue);
12477 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12478 let mut plain = q.clone();
12479 plain.actions.clear();
12480 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
12481
12482 t.mark_action_applied(&q.id);
12484 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
12485
12486 let mut t = held_task_with("r1");
12488 t.start("r2".to_owned());
12489 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
12490 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
12492 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12493 t.status = TaskStatus::Blocked;
12494 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12495
12496 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
12498 c.node = crate::conduct::NODE.to_owned();
12499 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
12500 assert!(!ActionStanding::Busy.daemon_owns());
12501 }
12502
12503 #[test]
12504 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
12505 let dir = tempfile::tempdir().unwrap();
12506 let queue = Queue::at(dir.path().join("queue"));
12507 let questions = Questions::at(dir.path().join("questions"));
12508 let home = dir.path().join("home");
12509 let mut state = run_state(RunStatus::Blocked);
12510 state.id = "20260101-000000-act4".to_owned();
12511 state.save_under(&home).unwrap();
12512 let mut t = held_task_with(&state.id);
12513 t.status = TaskStatus::Running;
12514 queue.put(&mut t).unwrap();
12515 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
12516 q.cwd = Some(dir.path().display().to_string());
12517 questions.put(&mut q).unwrap();
12518
12519 let running = queue.get(&t.id).unwrap();
12522 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
12523 assert!(matches!(
12524 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
12525 crate::waiter::Action::Deliver(_)
12526 ));
12527 apply_task_choice_actions(&queue, &questions, &home);
12529 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
12530 let mut back = queue.get(&t.id).unwrap();
12531 back.hold_machine(Some("later".into()));
12532 queue.put(&mut back).unwrap();
12533 apply_task_choice_actions(&queue, &questions, &home);
12534 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
12535 let mut again = queue.get(&t.id).unwrap();
12536 again.hold_machine(Some("again".into()));
12537 queue.put(&mut again).unwrap();
12538 apply_task_choice_actions(&queue, &questions, &home);
12539 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12540 }
12541}