1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.execution_runs().last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id.clone(),
519 run: task.execution_runs().last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if task.stale_question(q) {
887 ActionStanding::Stale
888 } else if q.chosen_action().is_none() {
889 ActionStanding::NoAction
890 } else if task.action_applied(&q.id) {
891 ActionStanding::Applied
892 } else if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
893 ActionStanding::Stale
894 } else if matches!(
895 task.status,
896 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
897 ) {
898 ActionStanding::Busy
899 } else {
900 ActionStanding::Pending
901 }
902}
903
904#[derive(Debug, Clone, PartialEq, Eq)]
906enum ActionDecision {
907 Skip,
909 Resume(String),
911 Requeue,
913 Hold,
915 Done,
917 Stale,
920 Refuse(String),
923}
924
925fn decide_action<F>(
934 home: &Path,
935 task: &Task,
936 q: &ask::Question,
937 p: &Phrases,
938 load: F,
939) -> ActionDecision
940where
941 F: FnOnce(&str) -> Result<RunState>,
942{
943 if task.stale_question(q) {
944 return ActionDecision::Stale;
945 }
946 let Some(action) = q.chosen_action() else {
947 return ActionDecision::Skip;
948 };
949 if task.action_applied(&q.id)
950 || matches!(
951 task.status,
952 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
953 )
954 {
955 return ActionDecision::Skip;
956 }
957 if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
960 return ActionDecision::Stale;
961 }
962 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
963 let stopping = q.node == land::APPROVAL_NODE
964 && q.resolution()
965 .as_deref()
966 .is_some_and(crate::merge_decision::stops);
967 let decision_at = q.answered_at.unwrap_or(q.asked_at);
971 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
972 return if repair || stopping {
973 ActionDecision::Refuse(format!(
974 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
975 if repair { "repair" } else { "stop" },
976 q.id,
977 q.run,
978 task.id
979 ))
980 } else {
981 ActionDecision::Stale
982 };
983 }
984 match action {
985 a if a.maintenance() => ActionDecision::Skip,
986 ask::ChoiceAction::RecoverBase { .. }
987 | ask::ChoiceAction::ResumeRun { .. }
988 | ask::ChoiceAction::ReleaseTask { .. }
989 | ask::ChoiceAction::RerunChecks { .. }
990 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
991 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
992 ask::ChoiceAction::Hold => ActionDecision::Hold,
993 ask::ChoiceAction::Done => ActionDecision::Done,
994 ask::ChoiceAction::Resume { run } => {
995 if task.execution_runs().last() != Some(run) {
996 return ActionDecision::Refuse((p.resume_not_latest)(
997 q.short(),
998 ask::short_id(run),
999 ));
1000 }
1001 match load(run) {
1002 Ok(s) if stopping => {
1003 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
1004 || !s
1005 .land_approval
1006 .as_ref()
1007 .is_some_and(|binding| binding.question == q.id)
1008 {
1009 ActionDecision::Stale
1010 } else if s.liveness_under(home, false) == Liveness::Live {
1011 ActionDecision::Skip
1012 } else if s.status == RunStatus::Landing
1013 && !s.released()
1014 && s.retired_to.is_none()
1015 {
1016 ActionDecision::Resume(run.clone())
1017 } else {
1018 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1019 }
1020 }
1021 Ok(s)
1022 if q.node == land::APPROVAL_NODE
1023 && q.resolution().as_deref() == Some(land::REPAIR) =>
1024 {
1025 if s.merge_repairs.iter().any(|r| r.question == q.id)
1026 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1027 {
1028 ActionDecision::Stale
1031 } else if s.liveness_under(home, false) == Liveness::Live {
1032 ActionDecision::Skip
1033 } else if s.status != RunStatus::Landing
1034 || s.released()
1035 || s.reviews.is_empty()
1036 || !implementers_match(task, &s)
1037 || !s
1038 .land_approval
1039 .as_ref()
1040 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1041 {
1042 ActionDecision::Refuse(format!(
1043 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1044 q.id, run, run
1045 ))
1046 } else {
1047 ActionDecision::Resume(run.clone())
1048 }
1049 }
1050 Ok(s) if s.liveness_under(home, false) == Liveness::Live => {
1051 ActionDecision::Refuse("run already has a live driver".into())
1052 }
1053 Ok(s) => match crate::handover::resume_refusal_under(&s, Some(home)) {
1054 None => ActionDecision::Resume(run.clone()),
1055 Some(reason) => ActionDecision::Refuse((p.resume_cannot_progress)(
1056 q.short(),
1057 ask::short_id(run),
1058 &reason.diagnostic(&s),
1059 )),
1060 },
1061 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1062 q.short(),
1063 ask::short_id(run),
1064 &format!("{e:#}"),
1065 )),
1066 }
1067 }
1068 }
1069}
1070
1071struct Phrases {
1085 graph_stopped: fn(&str, &str) -> String,
1087 quorum_lost: &'static str,
1088 quota_took_out: &'static str,
1090 run_ended: &'static str,
1092 waiting_for_answer: &'static str,
1094 recovered_running: &'static str,
1095 no_run_to_recover: &'static str,
1096 could_not_start: &'static str,
1097 resume_not_latest: fn(&str, &str) -> String,
1098 resume_cannot_progress: fn(&str, &str, &str) -> String,
1099 resume_unreadable: fn(&str, &str, &str) -> String,
1100 handover_refused: fn(&crate::handover::Refused) -> String,
1103 handover_hint: &'static str,
1105 reviewers_never_answered: fn(usize, usize) -> String,
1108}
1109
1110const PHRASES_EN: Phrases = Phrases {
1111 graph_stopped: |status, detail| {
1112 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1113 },
1114 quorum_lost: "the judging panel lost its quorum",
1115 quota_took_out: "; quota took out ",
1116 run_ended: "run ended ",
1117 waiting_for_answer: " - waiting for operator answer to question ",
1118 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1119 no_run_to_recover: "task was `running` with no live daemon and no readable \
1120 run to recover; held for a human to check what happened",
1121 could_not_start: "could not start the run: ",
1122 resume_not_latest: |q, run| {
1123 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1124 },
1125 resume_cannot_progress: |q, run, reason| {
1126 format!("question {q} asked to resume run {run}, which cannot make progress: {reason}")
1127 },
1128 resume_unreadable: |q, run, e| {
1129 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1130 },
1131 handover_refused: |r| r.to_string(),
1132 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1133 reviewers_never_answered: |missing, rounds| {
1134 format!(
1135 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1136 refusing to call it clean"
1137 )
1138 },
1139};
1140
1141const PHRASES_JA: Phrases = Phrases {
1142 graph_stopped: |status, detail| {
1143 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1144 },
1145 quorum_lost: "審査パネルが定足数を失いました",
1146 quota_took_out: "。クォータで脱落: ",
1147 run_ended: "run 終了: ",
1148 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1149 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1150 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1151 何が起きたか人が確認するため保留にしました",
1152 could_not_start: "run を開始できませんでした: ",
1153 resume_not_latest: |q, run| {
1154 format!(
1155 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1156 )
1157 },
1158 resume_cannot_progress: |q, run, reason| {
1159 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません: {reason}")
1160 },
1161 resume_unreadable: |q, run, e| {
1162 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1163 },
1164 handover_refused: |r| {
1165 use crate::handover::Refused;
1166 match r {
1167 Refused::Foreign { branch, path, why } => format!(
1168 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1169 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1170 から、やり直してください(詳細: {why})"
1171 ),
1172 Refused::Unsafe { branch, path, why } => format!(
1173 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1174 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1175 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1176 ),
1177 Refused::ReleaseFailed { branch, path, run } => format!(
1178 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1179 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1180 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1181 ),
1182 }
1183 },
1184 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1185 reviewers_never_answered: |missing, rounds| {
1186 format!(
1187 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1188 クリーンとは認めません"
1189 )
1190 },
1191};
1192
1193fn phrases(language: &str) -> &'static Phrases {
1194 if crate::lang::is_japanese(language) {
1195 &PHRASES_JA
1196 } else {
1197 &PHRASES_EN
1198 }
1199}
1200
1201fn language_of(task: &Task, fallback: &Path) -> String {
1205 crate::lang::of_repo(&repo_for(task, fallback))
1206}
1207
1208pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1212 if q.node == crate::conduct::NODE {
1213 return tasks.iter().find(|t| t.id == q.run);
1214 }
1215 tasks.iter().find(|t| t.runs.contains(&q.run))
1216}
1217
1218fn repair_choice_offers(questions: &Questions) {
1221 for q in questions.list().into_iter().filter(|q| {
1222 q.status.open()
1223 && q.node == crate::conduct::NODE
1224 && q.choices
1225 .iter()
1226 .any(|c| crate::choice_effect::describe(q, c).is_none())
1227 }) {
1228 let _ = questions.update(&q.id, |current| {
1229 if current.status.open() {
1230 crate::choice_effect::executable_conductor(current);
1231 if let Some(deputy) = &mut current.deputy {
1232 deputy.brief = crate::deputy::brief(
1233 ¤t.run,
1234 ¤t.detail,
1235 ¤t.choices,
1236 ¤t.actions,
1237 );
1238 }
1239 }
1240 Ok(())
1241 });
1242 }
1243}
1244
1245pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1248 apply_maintenance_choices(queue, questions, home).await;
1249 apply_task_choice_actions(queue, questions, home);
1250}
1251
1252async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1255 for q in questions.list() {
1256 let Ok(_execution) = crate::task_inquiry::Outbox::at(home).execution_claim(&q.id) else {
1258 continue;
1259 };
1260 let Ok(q) = questions.get(&q.id) else {
1261 continue;
1262 };
1263 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1264 continue;
1265 };
1266 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1269 continue;
1270 }
1271 if queue.list().iter().any(|task| {
1272 (task.id == q.run
1273 || task.runs.contains(&q.run)
1274 || q.routed_task.as_deref() == Some(&task.id))
1275 && task.stale_question(&q)
1276 }) {
1277 continue;
1278 }
1279 if let ask::ChoiceAction::RecoverBase { run, stop, episode } = action {
1280 let tasks = queue.list();
1281 let owner = crate::task_chat::task_of(&q, &tasks);
1282 if let Some(owner) = owner
1283 && q.routed_task
1284 .as_deref()
1285 .is_none_or(|routed| routed == owner)
1286 {
1287 let result = crate::base_recovery_control::recover(
1288 queue,
1289 home,
1290 crate::base_recovery_control::Request {
1291 run: run.clone(),
1292 head: stop.head.clone(),
1293 actor: "decisions".into(),
1294 request: q.id.clone(),
1295 decision: Some((owner, q.answered_at.unwrap_or(q.asked_at))),
1296 expected_stop: Some((stop.clone(), *episode)),
1297 },
1298 )
1299 .await;
1300 if let Err(error) = result {
1301 tracing::warn!(question = %q.id, %error, "base recovery decision refused");
1302 notices::raise_in(home, Notice::warn(
1303 &format!("base-recovery:{}", q.id),
1304 format!("Base recovery decision {} was refused: {error:#}. Inspect the existing run and request; no replacement run was started.", q.id),
1305 ).link(Link::Run { id: run.clone() }));
1306 }
1307 }
1308 continue;
1309 }
1310 let dir = home.join("choice-actions");
1311 if std::fs::create_dir_all(&dir).is_err() {
1312 continue;
1313 }
1314 let Ok(lock) = std::fs::OpenOptions::new()
1315 .create(true)
1316 .truncate(false)
1317 .write(true)
1318 .open(dir.join(format!("{}.lock", q.id)))
1319 else {
1320 continue;
1321 };
1322 if crate::pr_head::try_lock_file(&lock).is_err() {
1323 continue;
1324 }
1325 let fresh = !home
1326 .join("choice-actions")
1327 .join(format!("{}.json", q.id))
1328 .exists();
1329 let result = async {
1330 let receipt = dir.join(format!("{}.json", q.id));
1331 if receipt.exists() {
1332 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1333 anyhow::ensure!(
1334 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1335 "previous maintenance attempt did not confirm completion; inspect its receipt"
1336 );
1337 return Ok::<_, anyhow::Error>(());
1338 }
1339 let record = |status: &str| -> anyhow::Result<()> {
1342 let tmp = receipt.with_extension("tmp");
1343 std::fs::write(
1344 &tmp,
1345 serde_json::to_vec(&serde_json::json!({
1346 "action": action, "status": status,
1347 }))?,
1348 )?;
1349 std::fs::rename(tmp, &receipt)?;
1350 Ok(())
1351 };
1352 let applied = async {
1353 match action {
1354 ask::ChoiceAction::ResumeRun { run } => {
1355 crate::resume::validate(home, run)?;
1356 record("started")?;
1357 let decided = q.answered_at.unwrap_or(q.asked_at);
1358 if !crate::resume::pin_queued(queue, run, Some(decided), Some(&q.id))? {
1359 crate::resume::validate_unqueued(home, run)?;
1360 crate::resume::start(home, run.clone()).await?;
1361 }
1362 }
1363 ask::ChoiceAction::ReleaseTask {
1364 task,
1365 held_at,
1366 reason,
1367 } => {
1368 anyhow::ensure!(
1369 queue.resolve_id(task)? == *task,
1370 "task action requires an exact task ID"
1371 );
1372 let _claim = queue.claim(task)?;
1373 let mut task = queue.get(task)?;
1374 if task.action_applied(&q.id) {
1375 return Ok(());
1376 }
1377 if q.node == crate::conduct::NODE
1382 && q.run == task.id
1383 && task.status == TaskStatus::Blocked
1384 && task.blocked_from == Some(TaskStatus::Held)
1385 && task.blocked_by.as_slice() == [q.id.as_str()]
1386 {
1387 task.unblock(&q.id);
1388 }
1389 anyhow::ensure!(
1390 task.status == TaskStatus::Held
1391 && task.held_at == Some(*held_at)
1392 && task.hold_reason.as_deref() == reason.as_deref()
1393 && task
1394 .operator_decided_at
1395 .is_none_or(|at| at <= q.answered_at.unwrap_or(q.asked_at)),
1396 "task no longer has the hold this question names"
1397 );
1398 task.release_operator("choice", Some(&q.id));
1399 task.mark_action_applied(&q.id);
1400 queue.put(&mut task)?;
1401 }
1402 ask::ChoiceAction::AbandonRun { run } => {
1403 record("started")?;
1404 crate::driver::abandon_proven_dead(home, run)?;
1405 }
1406 ask::ChoiceAction::RerunChecks {
1407 repo,
1408 pr,
1409 head,
1410 runs,
1411 } => {
1412 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1413 .await?;
1414 }
1415 _ => unreachable!(),
1416 }
1417 Ok::<_, anyhow::Error>(())
1418 }
1419 .await;
1420 match applied {
1421 Ok(()) => record("applied"),
1422 Err(e) => {
1423 record(&format!("refused or failed: {e:#}"))?;
1424 Err(e)
1425 }
1426 }
1427 }
1428 .await;
1429 if home
1434 .join("choice-actions")
1435 .join(format!("{}.json", q.id))
1436 .exists()
1437 && let Some(listed) = task_of_question(&queue.list(), &q)
1438 && !listed.action_applied(&q.id)
1439 && let Ok(_claim) = queue.claim(&listed.id)
1440 && let Ok(mut origin) = queue.get(&listed.id)
1441 {
1442 if !origin.operator_held()
1443 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1444 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1445 {
1446 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1447 origin.hold_machine(Some(format!(
1448 "Maintenance choice {} settled; inspect its target before releasing this task",
1449 q.id
1450 )));
1451 } else {
1452 origin.unblock(&q.id);
1453 }
1454 }
1455 origin.mark_action_applied(&q.id);
1456 let _ = queue.put(&mut origin);
1457 }
1458 if let Err(e) = result
1459 && fresh
1460 {
1461 notices::raise_in(
1462 home,
1463 Notice::warn(
1464 &format!("action:{}", q.id),
1465 format!("Choice action {} was not completed: {e:#}", q.id),
1466 )
1467 .link(if q.run_names_task() {
1468 Link::Task { id: q.run.clone() }
1469 } else {
1470 Link::Run { id: q.run.clone() }
1471 }),
1472 );
1473 }
1474 }
1475}
1476
1477pub(crate) async fn rerun_checks_once(
1484 home: &Path,
1485 request: &str,
1486 repo: &Path,
1487 pr: &str,
1488 head: &str,
1489 runs: &[String],
1490 started: impl FnOnce() -> anyhow::Result<()>,
1491) -> anyhow::Result<()> {
1492 use crate::release_watch::{GhForge, ReleaseForge};
1493 anyhow::ensure!(
1494 !runs.is_empty()
1495 && runs.len() <= 20
1496 && runs
1497 .iter()
1498 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1499 && !pr.starts_with('-'),
1500 "invalid rerun targets"
1501 );
1502 let snap = GhForge.snapshot(repo, pr).await?;
1503 anyhow::ensure!(
1504 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1505 "pull request head or lifecycle changed"
1506 );
1507 anyhow::ensure!(
1508 runs.iter().all(|r| snap
1509 .checks
1510 .iter()
1511 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1512 "named workflow is no longer failed on this pull request"
1513 );
1514 started()?;
1515 let dir = home.join("choice-actions");
1516 std::fs::create_dir_all(&dir)?;
1517 let rerun_lock = std::fs::OpenOptions::new()
1518 .create(true)
1519 .truncate(false)
1520 .write(true)
1521 .open(dir.join("reruns.lock"))?;
1522 crate::pr_head::try_lock_file(&rerun_lock)?;
1523 let ledger = dir.join("reruns.json");
1524 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1525 serde_json::from_slice(&std::fs::read(&ledger)?)?
1526 } else {
1527 Default::default()
1528 };
1529 for run in runs {
1530 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1531 if !recorded.insert(key) {
1532 continue;
1533 }
1534 let tmp = ledger.with_extension("tmp");
1535 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1536 std::fs::rename(tmp, &ledger)?;
1537 GhForge.rerun(repo, run).await?;
1538 }
1539 Ok(())
1540}
1541
1542pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1551 let tasks = queue.list();
1552 for q in questions.list() {
1553 let Ok(_execution) = crate::task_inquiry::Outbox::at(home).execution_claim(&q.id) else {
1555 continue;
1556 };
1557 let Ok(q) = questions.get(&q.id) else {
1558 continue;
1559 };
1560 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1562 continue;
1563 }
1564 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1565 continue;
1566 }
1567 let Some(listed) = task_of_question(&tasks, &q) else {
1568 if q.node == land::APPROVAL_NODE
1569 && q.resolution().as_deref().is_some_and(|choice| {
1570 choice == land::REPAIR || crate::merge_decision::stops(choice)
1571 })
1572 {
1573 notices::raise_in(
1574 home,
1575 Notice::warn(
1576 &format!("repair-owner:{}", q.id),
1577 format!(
1578 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1579 q.run, q.run
1580 ),
1581 ).link(Link::Run { id: q.run.clone() }),
1582 );
1583 }
1584 continue;
1585 };
1586 if listed.action_applied(&q.id) {
1587 continue;
1588 }
1589 let Ok(_claim) = queue.claim(&listed.id) else {
1590 continue;
1591 };
1592 let Ok(mut task) = queue.get(&listed.id) else {
1593 continue;
1594 };
1595 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1598 let blockers: Vec<_> = task
1599 .blocked_by
1600 .iter()
1601 .filter(|id| {
1602 crate::queue::dependency(queue, questions, id)
1603 == crate::queue::Dependency::Question(q.id.clone())
1604 })
1605 .cloned()
1606 .collect();
1607 if !blockers.is_empty() {
1608 for id in blockers {
1609 task.unblock(&id);
1610 }
1611 let answer = match &q.answer {
1612 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1613 None => String::new(),
1614 };
1615 task.record_answer(q.summary.clone(), answer);
1616 }
1617 }
1618 let language = language_of(&task, Path::new("."));
1619 let decision = decide_action(home, &task, &q, phrases(&language), |id| {
1620 RunState::load_under(id, home)
1621 });
1622 let ran = matches!(
1623 decision,
1624 ActionDecision::Resume(_)
1625 | ActionDecision::Requeue
1626 | ActionDecision::Hold
1627 | ActionDecision::Done
1628 );
1629 if ran && asker_may_still_read(questions, &q, home) {
1630 continue;
1639 }
1640 let done = matches!(decision, ActionDecision::Done);
1641 let completed = matches!(
1644 decision,
1645 ActionDecision::Requeue | ActionDecision::Hold | ActionDecision::Done
1646 );
1647 match decision {
1648 ActionDecision::Skip => continue,
1649 ActionDecision::Resume(run) => {
1650 task.release_operator("choice", Some(&q.id));
1651 task.resume_override = Some(crate::queue::OperatorResume {
1652 question_id: q.id.clone(),
1653 at: Timestamp::now(),
1654 conductor_rehold: None,
1655 forced: true,
1656 pinned_run: Some(run),
1657 });
1658 }
1659 ActionDecision::Stale => {}
1660 ActionDecision::Requeue => {
1661 task.release_operator("choice-requeue", Some(&q.id));
1662 task.requeue();
1663 }
1664 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1665 ActionDecision::Done => task.succeed(),
1666 ActionDecision::Refuse(why) => {
1667 let _ =
1668 crate::task_inquiry::Outbox::at(home).update_decision(&q.id, |d| d.fail(&why));
1669 if task.operator_held() {
1670 task.hold_reason = Some(match task.hold_reason.take() {
1673 Some(reason) => format!("{reason}\n\n{why}"),
1674 None => why,
1675 });
1676 } else {
1677 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1678 task.hold_machine(Some(why));
1679 }
1680 notices::raise(
1681 Notice::warn(
1682 &format!("action:{}", q.id),
1683 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1684 )
1685 .link(Link::Task {
1686 id: task.id.clone(),
1687 }),
1688 );
1689 }
1690 }
1691 task.mark_action_applied(&q.id);
1692 if completed {
1693 task.decision_effects.insert(
1694 q.id.clone(),
1695 format!("decision: {}", q.resolution().expect("answered action")),
1696 );
1697 }
1698 record(queue, &mut task);
1701 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1702 supersede_prior_runs(&task, home);
1703 }
1704 let _ = questions.update(&q.id, |r| {
1707 r.answer_delivered = true;
1708 Ok(())
1709 });
1710 }
1711}
1712
1713fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1717 let now = Timestamp::now();
1718 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1719 return true;
1720 }
1721 q.node != crate::conduct::NODE
1722 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1723 s.seats_active()
1724 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1725 })
1726}
1727
1728fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1739 let tasks = queue.list();
1740 let by_id: std::collections::BTreeMap<&str, &Task> =
1741 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1742 let referenced: Vec<&str> = tasks
1743 .iter()
1744 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1745 .collect();
1746
1747 for mut question in questions.list() {
1748 if !question.status.open() || question.node != crate::conduct::NODE {
1749 continue;
1750 }
1751 if referenced
1756 .iter()
1757 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1758 {
1759 continue;
1760 }
1761 let Some(task) = by_id.get(question.run.as_str()) else {
1762 continue;
1763 };
1764 question.abandon(format!(
1765 "task {} no longer waits for this answer",
1766 task.short()
1767 ));
1768 if let Err(e) = questions.put(&mut question) {
1769 tracing::warn!(
1770 "could not retire question {} for task {}: {e:#}",
1771 question.short(),
1772 task.short()
1773 );
1774 }
1775 }
1776}
1777
1778#[derive(Debug, Clone, Copy)]
1785pub struct Verdict {
1786 pub status: RunStatus,
1788 pub left_pr: bool,
1790 pub quota_hit: bool,
1792 pub parked: bool,
1794 pub no_viable_candidates: bool,
1801}
1802
1803pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1862 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1863}
1864
1865fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1867 if verdict.parked {
1873 task.park(detail);
1874 return;
1875 }
1876 match verdict.status {
1877 RunStatus::Merged | RunStatus::Ready
1878 if task.pr_recovery.as_ref().is_some_and(|s| {
1879 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1880 }) =>
1881 {
1882 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1883 }
1884 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1885 RunStatus::AlreadyInBase => task.already_landed(detail),
1886 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1887 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1888 task.stall(detail)
1889 }
1890 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1891 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1892 RunStatus::Blocked => task.fail(detail, max_attempts),
1893 RunStatus::VerifiedNoop => task.handed_off(detail),
1894 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1895 }
1896}
1897
1898pub fn supersede_prior_runs(task: &Task, home: &Path) {
1947 let now = Timestamp::now();
1948 let last_run_succeeded = task
1955 .runs
1956 .last()
1957 .and_then(|id| RunState::load_under(id, home).ok())
1958 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1959 for id in task.superseded_attempts(last_run_succeeded) {
1960 let mut state = match RunState::load_under(id, home) {
1961 Ok(s) => s,
1962 Err(e) => {
1963 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1964 continue;
1965 }
1966 };
1967 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1968 continue;
1969 }
1970 let daemon_claims = is_working_on(home, id, now);
1971 if state.liveness(daemon_claims) == Liveness::Live {
1972 continue;
1973 }
1974 state.status = RunStatus::Superseded;
1975 if let Err(e) = state.save_under(home) {
1976 tracing::warn!("could not mark run {id} superseded: {e:#}");
1977 }
1978 }
1979}
1980
1981fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
2003 for task in queue.list() {
2004 if task.status != TaskStatus::Done || task.runs.len() < 2 {
2005 continue;
2006 }
2007 supersede_prior_runs(&task, home);
2008 }
2009}
2010
2011fn settle_and_diagnose(
2018 task: &mut Task,
2019 verdict: Verdict,
2020 detail: &str,
2021 max_attempts: usize,
2022 state: &RunState,
2023) {
2024 let p = phrases(&state.config.graph.language);
2025 if let Some(d) = &state.merge_decision {
2028 use crate::merge_decision as md;
2029 if task
2030 .execution_runs()
2031 .last()
2032 .is_some_and(|id| id != &state.id)
2033 {
2034 return;
2035 }
2036 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
2037 task.succeed();
2038 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
2039 if state.liveness(false) != Liveness::Dead {
2040 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
2041 return;
2042 }
2043 let replacement = md::Replacement {
2044 question: d.question.clone(),
2045 run: state.id.clone(),
2046 pr: d.target.pr.clone(),
2047 agent: agent.to_owned(),
2048 context: format!(
2049 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
2050 d.target.pr,
2051 state.id,
2052 serde_json::to_string(&state.reviews).unwrap_or_default()
2053 ),
2054 };
2055 task.requeue();
2056 task.pr_recovery = None;
2057 task.review_of = None;
2058 task.replacement = Some(replacement);
2059 } else {
2060 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
2061 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
2062 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
2063 }
2064 task.mark_action_applied(&d.question);
2065 if d.choice.eq_ignore_ascii_case(land::HOLD) {
2066 task.decision_effects
2067 .insert(d.question.clone(), format!("decision: {}", d.choice));
2068 }
2069 if let Some(pass) = state.passes.last() {
2070 task.record_pass_at(
2071 &state.id,
2072 pass.index,
2073 Some(pass.started_at),
2074 task.attempts,
2075 None,
2076 );
2077 }
2078 return;
2079 }
2080 if let Some(replacement) = &task.replacement
2081 && !state.parked
2082 && state.resource_block.is_none()
2083 && (state.viable().is_empty()
2084 || (state.status == RunStatus::Blocked && state.tally.is_none()))
2085 {
2086 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
2087 return;
2088 }
2089 if !verdict.parked && verdict.quota_hit && state.status == RunStatus::Stalled {
2090 let mut agents: Vec<_> = state
2091 .quota
2092 .iter()
2093 .filter(|q| {
2094 q.cause == crate::agent::QuotaCause::Authentication
2095 && state
2096 .passes
2097 .last()
2098 .is_none_or(|pass| q.at >= pass.started_at)
2099 })
2100 .map(|q| q.agent.as_str())
2101 .collect();
2102 agents.sort_unstable();
2103 agents.dedup();
2104 if !agents.is_empty() {
2105 task.stall(detail);
2106 task.notice_cause("agent-authentication", None, None);
2107 task.hold_machine(Some(format!("[authentication] Agents {} cannot authenticate; re-authenticate, then resume the held task. No attempt was spent.", agents.join(", "))));
2108 return;
2109 }
2110 }
2111 let attempts_before = task.attempts;
2112 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2119 task.notice_cause(
2120 &format!("resource:{}:{}", block.kind, block.node),
2121 None,
2122 None,
2123 );
2124 task.last_error = Some(detail.to_owned());
2125 task.diagnostic = None;
2126 task.attempts = task.attempts.saturating_sub(1);
2127 if block.kind == "agent authentication" {
2128 task.hold_machine(Some(format!(
2129 "[authentication] {}; no attempt was spent",
2130 block.detail
2131 )));
2132 } else if block.kind == "agent usage limit" {
2133 if let Some(agent) = &state.config.graph.selected_implementer {
2135 task.hold_machine(Some(format!("[agent usage limit] Selected implementer `{agent}` is unavailable. No other implementer was dispatched; resume after usage is available. No attempt was spent.")));
2136 } else {
2137 task.status = TaskStatus::Failed;
2138 task.last_error = Some(block.detail.clone());
2139 }
2140 } else if block.kind == "disk pressure" {
2141 task.hold_machine(Some(format!(
2142 "{}{}",
2143 crate::disk::RUNNING_PRESSURE,
2144 block.detail
2145 )));
2146 } else {
2147 task.hold_machine(Some(format!(
2148 "{}{}; no attempt was spent; resume to retry the affected operation",
2149 crate::cache::RESOURCE_HOLD,
2150 block.description()
2151 )));
2152 }
2153 if let Some(pass) = state.passes.last() {
2154 task.record_pass_at(
2155 &state.id,
2156 pass.index,
2157 Some(pass.started_at),
2158 attempts_before,
2159 None,
2160 );
2161 }
2162 return;
2163 }
2164 if let Some(hold) = state.land_unrelated.as_ref().filter(|hold| {
2165 state.parked
2166 && state.status == RunStatus::Landing
2167 && state
2168 .pr_pushed_head
2169 .as_ref()
2170 .is_none_or(|h| h == &hold.head)
2171 && task.execution_runs().last() == Some(&state.id)
2172 }) {
2173 let pass = state.passes.last().map_or(0, |p| p.index);
2176 if !task
2177 .passes
2178 .iter()
2179 .any(|p| p.run == state.id && p.pass_index == pass)
2180 {
2181 task.attempts = task.attempts.saturating_sub(1);
2182 }
2183 task.hold_machine(Some(format!(
2184 "[ci-unrelated] CI failure reproduced on base; owner question {}",
2185 hold.question
2186 )));
2187 task.last_error = Some(detail.to_owned());
2188 task.diagnostic = None;
2189 task.record_pass_at(
2190 &state.id,
2191 pass,
2192 state.passes.last().map(|p| p.started_at),
2193 attempts_before,
2194 Some(crate::run::ParkCause::Question),
2195 );
2196 return;
2197 }
2198 let unresolved = task.hold_notice_cause.clone();
2199 if state.base_sync_blocked() {
2200 task.hold_machine(Some(detail.to_owned()));
2202 task.last_error = Some(detail.to_owned());
2203 } else if !verdict.parked
2204 && task
2205 .pr_recovery
2206 .as_ref()
2207 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2208 && matches!(
2209 verdict.status,
2210 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2211 )
2212 {
2213 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2214 task.succeed();
2215 } else {
2216 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2217 }
2218 } else if !verdict.parked
2219 && task.pr_recovery.is_some()
2220 && matches!(
2221 verdict.status,
2222 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2223 )
2224 && !verdict.quota_hit
2225 {
2226 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2227 } else {
2228 settle_in(task, verdict, detail, max_attempts, p);
2229 }
2230 if !verdict.parked
2231 && !verdict.quota_hit
2232 && matches!(
2233 verdict.status,
2234 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2235 )
2236 {
2237 task.recovery_failed(state);
2238 }
2239 if state.prep_failure.is_some() {
2240 task.hold_notice_cause = unresolved;
2243 task.startup_notice_cause("startup-candidates");
2244 }
2245 if let Some(pass) = state.passes.last() {
2248 let parked = match &pass.end {
2249 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2250 _ => None,
2251 };
2252 task.record_pass_at(
2253 &state.id,
2254 pass.index,
2255 Some(pass.started_at),
2256 attempts_before,
2257 parked,
2258 );
2259 }
2260 if task.status == TaskStatus::Held {
2261 if state.base_sync_blocked() {
2262 task.base_sync_notice(state);
2263 } else if state.prep_failure.is_none() {
2264 task.notice_cause(
2265 label(verdict.status),
2266 Some(&state.id),
2267 state.passes.last().map(|p| p.index),
2268 );
2269 }
2270 task.diagnostic = diagnostic(state);
2271 note_open_question(task, &state.id, p);
2272 }
2273}
2274
2275fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2290 let Some(home) = crate::run::try_home() else {
2291 return;
2292 };
2293 let open = Questions::at(home.join("questions")).open_for(run);
2294 let Some(q) = open.first() else {
2295 return;
2296 };
2297 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2298 decisions.sort_unstable();
2299 task.notice_cause(
2300 &format!("questions:{}", decisions.join(",")),
2301 Some(run),
2302 None,
2303 );
2304 let base = task.hold_reason.clone().unwrap_or_default();
2305 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2306}
2307
2308fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2319 match last_run {
2320 Some(state) => {
2321 let verdict = Verdict {
2322 status: state.status,
2323 left_pr: state.pr.is_some(),
2324 quota_hit: state.quota.iter().any(|loss| {
2325 state
2326 .passes
2327 .last()
2328 .is_none_or(|pass| loss.at >= pass.started_at)
2329 }),
2330 parked: state.parked,
2331 no_viable_candidates: state.viable().is_empty(),
2332 };
2333 let detail = format!(
2334 "{}{}",
2335 phrases(&state.config.graph.language).recovered_running,
2336 describe(&state)
2337 );
2338 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2339 }
2340 None => {
2341 let why = phrases(language).no_run_to_recover;
2344 task.last_error = Some(why.to_owned());
2345 task.notice_cause("missing-run", None, None);
2348 task.hold_machine(Some(why.to_owned()));
2349 }
2350 }
2351}
2352
2353fn reclaim_orphaned_running(
2375 queue: &Queue,
2376 max_attempts: usize,
2377 live_workers: &std::collections::BTreeSet<String>,
2378) -> Vec<String> {
2379 let mut reclaimed = Vec::new();
2380 for listed in queue.list() {
2381 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2382 continue;
2383 }
2384 let Ok(_claim) = queue.claim(&listed.id) else {
2385 continue;
2386 };
2387 let Ok(mut task) = queue.get(&listed.id) else {
2391 continue;
2392 };
2393 if task.status != TaskStatus::Running {
2394 continue;
2395 }
2396 let last_run = task
2397 .execution_runs()
2398 .last()
2399 .and_then(|id| RunState::load(id).ok());
2400 if let Some(state) = &last_run
2410 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2411 {
2412 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2413 }
2414 let language = if last_run.is_none() {
2415 language_of(&task, Path::new("."))
2416 } else {
2417 String::new()
2418 };
2419 reclaim(&mut task, last_run, max_attempts, &language);
2420 if task.status == TaskStatus::Done {
2421 supersede_prior_runs(&task, &crate::run::home());
2422 }
2423 record(queue, &mut task);
2424 reclaimed.push(task.id.clone());
2425 }
2426 reclaimed
2427}
2428
2429fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2461 reclaim_abandoned_runs_with(
2462 home,
2463 now,
2464 crate::proc::pid_status,
2465 crate::proc::process_started_at,
2466 )
2467}
2468
2469fn reclaim_abandoned_runs_with<F, G>(
2475 home: &Path,
2476 now: Timestamp,
2477 query: F,
2478 identity: G,
2479) -> Vec<String>
2480where
2481 F: Fn(u32) -> Option<bool>,
2482 G: Fn(u32) -> Option<String>,
2483{
2484 let mut abandoned = Vec::new();
2485 for entry in std::fs::read_dir(home.join("runs"))
2486 .into_iter()
2487 .flatten()
2488 .flatten()
2489 {
2490 let id = entry.file_name().to_string_lossy().into_owned();
2491 if !crate::run::is_run_id(&id) {
2492 continue;
2493 }
2494 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2497 continue;
2498 };
2499 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2507 continue;
2508 };
2509 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2510 continue;
2511 };
2512 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2513 continue;
2514 }
2515 let daemon_claims = is_working_on(home, &id, now);
2527 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2528 != crate::run::Liveness::Dead
2529 {
2530 continue;
2531 }
2532 state.abandon("daemon");
2533 if let Err(e) = state.save_under(home) {
2534 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2535 continue;
2536 }
2537 if let Some(notice) = notices::run_ended(&state) {
2540 notices::raise_in(home, notice);
2541 }
2542 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2550 tracing::warn!("abandon questions for {id}: {e:#}");
2551 }
2552 abandoned.push(id);
2553 }
2554 abandoned
2555}
2556
2557#[derive(Debug, Clone, Serialize, Deserialize)]
2558struct TaskWorker {
2559 opts: Opts,
2560 queue: PathBuf,
2561 task: String,
2562 #[serde(default)]
2563 lane: PermitKind,
2564 #[serde(default)]
2565 standalone: bool,
2566}
2567
2568fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2569 Ok(crate::worker::active(home)?
2570 .into_iter()
2571 .filter(|job| job.kind == "task")
2572 .collect())
2573}
2574
2575fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2576 task_jobs(home)?
2577 .into_iter()
2578 .filter_map(
2579 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2580 Ok(request) if !request.standalone => Some(Ok(job)),
2581 Ok(_) => None,
2582 Err(error) => Some(Err(error.into())),
2583 },
2584 )
2585 .collect()
2586}
2587
2588#[must_use]
2590pub fn managed(home: &Path) -> bool {
2591 match crate::worker::active(home) {
2592 Err(error) => {
2593 tracing::error!("cannot determine scheduler ownership: {error:#}");
2594 true
2595 }
2596 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2597 Ok(_) => {
2598 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2599 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2600 }
2601 }
2602}
2603
2604pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2606 anyhow::ensure!(
2607 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2608 "the scheduler is stopping; wait for retirement before starting it again"
2609 );
2610 crate::worker::spawn(
2611 home,
2612 "scheduler",
2613 "scheduler",
2614 serde_json::to_value(opts.absolute()?)?,
2615 )
2616 .await
2617}
2618
2619fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2620 home.join("worker-parks").join(format!("{}.json", job.id))
2621}
2622
2623fn request_task_park(
2624 home: &Path,
2625 job: &crate::worker::Job,
2626 cause: crate::run::ParkCause,
2627) -> Result<()> {
2628 let path = park_request_path(home, job);
2629 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2630 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2631 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2632}
2633
2634pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2636 if control == crate::worker::Control::Park {
2637 for job in owned_task_jobs(home)? {
2638 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2639 }
2640 }
2641 crate::worker::request(home, "scheduler", control)
2642}
2643
2644pub async fn restart_detached(home: &Path) -> Result<()> {
2646 let active = crate::worker::active(home)?;
2647 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2648 if prior.is_none() && !managed(home) {
2649 return Ok(());
2650 }
2651 let opts: Opts = match prior {
2652 Some(job) => serde_json::from_value(job.payload)?,
2653 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2654 };
2655 request_control(home, crate::worker::Control::Stop)?;
2656 loop {
2657 let reserved = crate::worker::active(home)?
2658 .iter()
2659 .any(|job| job.kind == "scheduler");
2660 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2661 break;
2662 }
2663 tokio::time::sleep(Duration::from_millis(100)).await;
2664 }
2665 spawn_detached(home, &opts).await?;
2666 Ok(())
2667}
2668
2669pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2671 let opts: Opts = serde_json::from_value(payload)?;
2672 let _ = home;
2673 serve_until(opts, Stop::new()).await
2674}
2675
2676pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2678 let request: TaskWorker = serde_json::from_value(payload)?;
2679 let queue = Queue::at(request.queue);
2680 let key = format!("task:{}", request.task);
2681 let job = task_jobs(home)?
2682 .into_iter()
2683 .find(|job| job.key == key)
2684 .context("task worker reservation is missing")?;
2685 let park_path = park_request_path(home, &job);
2686 let _claim = loop {
2689 match queue.claim(&request.task) {
2690 Ok(claim) => break claim,
2691 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2694 return Err(e.context("claim the task for its worker"));
2695 }
2696 Err(_) => {
2697 sweep_stale_claims(&queue, STALE_CLAIM);
2698 tokio::time::sleep(Duration::from_millis(50)).await;
2699 }
2700 }
2701 };
2702 let mut task = queue.get(&request.task)?;
2703 if !task.status.runnable()
2704 || (!request.standalone
2705 && request.lane
2706 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2707 {
2708 return Ok(());
2709 }
2710 let stop = Stop::new();
2711 let watcher = {
2712 let home = home.to_path_buf();
2713 let stop = stop.clone();
2714 let park_path = park_path.clone();
2715 tokio::spawn(async move {
2716 loop {
2717 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2718 let cause = std::fs::read(&park_path)
2719 .ok()
2720 .and_then(|body| serde_json::from_slice(&body).ok())
2721 .unwrap_or(crate::run::ParkCause::Operator);
2722 stop.park(cause);
2723 return;
2724 }
2725 tokio::time::sleep(Duration::from_millis(200)).await;
2726 }
2727 })
2728 };
2729 let quota = if request.standalone {
2730 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2731 } else {
2732 let status = Arc::new(Mutex::new(Status::new()));
2733 Box::pin(attempt(
2734 &request.opts,
2735 &queue,
2736 &status,
2737 &stop,
2738 crate::graph::Pause::new(),
2739 &mut task,
2740 ))
2741 .await
2742 };
2743 watcher.abort();
2744 let _ = std::fs::remove_file(park_path);
2745 persist_cooldown(home, "a).await?;
2746 Ok(())
2747}
2748
2749pub async fn spawn_claimed(
2752 home: &Path,
2753 opts: &Opts,
2754 queue: &Queue,
2755 task: &Task,
2756 claim: crate::queue::Claim,
2757) -> Result<crate::worker::Job> {
2758 let request = TaskWorker {
2759 opts: opts.absolute()?,
2760 queue: std::path::absolute(queue.root())?,
2761 task: task.id.clone(),
2762 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2763 standalone: true,
2764 };
2765 let job = crate::worker::spawn(
2766 home,
2767 "task",
2768 &format!("task:{}", task.id),
2769 serde_json::to_value(request)?,
2770 )
2771 .await?;
2772 drop(claim);
2773 Ok(job)
2774}
2775
2776pub async fn run_detached_claimed(
2779 opts: &Opts,
2780 queue: &Queue,
2781 task: &mut Task,
2782 claim: crate::queue::Claim,
2783) -> Result<()> {
2784 let home = crate::run::home();
2785 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2786 loop {
2787 crate::worker::active(&home)?;
2790 let current = crate::worker::get(&home, &job.id)?;
2791 if matches!(
2792 current.status,
2793 crate::worker::Status::Finished | crate::worker::Status::Failed
2794 ) {
2795 *task = queue.get(&task.id)?;
2796 anyhow::ensure!(
2797 current.status != crate::worker::Status::Failed,
2798 "task worker failed: {}",
2799 current.result.as_deref().unwrap_or("unknown failure")
2800 );
2801 return Ok(());
2802 }
2803 tokio::time::sleep(Duration::from_millis(200)).await;
2804 }
2805}
2806
2807async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2808 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2809 let _lease = loop {
2810 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2811 break lease;
2812 }
2813 tokio::time::sleep(Duration::from_millis(10)).await;
2814 };
2815 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2816 let path = home.join("task-quota-cooldown.json");
2817 let temporary = path.with_extension("json.tmp");
2818 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2819 std::fs::rename(temporary, path)?;
2820 }
2821 }
2822 Ok(())
2823}
2824
2825fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2826 match std::fs::read(home.join("task-quota-cooldown.json")) {
2827 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2828 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2829 Err(error) => Err(error.into()),
2830 }
2831}
2832
2833pub async fn serve(opts: Opts) -> Result<()> {
2839 tracing::info!(
2840 "automatic update and restart requires magi web; standalone serve does not restart"
2841 );
2842 serve_until(opts, Stop::new()).await
2843}
2844
2845pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2856 let opts = opts.absolute()?;
2857 let home = crate::run::home();
2858 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2859 .context("another scheduler already owns this home")?;
2860 if let Some(reading) = read_status(&home)
2861 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2862 {
2863 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2864 }
2865 if !crate::worker::active(&home)?
2866 .iter()
2867 .any(|job| job.kind == "scheduler")
2868 {
2869 crate::worker::clear_control(&home, "scheduler")?;
2870 }
2871 std::fs::write(
2872 home.join("scheduler-options.json"),
2873 serde_json::to_vec(&opts)?,
2874 )?;
2875 let control = {
2876 let home = home.clone();
2877 let stop = stop.clone();
2878 tokio::spawn(async move {
2879 loop {
2880 match crate::worker::control(&home, "scheduler").ok().flatten() {
2881 Some(crate::worker::Control::Park) => {
2882 let _ = request_control(&home, crate::worker::Control::Park);
2883 stop.park(crate::run::ParkCause::Operator);
2884 break;
2885 }
2886 Some(crate::worker::Control::Stop) => {
2887 stop.stop();
2888 break;
2889 }
2890 None => {}
2891 }
2892 tokio::time::sleep(Duration::from_millis(100)).await;
2893 }
2894 })
2895 };
2896 let signal = {
2897 let stop = stop.clone();
2898 tokio::spawn(async move {
2899 if tokio::signal::ctrl_c().await.is_ok() {
2900 stop.stop();
2901 tracing::info!("scheduler shutdown requested; independent workers continue");
2902 }
2903 })
2904 };
2905
2906 let worktrees_root = opts
2907 .worktrees_root
2908 .clone()
2909 .unwrap_or_else(crate::run::default_worktree_root);
2910 let outcome = drive(
2911 &opts,
2912 &Queue::open(),
2913 &status_path(),
2914 &crate::run::home(),
2915 &worktrees_root,
2916 &stop,
2917 )
2918 .await;
2919
2920 signal.abort();
2921 control.abort();
2922 outcome
2923}
2924
2925async fn drive(
2938 opts: &Opts,
2939 queue: &Queue,
2940 status_file: &Path,
2941 home: &Path,
2942 worktrees_root: &Path,
2943 stop: &Stop,
2944) -> Result<()> {
2945 let recovery_cfg = prepare(&opts.repo, opts).ok();
2953 let daemon_cfg = recovery_cfg
2954 .as_ref()
2955 .map(|c| c.daemon.clone())
2956 .unwrap_or_default();
2957 let reconciled = crate::driver::reconcile(home);
2958 if !reconciled.is_empty() {
2959 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2960 }
2961 let mut initial_status = Status::new();
2962 let owned = owned_task_jobs(home)?;
2963 initial_status.current = current_work(home, Timestamp::now())
2964 .into_iter()
2965 .filter(|current| {
2966 owned
2967 .iter()
2968 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2969 })
2970 .collect();
2971 initial_status.idle = initial_status.current.is_empty();
2972 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2973 recovery_cfg.as_ref(),
2974 daemon_cfg.max_deputies,
2975 ));
2976 let status = Arc::new(Mutex::new(initial_status));
2977 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2978 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2979
2980 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2986
2987 tracing::info!(
2988 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2989 queue.root().display(),
2990 opts.poll.as_secs(),
2991 opts.max_attempts,
2992 concurrency,
2993 if daemon_cfg.pause_for_interrupts {
2994 ", interrupts enabled"
2995 } else {
2996 ""
2997 }
2998 );
2999
3000 let outcome = tokio::select! {
3001 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
3002 max_concurrent: concurrency,
3003 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
3004 recovery_cfg,
3005 }) => outcome,
3006 () = async {
3007 while !stop.stopped() {
3008 tokio::time::sleep(Duration::from_millis(25)).await;
3009 }
3010 } => {
3011 if stop.parking() {
3012 request_control(home, crate::worker::Control::Park)?;
3013 }
3014 Ok(())
3015 }
3016 };
3017
3018 stop.busy.store(0, Ordering::SeqCst);
3019 beat.abort();
3020 clear_status_at(status_file);
3021 outcome
3022}
3023
3024pub(crate) fn release_watch_settings_from_config(
3027 home: &Path,
3028 repo: &Path,
3029 cfg: &Config,
3030) -> (Vec<PathBuf>, u64) {
3031 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
3032 .into_iter()
3033 .map(|r| r.path)
3034 .collect();
3035 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
3038 .list()
3039 .into_iter()
3040 .map(|t| t.repo)
3041 .collect();
3042 let runs = home.join("runs");
3043 for id in crate::run::list_ids_in(&runs) {
3044 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
3045 .ok()
3046 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
3047 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
3048 known.extend(repo);
3049 }
3050 for k in known {
3051 let k = k.canonicalize().unwrap_or(k);
3052 if k.join(".git").exists() && !paths.contains(&k) {
3053 paths.push(k);
3054 }
3055 }
3056 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
3057 if !paths.contains(&own) {
3058 paths.push(own);
3059 }
3060 (paths, cfg.daemon.release_stall_minutes)
3061}
3062
3063const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
3065
3066async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
3072 loop {
3073 tokio::time::sleep(HEARTBEAT).await;
3074 let snapshot = {
3075 let mut guard = lock(&status);
3076 guard.updated_at = Timestamp::now();
3077 guard.clone()
3078 };
3079 if let Err(e) = write_status_to(&path, &snapshot) {
3080 tracing::warn!("could not refresh the daemon status file: {e:#}");
3083 }
3084 }
3085}
3086
3087#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3090enum LandResume {
3091 NotLanding,
3094 StillWaiting,
3099 Ready,
3103}
3104
3105fn land_resume_state(task: &Task) -> LandResume {
3109 let Some(run_id) = task.execution_runs().last() else {
3110 return LandResume::NotLanding;
3111 };
3112 let Ok(state) = RunState::load(run_id) else {
3113 return LandResume::NotLanding;
3114 };
3115 if !state.parked {
3116 return LandResume::NotLanding;
3117 }
3118 if state.status == RunStatus::Landing
3119 && state.land_approval.as_ref().is_some_and(|binding| {
3120 ask::Questions::open()
3121 .get(&binding.question)
3122 .is_ok_and(|q| {
3123 q.run == state.id
3124 && q.resolution()
3125 .as_deref()
3126 .is_some_and(crate::merge_decision::stops)
3127 })
3128 })
3129 {
3130 return LandResume::Ready;
3131 }
3132 if !implementers_match(task, &state) {
3133 return LandResume::NotLanding;
3134 }
3135 let node = if state.status == RunStatus::Landing {
3139 land::APPROVAL_NODE
3140 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
3141 crate::github_text::ASK_NODE
3142 } else {
3143 return LandResume::NotLanding;
3144 };
3145 let store = ask::Questions::open();
3146 let waiting = store
3147 .list()
3148 .into_iter()
3149 .filter(|q| &q.run == run_id && q.node == node)
3150 .max_by(|a, b| a.id.cmp(&b.id));
3151 let Some(mut q) = waiting else {
3152 return LandResume::Ready;
3153 };
3154 if !q.status.open() {
3155 if q.resolution().as_deref() == Some(land::REPAIR) {
3158 return if state.liveness(false) == Liveness::Live {
3159 LandResume::StillWaiting
3160 } else {
3161 LandResume::NotLanding
3162 };
3163 }
3164 return LandResume::Ready;
3165 }
3166 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
3173 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
3174 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3175 q.abandon(format!(
3176 "no answer within {}s of asking",
3177 timeout.as_secs().max(1)
3178 ));
3179 if store.put(&mut q).is_ok() {
3182 return LandResume::Ready;
3183 }
3184 }
3185 LandResume::StillWaiting
3186}
3187
3188const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3196
3197const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3209
3210#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3223enum Interrupt {
3224 Idle,
3226 Parking {
3238 parked: Vec<String>,
3239 interrupt_task: String,
3240 },
3241 Running {
3249 parked: Vec<String>,
3250 interrupt_task: String,
3251 },
3252 Resuming { parked: Vec<String> },
3259}
3260
3261fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3282 match state {
3283 Interrupt::Idle => {
3284 if in_flight.len() != 1 {
3296 return Interrupt::Idle;
3297 }
3298 match runnable.iter().find(|t| t.interrupt) {
3299 Some(t) => Interrupt::Parking {
3300 parked: in_flight.to_vec(),
3301 interrupt_task: t.id.clone(),
3302 },
3303 None => Interrupt::Idle,
3304 }
3305 }
3306 Interrupt::Parking {
3307 parked,
3308 interrupt_task,
3309 } => {
3310 if in_flight.iter().any(|id| parked.contains(id)) {
3311 Interrupt::Parking {
3313 parked,
3314 interrupt_task,
3315 }
3316 } else if in_flight.contains(&interrupt_task) {
3317 Interrupt::Running {
3318 parked,
3319 interrupt_task,
3320 }
3321 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3322 Interrupt::Parking {
3326 parked,
3327 interrupt_task,
3328 }
3329 } else {
3330 Interrupt::Resuming { parked }
3335 }
3336 }
3337 Interrupt::Running {
3338 parked,
3339 interrupt_task,
3340 } => {
3341 if in_flight.contains(&interrupt_task) {
3342 Interrupt::Running {
3343 parked,
3344 interrupt_task,
3345 }
3346 } else {
3347 Interrupt::Resuming { parked }
3353 }
3354 }
3355 Interrupt::Resuming { parked } => {
3356 if in_flight.iter().any(|id| parked.contains(id)) {
3357 Interrupt::Idle
3363 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3364 Interrupt::Resuming { parked }
3365 } else {
3366 Interrupt::Idle
3369 }
3370 }
3371 }
3372}
3373
3374fn advance_interrupt_tick(
3380 enabled: bool,
3381 state: Interrupt,
3382 in_flight: &[String],
3383 runnable: &[Task],
3384) -> Interrupt {
3385 if !enabled {
3386 return Interrupt::Idle;
3387 }
3388 advance_interrupt(state, in_flight, runnable)
3389}
3390
3391fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3396 match state {
3397 Interrupt::Idle => candidates,
3398 Interrupt::Parking {
3399 parked,
3400 interrupt_task,
3401 } => {
3402 if in_flight.iter().any(|id| parked.contains(id)) {
3403 Vec::new()
3404 } else {
3405 candidates
3406 .into_iter()
3407 .filter(|t| &t.id == interrupt_task)
3408 .collect()
3409 }
3410 }
3411 Interrupt::Running { .. } => Vec::new(),
3412 Interrupt::Resuming { parked } => candidates
3420 .into_iter()
3421 .find(|t| parked.contains(&t.id))
3422 .into_iter()
3423 .collect(),
3424 }
3425}
3426
3427struct DispatchLimits {
3431 max_concurrent: usize,
3434 pause_for_interrupts: bool,
3436 recovery_cfg: Option<Config>,
3437}
3438
3439#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3448enum PermitKind {
3449 None,
3454 Urgent,
3461 #[default]
3464 Ordinary,
3465}
3466
3467fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3470 if priority {
3471 PermitKind::None
3472 } else if urgent {
3473 PermitKind::Urgent
3474 } else {
3475 PermitKind::Ordinary
3476 }
3477}
3478
3479fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3480 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3481 (0usize, 0usize),
3482 |(ordinary, urgent), request| match request.lane {
3483 PermitKind::Ordinary => (ordinary + 1, urgent),
3484 PermitKind::Urgent => (ordinary, urgent + 1),
3485 PermitKind::None => (ordinary, urgent),
3486 },
3487 );
3488 (
3489 limit.saturating_sub(ordinary),
3490 1usize.saturating_sub(urgent),
3491 )
3492}
3493
3494struct DispatchClaim {
3495 task: Task,
3496 claim: crate::queue::Claim,
3497 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3498}
3499
3500fn claim_dispatch(
3503 queue: &Queue,
3504 candidate: &Task,
3505 cooling_down: bool,
3506 ordinary: &Arc<tokio::sync::Semaphore>,
3507 urgent: &Arc<tokio::sync::Semaphore>,
3508) -> Option<DispatchClaim> {
3509 let Ok(claim) = queue.claim(&candidate.id) else {
3510 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3511 return None;
3512 };
3513 let task = match queue.get(&candidate.id) {
3514 Ok(t) if t.status.runnable() => t,
3515 Ok(_) => return None,
3516 Err(e) => {
3517 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3518 return None;
3519 }
3520 };
3521 let resume = land_resume_state(&task);
3522 if resume == LandResume::StillWaiting {
3523 return None;
3524 }
3525 let priority = resume == LandResume::Ready;
3526 if !priority && cooling_down {
3527 return None;
3528 }
3529 let permit = match permit_kind(priority, task.urgent) {
3532 PermitKind::None => None,
3533 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3534 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3535 };
3536 let stamp = Timestamp::now();
3539 let mut task = task;
3540 task.dispatched_at = Some(stamp);
3541 if let Err(e) = queue.modify(&task.id, |t| {
3542 t.dispatched_at = Some(stamp);
3543 true
3544 }) {
3545 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3546 }
3547 Some(DispatchClaim {
3548 task,
3549 claim,
3550 permit,
3551 })
3552}
3553
3554#[derive(Serialize, Deserialize)]
3573struct ServiceWorker {
3574 opts: Opts,
3575 queue: PathBuf,
3576 worktrees_root: PathBuf,
3577 operation: String,
3578}
3579
3580fn load_conductor(home: &Path) -> Result<Conductor> {
3581 match std::fs::read(home.join("conductor-state.json")) {
3582 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3583 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3584 Err(error) => Err(error.into()),
3585 }
3586}
3587
3588fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3589 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3590 .into_iter()
3591 .map(|c| c.task)
3592 .collect();
3593 let stalled = stalled_tasks(queue, home, Timestamp::now());
3594 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3595 let finished = finished_tasks(queue)
3596 .into_iter()
3597 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3598 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3599 .collect();
3600 let queued = queued_tasks(queue)
3601 .into_iter()
3602 .filter(|t| !live.contains(&t.id))
3603 .collect();
3604 (queued, stalled, finished)
3605}
3606
3607fn retention_service_receipt(
3610 home: &Path,
3611 operation: &str,
3612 phase: &str,
3613 outcome: &str,
3614 detail: &str,
3615 cfg: Option<&Config>,
3616) {
3617 if !matches!(operation, "housekeeping" | "disk-pressure") {
3618 return;
3619 }
3620 let receipt = serde_json::json!({"checked_at": Timestamp::now(),
3621 "version": env!("CARGO_PKG_VERSION"), "pid": std::process::id(),
3622 "operation": operation, "outcome": outcome, "detail": detail,
3623 "auto_fold": cfg.map(|c| c.disk.auto_fold),
3624 "reclaim_bytes": cfg.map(|c| c.disk.worktree_reclaim_bytes)});
3625 if let Err(error) = crate::fold_evidence::atomic_write(
3626 &home.join(format!("source-eviction-{operation}-{phase}.json")),
3627 &serde_json::to_vec(&receipt).expect("service receipt"),
3628 ) {
3629 tracing::warn!("retention service receipt failed: {error:#}");
3630 }
3631}
3632
3633async fn start_service(
3634 home: &Path,
3635 opts: &Opts,
3636 queue: &Queue,
3637 worktrees_root: &Path,
3638 operation: &str,
3639 cfg: Option<&Config>,
3640 busy: bool,
3641) -> Result<()> {
3642 let fetch_cfg = if operation == "fetch" {
3645 match prepare(&opts.repo, opts) {
3646 Ok(cfg) => Some(cfg),
3647 Err(error) => {
3648 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3649 return Ok(());
3650 }
3651 }
3652 } else {
3653 None
3654 };
3655 let cfg = fetch_cfg.as_ref().or(cfg);
3656 let ticks = home.join("service-ticks");
3657 std::fs::create_dir_all(&ticks)?;
3658 let receipt = ticks.join(format!("daemon-{operation}"));
3659 let last = std::fs::read(&receipt)
3660 .ok()
3661 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3662 let interval = match operation {
3663 "disk-pressure" => 30,
3664 "cache" => CACHE_CHECK_INTERVAL_SECS,
3665 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3666 _ => opts.poll.as_secs().max(1),
3667 };
3668 let now = Timestamp::now();
3669 if !cache_check_due(last, now, interval) {
3670 retention_service_receipt(
3671 home,
3672 operation,
3673 "dispatch",
3674 "skipped",
3675 "cadence not due",
3676 cfg,
3677 );
3678 return Ok(());
3679 }
3680 let key = format!("service:daemon-{operation}");
3681 if crate::worker::active(home)?
3682 .iter()
3683 .any(|job| job.key == key)
3684 {
3685 retention_service_receipt(
3686 home,
3687 operation,
3688 "dispatch",
3689 "skipped",
3690 "service worker already active",
3691 cfg,
3692 );
3693 return Ok(());
3694 }
3695 let eligible = match operation {
3696 "conductor" => {
3697 let (queued, stalled, finished) = conductor_work(queue, home);
3698 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3699 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3700 }
3701 "maintenance" => Questions::at(home.join("questions"))
3702 .list()
3703 .iter()
3704 .any(|q| {
3705 q.chosen_action().is_some_and(|action| action.maintenance())
3706 && !home
3707 .join("choice-actions")
3708 .join(format!("{}.json", q.id))
3709 .exists()
3710 }),
3711 "housekeeping" => !busy && runnable(queue).is_empty(),
3712 "cache" => {
3713 !busy
3714 && !runnable(queue).is_empty()
3715 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3716 }
3717 "disk-pressure" => busy,
3718 "fetch" => {
3719 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3720 }
3721 _ => false,
3722 };
3723 if !eligible {
3724 retention_service_receipt(
3725 home,
3726 operation,
3727 "dispatch",
3728 "skipped",
3729 if operation == "housekeeping" {
3730 "active work or runnable task queue"
3731 } else {
3732 "no active work; idle housekeeping owns scan"
3733 },
3734 cfg,
3735 );
3736 return Ok(());
3737 }
3738 retention_service_receipt(
3739 home,
3740 operation,
3741 "dispatch",
3742 "launching",
3743 "eligible and cadence due",
3744 cfg,
3745 );
3746 let request = ServiceWorker {
3747 opts: opts.absolute()?,
3748 queue: std::path::absolute(queue.root())?,
3749 worktrees_root: std::path::absolute(worktrees_root)?,
3750 operation: operation.to_owned(),
3751 };
3752 if let Err(error) =
3753 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3754 {
3755 retention_service_receipt(
3756 home,
3757 operation,
3758 "dispatch",
3759 "error",
3760 &format!("worker launch: {error:#}"),
3761 cfg,
3762 );
3763 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3764 }
3765 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3766 Ok(())
3767}
3768
3769pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3772 let request: ServiceWorker = serde_json::from_value(payload)?;
3773 let opts = &request.opts;
3774 let queue = Queue::at(request.queue);
3775 let questions = Questions::at(home.join("questions"));
3776 match request.operation.as_str() {
3777 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3778 "conductor" => {
3779 let path = home.join("conductor-state.json");
3780 let mut conductor = load_conductor(home)?;
3781 let (queued, stalled, finished) = conductor_work(&queue, home);
3782 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3783 && conductor.worth_a_look(&queue, &stalled, &finished)
3784 {
3785 let cfg = prepare(&opts.repo, opts)?;
3786 conductor
3787 .maybe_run(
3788 &cfg,
3789 &opts.repo,
3790 &queue,
3791 &questions,
3792 home,
3793 &queued,
3794 &stalled,
3795 &finished,
3796 opts.max_attempts,
3797 )
3798 .await;
3799 let temporary = path.with_extension("json.tmp");
3800 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3801 std::fs::rename(temporary, path)?;
3802 }
3803 }
3804 "fetch" => {
3805 let cfg = prepare(&opts.repo, opts)?;
3806 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3807 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3808 tracing::debug!("fetch origins: {outcome:?}");
3809 }
3810 }
3811 "cache" => {
3812 if task_jobs(home)?.is_empty() {
3813 let mut checked = None;
3814 maybe_prune_cache_between_runs(
3815 &opts.repo,
3816 opts,
3817 home,
3818 &Stop::new(),
3819 &mut checked,
3820 Timestamp::now(),
3821 )
3822 .await;
3823 }
3824 }
3825 "disk-pressure" => {
3826 if let Ok(cfg) = prepare(&opts.repo, opts) {
3827 retention_service_receipt(
3828 home,
3829 "disk-pressure",
3830 "worker",
3831 "running",
3832 "retention sweep",
3833 Some(&cfg),
3834 );
3835 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3836 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3837 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3838 }
3839 for current in current_work(home, Timestamp::now()) {
3840 if let Ok(state) = RunState::load_under(¤t.run, home)
3841 && let Some(reason) = disk_gate(&state.repo, &state.config)
3842 {
3843 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3844 job.key.strip_prefix("task:") == Some(current.task.as_str())
3845 }) {
3846 request_task_park(
3847 home,
3848 &job,
3849 crate::run::ParkCause::Interrupt {
3850 reason: reason.clone(),
3851 },
3852 )?;
3853 }
3854 notices::raise_in(
3855 home,
3856 Notice::warn(
3857 "disk:pressure",
3858 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3859 ),
3860 );
3861 }
3862 }
3863 } else {
3864 retention_service_receipt(
3865 home,
3866 "disk-pressure",
3867 "worker",
3868 "error",
3869 "configuration could not be loaded",
3870 None,
3871 );
3872 }
3873 }
3874 "housekeeping" => {
3875 if task_jobs(home)?.is_empty() {
3876 match prepare(&opts.repo, opts) {
3877 Ok(cfg) => {
3878 retention_service_receipt(
3879 home,
3880 "housekeeping",
3881 "worker",
3882 "running",
3883 "retention sweep",
3884 Some(&cfg),
3885 );
3886 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3887 }
3888 Err(error) => retention_service_receipt(
3889 home,
3890 "housekeeping",
3891 "worker",
3892 "error",
3893 &format!("configuration: {error:#}"),
3894 None,
3895 ),
3896 }
3897 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3898 resweep_superseded_attempts(&queue, home);
3899 triage_held(&queue, home, opts).await;
3900 } else {
3901 retention_service_receipt(
3902 home,
3903 "housekeeping",
3904 "worker",
3905 "skipped",
3906 "task worker became active",
3907 None,
3908 );
3909 }
3910 }
3911 operation => bail!("unknown daemon service {operation}"),
3912 }
3913 Ok(())
3914}
3915
3916fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3920 let key = format!("task:{}", request.task);
3921 let payload = serde_json::to_value(request)?;
3922 if crate::worker::active(home)?
3923 .iter()
3924 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3925 {
3926 tracing::warn!(
3927 "task {} acquired a competing worker reservation: {error:#}",
3928 crate::run::short_of(&request.task)
3929 );
3930 return Ok(());
3931 }
3932 tracing::error!("task worker startup stopped dispatch: {error:#}");
3933 notices::raise_in(
3934 home,
3935 Notice::error(
3936 "loop:worker-startup",
3937 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
3940 ).link(Link::Task { id: request.task.clone() }),
3941 );
3942 Err(error).with_context(|| {
3943 format!(
3944 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3945 crate::run::short_of(&request.task)
3946 )
3947 })
3948}
3949
3950async fn poll(
3951 opts: &Opts,
3952 queue: &Queue,
3953 status: &Arc<Mutex<Status>>,
3954 home: &Path,
3955 worktrees_root: &Path,
3956 stop: &Stop,
3957 limits: DispatchLimits,
3958) -> Result<()> {
3959 let mut attempted = Vec::new();
3960 let mut settling_once_services = false;
3961 let interrupt_path = home.join("scheduler-interrupt.json");
3962 let mut interrupt = match std::fs::read(&interrupt_path) {
3963 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3964 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3965 Err(error) => return Err(error.into()),
3966 };
3967 let mut previous = std::collections::BTreeSet::<String>::new();
3968 while !stop.stopped() {
3969 let jobs: Vec<_> = crate::worker::active(home)?
3970 .into_iter()
3971 .filter(|job| job.kind == "task")
3972 .collect();
3973 let requests: Vec<_> = jobs
3974 .iter()
3975 .map(|job| {
3976 serde_json::from_value::<TaskWorker>(job.payload.clone())
3977 .with_context(|| format!("read live task worker {}", job.id))
3978 })
3979 .collect::<Result<_>>()?;
3980 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3981 let owned: std::collections::BTreeSet<_> = requests
3982 .iter()
3983 .filter(|r| !r.standalone)
3984 .map(|r| r.task.clone())
3985 .collect();
3986 let current: Vec<_> = requests
3987 .iter()
3988 .zip(&jobs)
3989 .filter(|(request, _)| !request.standalone)
3990 .map(|(request, job)| Current {
3991 worker: Some(job.id.clone()),
3992 task: request.task.clone(),
3993 run: queue
3994 .get(&request.task)
3995 .ok()
3996 .and_then(|task| task.execution_runs().last().cloned())
3997 .unwrap_or_default(),
3998 })
3999 .collect();
4000 {
4001 let mut snapshot = lock(status);
4002 snapshot.polls += 1;
4003 snapshot.completed += previous.difference(&owned).count();
4004 snapshot.current = current;
4005 snapshot.idle = owned.is_empty();
4006 }
4007 previous = owned.clone();
4008 stop.busy.store(owned.len(), Ordering::SeqCst);
4009
4010 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
4013 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
4014 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
4015 let mut permits = Vec::new();
4016 sweep_stale_claims(queue, STALE_CLAIM);
4017 reclaim_orphaned_running(queue, opts.max_attempts, &live);
4018 reclaim_abandoned_runs(home, Timestamp::now());
4019 let questions = Questions::at(home.join("questions"));
4020
4021 crate::task_chat::sweep(
4024 queue,
4025 &crate::talk::Talks::at(home.join("talks")),
4026 &questions,
4027 home,
4028 );
4029 if let Err(error) = crate::task_inquiry::Outbox::at(home)
4030 .sweep(queue, &crate::talk::Talks::at(home.join("talks")))
4031 {
4032 tracing::warn!("task inquiry delivery remains pending: {error:#}");
4033 }
4034 repair_choice_offers(&questions);
4035 resolve_blockers(queue, &questions);
4036 apply_task_choice_actions(queue, &questions, home);
4037 triage::apply_answers(queue, &questions);
4038 reconcile_task_questions(queue, &questions);
4039 let talks = crate::talk::Talks::at(home.join("talks"));
4040 crate::chat_notify::sweep(queue, &talks, home);
4041 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
4042 crate::chat_watch::sweep(home, &talks);
4043 if let Err(error) = crate::chat_control::reconcile_decisions(queue, home) {
4044 tracing::warn!("decision receipt reconciliation remains pending: {error:#}");
4045 }
4046 let mut turns = tokio::task::JoinSet::new();
4047 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
4048 while turns.join_next().await.is_some() {}
4050
4051 crate::consult::start_services(
4052 home,
4053 &opts.repo,
4054 limits.recovery_cfg.as_ref(),
4055 opts.config.as_deref(),
4056 )
4057 .await?;
4058 for operation in [
4059 "conductor",
4060 "maintenance",
4061 "housekeeping",
4062 "disk-pressure",
4063 "cache",
4064 "fetch",
4065 ] {
4066 if settling_once_services {
4070 break;
4071 }
4072 start_service(
4073 home,
4074 opts,
4075 queue,
4076 worktrees_root,
4077 operation,
4078 limits.recovery_cfg.as_ref(),
4079 !live.is_empty(),
4080 )
4081 .await?;
4082 }
4083 let latest = crate::worker::active(home)?;
4084 if latest.iter().any(|job| job.key == "service:daemon-cache")
4085 || latest
4086 .iter()
4087 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
4088 {
4089 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
4090 continue;
4091 }
4092 let candidates: Vec<Task> = runnable(queue)
4093 .into_iter()
4094 .filter(|task| !live.contains(&task.id))
4095 .filter(|task| !opts.once || !attempted.contains(&task.id))
4096 .collect();
4097 let in_flight: Vec<_> = owned.iter().cloned().collect();
4098 let prior_interrupt = std::mem::discriminant(&interrupt);
4099 interrupt = advance_interrupt_tick(
4100 limits.pause_for_interrupts,
4101 interrupt,
4102 &in_flight,
4103 &candidates,
4104 );
4105 if prior_interrupt != std::mem::discriminant(&interrupt) {
4108 let temporary = interrupt_path.with_extension("json.tmp");
4109 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
4110 std::fs::rename(temporary, &interrupt_path)?;
4111 }
4112 if let Interrupt::Parking {
4113 parked,
4114 interrupt_task,
4115 } = &interrupt
4116 {
4117 for job in jobs.iter().filter(|job| {
4118 parked
4119 .iter()
4120 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
4121 }) {
4122 if crate::worker::control(home, &job.key)?.is_none() {
4123 request_task_park(
4124 home,
4125 job,
4126 crate::run::ParkCause::Interrupt {
4127 reason: format!(
4128 "task {} asked to run first",
4129 crate::run::short_of(interrupt_task)
4130 ),
4131 },
4132 )?;
4133 }
4134 }
4135 }
4136 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
4137 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
4138 let mut started = false;
4139 let mut resync = false;
4140 for candidate in candidates {
4141 if stop.stopped() {
4142 break;
4143 }
4144 let Some(DispatchClaim {
4145 task,
4146 claim,
4147 permit,
4148 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
4149 else {
4150 continue;
4151 };
4152 if crate::worker::active(home)?
4153 .iter()
4154 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
4155 {
4156 drop(claim);
4157 resync = true;
4158 break;
4159 }
4160 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
4161 let request = TaskWorker {
4162 opts: opts.absolute()?,
4163 queue: std::path::absolute(queue.root())?,
4164 task: task.id.clone(),
4165 lane,
4166 standalone: false,
4167 };
4168 if let Err(error) = crate::worker::spawn(
4171 home,
4172 "task",
4173 &format!("task:{}", task.id),
4174 serde_json::to_value(&request)?,
4175 )
4176 .await
4177 {
4178 task_worker_spawn_error(home, &request, error)?;
4179 drop(claim);
4180 resync = true;
4181 break;
4182 }
4183 drop(claim);
4184 permits.extend(permit);
4185 attempted.push(task.id);
4186 started = true;
4187 }
4188 if started || resync {
4189 continue;
4190 }
4191 if owned.is_empty() && opts.once {
4192 settling_once_services = true;
4193 if crate::worker::active(home)?.iter().any(|job| {
4197 job.kind == "daemon-service"
4198 && matches!(
4199 job.key.as_str(),
4200 "service:daemon-housekeeping"
4201 | "service:daemon-conductor"
4202 | "service:daemon-maintenance"
4203 )
4204 }) {
4205 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
4206 continue;
4207 }
4208 break;
4209 }
4210 stop.idle(if live.is_empty() {
4211 opts.poll
4212 } else {
4213 RECHECK_WHILE_BUSY.min(opts.poll)
4214 })
4215 .await;
4216 }
4217 stop.busy.store(0, Ordering::SeqCst);
4220 Ok(())
4221}
4222
4223async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
4227 let busy = |error: &anyhow::Error| {
4228 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
4229 error.kind() == std::io::ErrorKind::WouldBlock
4230 || cfg!(windows) && error.raw_os_error() == Some(33)
4231 })
4232 };
4233 match crate::pr_head::owner_lock(home, id) {
4234 Ok(lock) => return Ok(lock),
4235 Err(error) if busy(&error) => {}
4236 Err(error) => return Err(error),
4237 }
4238 let wait = RunState::load_under(id, home)?
4239 .config
4240 .graph
4241 .timeout_resource_wait;
4242 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
4243}
4244
4245async fn public_resume_owner(
4249 home: &Path,
4250 runs: &[String],
4251 repo: &Path,
4252) -> Result<Option<RunState>> {
4253 let mut owner = None;
4254 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
4255 let _lock = resume_owner_lock(home, id).await?;
4256 let state = RunState::load_under(id, home)?;
4257 if state.released()
4258 || state.retired_to.is_some()
4259 || state.pr_taken_over_by.is_some()
4260 || !crate::handover::landing_checkpoint(&state)
4261 {
4262 continue;
4263 }
4264 anyhow::ensure!(
4265 crate::git::toplevel(&state.repo).await?.canonicalize()?
4266 == crate::git::toplevel(repo).await?.canonicalize()?,
4267 "public owner repository differs; inspect run lineage"
4268 );
4269 anyhow::ensure!(
4270 owner.is_none(),
4271 "multiple retained public owners; inspect run lineage"
4272 );
4273 if let Some(reason) = crate::handover::resume_refusal_under(&state, Some(home)) {
4274 anyhow::bail!(reason.diagnostic(&state));
4275 }
4276 let claimed = is_working_on_other(home, id, Timestamp::now())?;
4277 anyhow::ensure!(
4278 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
4279 "public owner's driver is live or unknown; wait before retrying"
4280 );
4281 owner = Some(state);
4282 }
4283 Ok(owner)
4284}
4285
4286async fn attempt(
4292 opts: &Opts,
4293 queue: &Queue,
4294 status: &Arc<Mutex<Status>>,
4295 stop: &Stop,
4296 interrupt_pause: crate::graph::Pause,
4297 task: &mut Task,
4298) -> Vec<QuotaLoss> {
4299 let repo = repo_for(task, &opts.repo);
4300 tracing::info!(
4301 "task {} — {} (repo {})",
4302 task.short(),
4303 task.title,
4304 repo.display()
4305 );
4306
4307 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
4308 Ok(owner) => owner,
4309 Err(e) => {
4310 task.hold_machine(Some(format!(
4311 "Cannot verify existing public owner: {e:#}; no attempt started."
4312 )));
4313 record(queue, task);
4314 return Vec::new();
4315 }
4316 };
4317 let resume_runs = public_owner
4318 .as_ref()
4319 .map(|s| vec![s.id.clone()])
4320 .unwrap_or_else(|| task.execution_runs().to_vec());
4321
4322 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4326 && previous
4327 .merge_decision
4328 .as_ref()
4329 .is_some_and(|decision| !decision.closed)
4330 && previous.status != RunStatus::Landing
4331 && !crate::merge_decision::resumable_hold(&previous)
4332 {
4333 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4334 record(queue, task);
4335 return Vec::new();
4336 }
4337 let stop_owner = resume_runs
4338 .last()
4339 .and_then(|id| RunState::load(id).ok())
4340 .filter(|state| {
4341 crate::merge_decision::resumable_hold(state)
4342 || state.status == RunStatus::Landing
4343 && state.land_approval.as_ref().is_some_and(|binding| {
4344 Questions::open().get(&binding.question).is_ok_and(|q| {
4345 q.run == state.id
4346 && q.resolution()
4347 .as_deref()
4348 .is_some_and(crate::merge_decision::stops)
4349 })
4350 })
4351 });
4352 if let Some(owner) = stop_owner {
4353 let mut runner = match Runner::resume(&owner.id) {
4354 Ok(runner) => runner,
4355 Err(e) => {
4356 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4357 record(queue, task);
4358 return Vec::new();
4359 }
4360 };
4361 runner.on_pause(stop.pause());
4362 runner.watch_interrupt(interrupt_pause);
4363 let quota_before = runner.state.quota.clone();
4364 let before_start = task.clone();
4365 task.start(owner.id.clone());
4366 if let Err(error) = queue.put(task) {
4367 *task = before_start;
4368 let error = runner.fail_startup("task-start", error);
4369 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4370 task.record_startup_failure(failure.clone());
4371 }
4372 task.hold_manual(Some(format!(
4373 "Could not persist task start; no agents dispatched: {error:#}"
4374 )));
4375 record(queue, task);
4376 return Vec::new();
4377 }
4378 lock(status).current.push(Current {
4379 worker: None,
4380 task: task.id.clone(),
4381 run: owner.id,
4382 });
4383 runner.state.carry_dispatched_resumes(task);
4384 runner.state.next_pass_task_entry = true;
4385 let result = runner.execute().await;
4386 return finish_attempt(
4387 opts.max_attempts,
4388 queue,
4389 task,
4390 &runner.state,
4391 "a_before,
4392 result,
4393 );
4394 }
4395
4396 if let Some(owner) = &public_owner
4397 && (task.fresh_start || !may_resume_run(owner, task))
4398 {
4399 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4400 record(queue, task);
4401 return Vec::new();
4402 }
4403
4404 if let Some(state) = task
4405 .execution_runs()
4406 .last()
4407 .and_then(|id| RunState::load(id).ok())
4408 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
4409 {
4410 task.hold_machine(Some(reason));
4411 record(queue, task);
4412 return Vec::new();
4413 }
4414
4415 if let Some(held) = crate::base_sync_recovery::held_run(queue, task)
4416 && !task.fresh_start
4417 && task.pr_recovery.is_none()
4418 {
4419 let authorized = if crate::repair_budget::conflict_pending(task, &held) {
4422 true
4423 } else if task.resumption.is_some() && compatible_run_instruction(&held, task) {
4424 let home = queue.root().parent().expect("queue has a home");
4425 match Runner::authorize_base_recovery_from_resumption(home, &held.id, &task.id).await {
4426 Ok(()) => true,
4427 Err(error) => {
4428 tracing::debug!(run = %held.id, %error, "base recovery resumption refused");
4429 false
4430 }
4431 }
4432 } else {
4433 false
4434 };
4435 if !authorized {
4436 task.hold_machine(Some(format!("Base synchronization remains stopped. {} A release without a new owner approval grants no additional synchronization budget and starts no replacement competition.", held.base_sync_next_action())));
4437 task.base_sync_notice(&held);
4439 task.resume_override = None;
4440 record(queue, task);
4441 return Vec::new();
4442 }
4443 }
4444 let unfinished = (!task.fresh_start)
4445 .then(|| unfinished_run(&resume_runs, task.short()))
4446 .flatten()
4447 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4448 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4449 if !task.fresh_start
4450 && task.pr_recovery.is_none()
4451 && task.review_branch.is_none()
4452 && task.resume_override.is_none()
4453 && previous
4454 .as_ref()
4455 .is_some_and(|run| run.retired_to.is_some())
4456 && unfinished.is_none()
4457 {
4458 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4459 record(queue, task);
4460 return Vec::new();
4461 }
4462
4463 let saved_recovery = previous.as_ref().filter(|run| {
4464 task.replacement.is_none()
4465 && task.pr_recovery.as_ref().is_some_and(|selected| {
4466 selected
4467 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4468 .is_ok()
4469 && selected_starter(selected, Some(run), unfinished.as_deref())
4470 == Starter::Resume(run.id.clone())
4471 })
4472 });
4473
4474 let explicit = match &task.overrides {
4478 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4479 Some(_) => opts.config.as_deref(),
4480 None => opts.config.as_deref(),
4481 };
4482 let (mut config, layers) = if let Some(run) = saved_recovery {
4483 let mut saved = run.clone();
4486 if let Err(e) = saved.configure_recovery_resume() {
4487 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4488 record(queue, task);
4489 return Vec::new();
4490 }
4491 (saved.config, Vec::new())
4492 } else {
4493 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4494 Ok(loaded) => loaded,
4495 Err(e) => {
4496 let reason = format!("[config] {e:#}");
4497 record_config_failure(task, &e);
4498 task.last_error = Some(reason.clone());
4499 task.hold_machine(Some(reason));
4500 record(queue, task);
4501 return Vec::new();
4502 }
4503 };
4504 if let Err(e) = apply_options(&mut config, opts, task) {
4505 task.hold_manual(Some(format!(
4506 "[config] {e:#}; correct the merge override before releasing the task"
4507 )));
4508 record(queue, task);
4509 return Vec::new();
4510 }
4511 if let Some(selected) = &task.pr_recovery
4512 && let Err(e) = selected.configure(&mut config)
4513 {
4514 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4515 record(queue, task);
4516 return Vec::new();
4517 }
4518 (config, layers)
4519 };
4520 let config_evidence = crate::pr_recovery::ConfigEvidence {
4521 layers,
4522 overrides: task.overrides.clone(),
4523 daemon_merge: task
4524 .overrides
4525 .as_ref()
4526 .is_none_or(|o| o.inherit_daemon_options)
4527 .then(|| opts.merge.clone())
4528 .flatten(),
4529 };
4530 apply_task_implementers(&mut config, task);
4531 if let Some(replacement) = &task.replacement {
4532 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4533 old.liveness(false) == Liveness::Dead
4534 && old.merge_decision.as_ref().is_some_and(|d| {
4535 d.closed
4536 && d.question == replacement.question
4537 && d.replacement_run.as_ref().is_none_or(|run| {
4538 task.execution_runs().last() == Some(run)
4539 && RunState::load(run).is_ok_and(|current| {
4540 current.replaces.as_ref().is_some_and(|pin| {
4541 pin.question == replacement.question
4542 && pin.agent == replacement.agent
4543 })
4544 })
4545 })
4546 })
4547 });
4548 if !safe {
4549 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4550 record(queue, task);
4551 return Vec::new();
4552 }
4553 match config.agent(&replacement.agent) {
4554 Ok(agent) if crate::agent::installed(agent) => {}
4555 _ => {
4556 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4557 record(queue, task);
4558 return Vec::new();
4559 }
4560 }
4561 config.graph.implementers = 1;
4562 config.graph.land_approval = true;
4563 config.merge.mode = MergeMode::Pr;
4564 config.graph.selected_implementer = Some(replacement.agent.clone());
4565 }
4566 let p = phrases(&config.graph.language);
4567 let start_failed = p.could_not_start;
4568 if let Ok(base) =
4573 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4574 .await
4575 && let Ok(head) =
4576 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4577 {
4578 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4579 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4580 let source = crate::git::rev_parse(&repo, branch)
4581 .await
4582 .unwrap_or_default();
4583 inputs.push_str(&format!(";review:{branch}:{source}"));
4584 }
4585 if let Some(reason) = task
4586 .recovery_exhausted(
4587 &head,
4588 &inputs,
4589 task.attempt_limit
4590 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4591 )
4592 .filter(|_| {
4593 !task.repair_grants.last().is_some_and(|receipt| {
4594 RunState::load(&receipt.request.run)
4595 .is_ok_and(|run| crate::repair_budget::pending(task, &run))
4596 })
4597 })
4598 {
4599 task.hold_unrequested_retry(Some(reason));
4600 record(queue, task);
4601 return Vec::new();
4602 }
4603 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4604 }
4605
4606 if let Some(reason) = disk_gate(&repo, &config) {
4614 task.notice_cause("disk-space", None, None);
4615 task.last_error = Some(reason.clone());
4616 task.hold_machine(Some(reason.clone()));
4617 record(queue, task);
4618 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4619 notices::raise(
4622 Notice::warn(
4623 &format!("disk:{}", repo.display()),
4624 format!("Task {} held for want of free disk space: {reason}. Free some, then release it from the queue.", task.id),
4625 )
4626 .link(Link::Task {
4627 id: task.id.clone(),
4628 }),
4629 );
4630 return Vec::new();
4631 }
4632
4633 let review_branch = if previous
4658 .as_ref()
4659 .is_some_and(|state| crate::repair_budget::pending(task, state))
4660 {
4661 None
4662 } else {
4663 take_review_branch(task, previous.as_ref())
4664 };
4665 let branch_exists = match &review_branch {
4666 Some(branch) => crate::git::branch_exists(&repo, branch)
4667 .await
4668 .unwrap_or(false),
4669 None => false,
4670 };
4671 let proposed = review_retry_starter(
4675 task,
4676 previous.as_ref(),
4677 review_branch.as_deref(),
4678 branch_exists,
4679 unfinished.as_deref(),
4680 );
4681 let retry_selected = task.replacement.is_some()
4682 && task.resumption.is_some()
4683 && !task.fresh_start
4684 && previous.as_ref().is_some_and(|run| {
4685 run.replaces.is_some()
4686 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4687 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4688 });
4689 let stop_pin = task
4690 .resume_override
4691 .as_ref()
4692 .filter(|pin| {
4693 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4694 q.node == land::APPROVAL_NODE
4695 && q.resolution()
4696 .as_deref()
4697 .is_some_and(crate::merge_decision::stops)
4698 })
4699 })
4700 .and_then(|pin| pin.pinned_run.clone());
4701 let granted_run = task.repair_grants.last().and_then(|receipt| {
4702 let state = RunState::load(&receipt.request.run).ok()?;
4703 crate::repair_budget::pending(task, &state).then(|| receipt.request.run.clone())
4704 });
4705 let starter = if let Some(run) = granted_run {
4706 Starter::Resume(run)
4707 } else if let Some(owner) = &public_owner {
4708 Starter::Resume(owner.id.clone())
4709 } else if let Some(run) = stop_pin {
4710 Starter::Resume(run)
4713 } else if retry_selected {
4714 Starter::Resume(previous.as_ref().unwrap().id.clone())
4715 } else if let Some(selected) = &task.pr_recovery {
4716 if selected
4717 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4718 .is_ok()
4719 {
4720 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4721 } else {
4722 Starter::Review(selected.branch.clone())
4723 }
4724 } else {
4725 starter_with_implementers(task, previous.as_ref(), proposed)
4726 };
4727 if let Some(pin) = &task.resume_override
4730 && let Some(run) = &pin.pinned_run
4731 && starter != Starter::Resume(run.clone())
4732 {
4733 let run = run.clone();
4734 task.notice_cause("resume-owner", Some(&run), None);
4735 task.hold_machine(Some(format!(
4736 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4737 task.id
4738 )));
4739 record(queue, task);
4740 return Vec::new();
4741 }
4742 if task.replacement.is_some()
4743 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4744 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4745 {
4746 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4747 record(queue, task);
4748 return Vec::new();
4749 }
4750 if let Some(reason) = retained_attempt_hold(
4751 task,
4752 &starter,
4753 config.disk.retained_attempt_limit,
4754 RunState::load,
4755 ) {
4756 if let Starter::Review(branch) = &starter {
4759 task.review_branch = Some(branch.clone());
4760 }
4761 task.hold_unrequested_retry(Some(reason));
4762 record(queue, task);
4763 return Vec::new();
4764 }
4765 let attachments = match task_attachments(queue, task) {
4770 Ok(a) => a,
4771 Err(e) => {
4772 task.attempts += 1;
4773 task.fail_startup_target(
4774 "startup-attachments",
4775 e.name.clone(),
4776 format!("{start_failed}{e:#}"),
4777 opts.max_attempts,
4778 );
4779 record(queue, task);
4780 return Vec::new();
4781 }
4782 };
4783 let startup_pause = stop.pause();
4784 let startup_interrupt = interrupt_pause.clone();
4785 let wait = crate::repo_lock::WaitPolicy::new(
4786 Duration::from_secs(config.graph.timeout_resource_wait),
4787 move || startup_pause.parked() || startup_interrupt.parked(),
4788 );
4789 let started = wait.scope(Box::pin(async { match &starter {
4792 Starter::Review(_) if task.pr_recovery.is_some() => {
4793 let selected = task.pr_recovery.clone().expect("selected recovery");
4794 match selected.check_task_assignment(&task.id, &task.instruction, task.reassignments.len()) {
4795 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4796 .await
4797 .map(|mut r| {
4798 r.state.attachments = attachments.clone();
4799 r
4800 }),
4801 Err(e) => Err(e),
4802 }
4803 }
4804 Starter::Review(branch) => {
4805 tracing::info!(
4806 "task {} reopens `{branch}` as a review-only pass",
4807 task.short()
4808 );
4809 let takeover = crate::handover::Takeover {
4812 earlier: task.earlier_attempts().to_vec(),
4813 home: crate::run::home(),
4814 choice: take_divergence_answer(branch, &config.merge.remote, task),
4815 task_instruction: Some(instruction_for(task)),
4816 };
4817 Runner::review_taking_over(
4818 &repo,
4819 branch,
4820 config,
4821 Some(takeover),
4822 crate::run::Origin::queue(&task.id),
4823 )
4824 .await
4825 }
4826 Starter::Resume(id) => {
4827 tracing::info!("resuming run {id} rather than competing again");
4828 Runner::resume(id).and_then(|mut r| {
4829 if retry_selected {
4830 for candidate in &mut r.state.candidates {
4833 candidate.failed = None;
4834 candidate.empty = false;
4835 candidate.commits = 0;
4836 }
4837 r.state.status = RunStatus::Implementing;
4838 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4839 }
4840 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4841 r.state.sources = task.sources.clone();
4842 }
4843 if r.state.prep_failure.is_some() {
4844 let approval = task
4845 .resumption
4846 .as_ref()
4847 .context("preparation retry requires an explicit task release")?;
4848 r.retry_preparation(approval.at)?;
4849 }
4850 if let Some(instruction) =
4851 prepare_instruction(&starter, Some(&r.state.instruction), task)
4852 {
4853 r.state.instruction = instruction;
4854 }
4855 r.state.attachments = attachments.clone();
4856 if task.replacement.is_some() {
4857 r.state.config.merge.mode = MergeMode::Pr;
4858 r.state.config.graph.land_approval = true;
4859 }
4860 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4864 .overrides
4865 .as_ref()
4866 .and_then(|o| o.merge.as_deref())
4867 .and_then(|m| merge_mode(m).ok())
4868 {
4869 r.state.config.merge.mode = mode;
4870 }
4871 Ok(r)
4872 })
4873 }
4874 Starter::Start => {
4875 if let Some(branch) = &review_branch {
4876 tracing::warn!(
4877 "conductor chose review for task {} but branch `{branch}` no longer \
4878 exists; requeuing as a fresh competition instead",
4879 task.short()
4880 );
4881 }
4882 let instruction = prepare_instruction(&starter, None, task)
4883 .unwrap_or_else(|| task.instruction.clone());
4884 Runner::start_naming(
4885 &repo,
4886 instruction,
4887 &task.title,
4888 config,
4889 crate::run::Origin::queue(&task.id),
4890 )
4891 .await
4892 .map(|mut r| {
4893 r.state.attachments = attachments.clone();
4894 r.state.sources = task.sources.clone();
4895 r
4896 })
4897 }
4898 }})).await;
4899 if let Err(error) = &started
4900 && let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>()
4901 {
4902 task.record_startup_failure(failure.clone());
4903 }
4904 let mut runner = match started {
4905 Ok(r) => r,
4906 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4907 task.last_error = Some(e.to_string());
4908 if stop.parking() || interrupt_pause.parked() {
4909 let attempts = task.attempts;
4910 task.park(e.to_string());
4911 task.attempts = attempts;
4913 } else {
4914 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4915 }
4916 record(queue, task);
4917 return Vec::new();
4918 }
4919 Err(e) if task.pr_recovery.is_some() => {
4923 task.startup_notice_cause("startup-public-head");
4924 task.attempts += 1;
4925 task.last_error = Some(format!("{start_failed}{e:#}"));
4926 task.hold_unrequested_retry(Some(format!(
4927 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4928 )));
4929 record(queue, task);
4930 return Vec::new();
4931 }
4932 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4933 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4934 Some(r) => (p.handover_refused)(r),
4935 None => format!("{e:#}"),
4936 };
4937 task.startup_notice_cause("startup-handover");
4938 let reason = format!("{start_failed}{detail}");
4939 task.last_error = Some(reason.clone());
4940 let branch = match &starter {
4943 Starter::Review(branch) => Some(branch.clone()),
4944 _ => None,
4945 };
4946 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4950 record(queue, task);
4951 tracing::warn!(
4952 "holding {} for a branch it cannot take over: {e:#}",
4953 task.short()
4954 );
4955 return Vec::new();
4956 }
4957 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4962 let d = e
4963 .downcast_ref::<crate::reconcile::Diverged>()
4964 .expect("checked by the guard");
4965 let mut q = ask::Question::new(
4966 task.id.clone(),
4967 crate::reconcile::NODE.to_owned(),
4968 crate::reconcile::SEAT.to_owned(),
4969 d.summary(),
4970 d.detail(),
4971 d.choices(),
4972 );
4973 match Questions::open().put(&mut q) {
4974 Ok(()) => {
4975 task.last_error = Some(format!("{e:#}"));
4976 task.review_branch = Some(d.branch.clone());
4979 task.review_instruction = Some(task.instruction.clone());
4980 task.block(vec![q.id.clone()], Some(d.summary()));
4981 }
4982 Err(put) => {
4983 tracing::warn!("could not file the divergence question: {put:#}");
4984 task.attempts += 1;
4985 task.fail_startup(
4986 "startup-question-store",
4987 format!("{start_failed}{e:#}"),
4988 opts.max_attempts,
4989 );
4990 }
4991 }
4992 record(queue, task);
4993 return Vec::new();
4994 }
4995 Err(e) => {
4996 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
5002 && let Starter::Review(branch) = &starter
5003 {
5004 task.review_branch = Some(branch.clone());
5005 task.review_instruction = Some(task.instruction.clone());
5006 task.last_error = Some(format!("{start_failed}{e:#}"));
5007 task.status = crate::queue::TaskStatus::Failed;
5008 record(queue, task);
5009 return Vec::new();
5010 }
5011 task.attempts += 1;
5012 task.last_error = Some(format!("{start_failed}{e:#}"));
5013 task.startup_notice_cause("startup-source");
5014 task.hold_unrequested_retry(Some(format!(
5015 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
5016 )));
5017 record(queue, task);
5018 return Vec::new();
5019 }
5020 };
5021 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
5022 runner.state.recovery_config = Some(config_evidence);
5023 }
5024 if let Some(replacement) = &task.replacement {
5025 if matches!(starter, Starter::Start) {
5026 runner.state.instruction.push_str(&format!(
5027 "\n\n# Replacement review context\n{}",
5028 replacement.context
5029 ));
5030 }
5031 runner.state.replaces = Some(replacement.clone());
5032 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
5033 if let Ok(mut old) = RunState::load(&replacement.run)
5034 && let Some(d) = &mut old.merge_decision
5035 {
5036 d.replacement_run = Some(runner.state.id.clone());
5037 old.event(
5038 "replacement",
5039 format!(
5040 "replacement run {} owns the next attempt; old PR remains closed",
5041 runner.state.id
5042 ),
5043 );
5044 if let Err(e) = old.save() {
5045 let error = runner.fail_startup("replacement-history", e);
5046 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
5047 task.record_startup_failure(failure.clone());
5048 }
5049 task.hold_manual(Some(format!(
5050 "Could not link replacement history: {error:#}; no agents dispatched"
5051 )));
5052 record(queue, task);
5053 return Vec::new();
5054 }
5055 }
5056 }
5057 runner.state.task_instruction = Some(task.instruction.clone());
5058 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
5061 if runner.state.origin_chat.is_none() {
5062 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
5063 }
5064 runner.on_pause(stop.pause());
5066 runner.watch_interrupt(interrupt_pause);
5070
5071 let run = runner.state.id.clone();
5074 let before_start = task.clone();
5075 task.start(run.clone());
5076 if let Err(error) = queue.put(task) {
5077 *task = before_start;
5078 let error = runner.fail_startup("task-start", error);
5079 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
5080 task.record_startup_failure(failure.clone());
5081 }
5082 task.hold_manual(Some(format!(
5083 "Could not persist task start; no agents dispatched: {error:#}"
5084 )));
5085 record(queue, task);
5086 return Vec::new();
5087 }
5088 lock(status).current.push(Current {
5089 worker: None,
5090 task: task.id.clone(),
5091 run,
5092 });
5093
5094 let quota_before = runner.state.quota.clone();
5097 runner.state.carry_dispatched_resumes(task);
5098 runner.state.next_pass_task_entry = true;
5099 let result = runner.execute().await;
5100 finish_attempt(
5101 opts.max_attempts,
5102 queue,
5103 task,
5104 &runner.state,
5105 "a_before,
5106 result,
5107 )
5108}
5109
5110pub fn finish_attempt(
5115 max_attempts: usize,
5116 queue: &Queue,
5117 task: &mut Task,
5118 state: &RunState,
5119 quota_before: &[QuotaLoss],
5120 result: Result<()>,
5121) -> Vec<QuotaLoss> {
5122 let detail = match result {
5123 Ok(()) => describe(state),
5124 Err(e) => format!("{e:#}"),
5125 };
5126 let fresh = losses_this_attempt(quota_before, &state.quota);
5127 let verdict = Verdict {
5128 status: state.status,
5129 left_pr: state.pr.is_some(),
5132 quota_hit: !fresh.is_empty(),
5138 parked: state.parked,
5142 no_viable_candidates: state.viable().is_empty(),
5145 };
5146 if let Some(replacement) = &state.replaces
5147 && let Some(pr) = &state.pr
5148 && let Ok(mut old) = RunState::load(&replacement.run)
5149 && let Some(d) = &mut old.merge_decision
5150 {
5151 d.replacement_pr = Some(pr.url.clone());
5152 let _ = old.save();
5153 }
5154 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
5155 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
5156 task.hold_unrequested_retry(Some(format!(
5157 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
5158 task.short()
5159 )));
5160 }
5161 if task.status == TaskStatus::Done {
5162 supersede_prior_runs(task, &crate::run::home());
5163 }
5164 record(queue, task);
5165 tracing::info!(
5166 "task {} is {} after run {} ({})",
5167 task.short(),
5168 task.status.as_str(),
5169 state.short(),
5170 label(state.status)
5171 );
5172 fresh
5173}
5174
5175pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
5179 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
5182 && !awaiting_resume_with(task, RunState::load)
5183 && withheld_text_wait(task).is_none()
5187 && !(task.fresh_start && task.replacement.is_some())
5188 {
5189 let why = task.last_error.clone().map_or_else(
5190 || "the run did not finish".to_owned(),
5191 |e| format!("the run did not finish: {e}"),
5192 );
5193 task.hold_unrequested_retry(Some(format!(
5194 "{why}. It was started by hand, so it is not retried \
5195 automatically; `magi task release` retries it."
5196 )));
5197 record(queue, task);
5198 }
5199}
5200
5201#[must_use]
5205pub fn loop_would_resume(run: &str) -> bool {
5206 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
5207}
5208
5209#[must_use]
5219pub fn foreign_loop(
5220 reading: Option<&Reading>,
5221 now: Timestamp,
5222 own_pid: u32,
5223) -> Option<Option<u32>> {
5224 let reading = reading.filter(|r| r.running(now))?;
5225 match reading.pid {
5226 Some(pid) if pid == own_pid => None,
5227 pid => Some(pid),
5228 }
5229}
5230
5231fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
5232 reading.worker_protocol != Some(1)
5233 && (foreign_loop(Some(reading), now, own_pid).is_some()
5234 || reading
5235 .pid
5236 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
5237}
5238
5239pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
5250 run_claimed_until(opts, queue, task, &Stop::new()).await;
5251}
5252
5253async fn run_claimed_until(
5254 opts: &Opts,
5255 queue: &Queue,
5256 task: &mut Task,
5257 stop: &Stop,
5258) -> Vec<QuotaLoss> {
5259 let status = Arc::new(Mutex::new(Status::new()));
5260 let mut quota = Box::pin(attempt(
5263 opts,
5264 queue,
5265 &status,
5266 stop,
5267 crate::graph::Pause::new(),
5268 task,
5269 ))
5270 .await;
5271 let mut resumes = 0;
5280 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
5281 let Some(timeout) = withheld_text_wait(task) else {
5282 break;
5283 };
5284 eprintln!(
5285 "waiting for the owner's answer on a withheld pull request text \
5286 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
5287 );
5288 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
5291 let ready = loop {
5292 match land_resume_state(task) {
5293 LandResume::StillWaiting
5294 if !stop.parking() && std::time::Instant::now() < limit =>
5295 {
5296 tokio::time::sleep(WITHHELD_POLL).await;
5297 }
5298 LandResume::Ready => break true,
5299 _ => break false,
5300 }
5301 };
5302 if !ready || stop.parking() {
5303 break;
5304 }
5305 resumes += 1;
5306 quota.extend(
5307 Box::pin(attempt(
5308 opts,
5309 queue,
5310 &status,
5311 stop,
5312 crate::graph::Pause::new(),
5313 task,
5314 ))
5315 .await,
5316 );
5317 }
5318 hold_if_runnable(queue, task);
5319 quota
5320}
5321
5322const MAX_WITHHELD_RESUMES: u32 = 3;
5325
5326const WITHHELD_POLL: Duration = Duration::from_secs(15);
5328
5329fn withheld_text_wait(task: &Task) -> Option<u64> {
5332 let id = task.execution_runs().last()?;
5333 let s = RunState::load(id).ok()?;
5334 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
5335 .then_some(s.config.graph.answer_timeout)
5336}
5337
5338fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
5348 after
5349 .iter()
5350 .filter(|q| !before.contains(q))
5351 .cloned()
5352 .collect()
5353}
5354
5355fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
5358 let quota: Vec<_> = quota
5359 .iter()
5360 .filter(|q| q.cause == crate::agent::QuotaCause::UsageLimit)
5361 .collect();
5362 if quota.is_empty() {
5363 return None;
5364 }
5365 let with_hint = quota.iter().find(|q| q.reset.is_some());
5366 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
5367 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
5368 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
5369 Some(
5370 now.checked_add(jiff::SignedDuration::from_secs(secs))
5371 .unwrap_or(Timestamp::MAX),
5372 )
5373}
5374
5375fn implementers_match(task: &Task, state: &RunState) -> bool {
5377 task.replacement.as_ref().map_or_else(
5378 || {
5379 task.implementers()
5380 .is_none_or(|n| n == state.config.graph.implementers)
5381 },
5382 |replacement| {
5383 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
5384 && state.config.graph.implementers == 1
5385 },
5386 )
5387}
5388
5389fn starter_with_implementers(
5391 task: &Task,
5392 previous: Option<&RunState>,
5393 proposed: Starter,
5394) -> Starter {
5395 if previous.is_some_and(|s| !implementers_match(task, s)) {
5396 task.review_of
5397 .as_ref()
5398 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
5399 } else {
5400 proposed
5401 }
5402}
5403
5404fn apply_task_implementers(config: &mut Config, task: &Task) {
5406 if let Some(n) = task.implementers() {
5407 config.graph.implementers = n;
5408 }
5409}
5410
5411#[cfg(test)]
5416fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5417 let explicit = match &task.overrides {
5418 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5419 _ => opts.config.as_deref(),
5420 };
5421 let (mut config, _) = Config::discover(repo, explicit)?;
5422 apply_options(&mut config, opts, task)?;
5423 Ok(config)
5424}
5425
5426fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5427 if task
5428 .overrides
5429 .as_ref()
5430 .is_none_or(|o| o.inherit_daemon_options)
5431 && let Some(mode) = opts.merge.as_deref()
5432 {
5433 config.merge.mode = merge_mode(mode)?;
5434 }
5435 if let Some(o) = &task.overrides {
5436 o.apply(config);
5437 }
5438 Ok(())
5439}
5440
5441fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5443 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5444 if let Some(mode) = &opts.merge {
5445 config.merge.mode = merge_mode(mode)?;
5446 }
5447 Ok(config)
5448}
5449
5450async fn maybe_prune_cache_between_runs(
5455 repo: &Path,
5456 opts: &Opts,
5457 home: &Path,
5458 stop: &Stop,
5459 last_checked: &mut Option<Timestamp>,
5460 now: Timestamp,
5461) {
5462 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5463 return;
5464 }
5465 *last_checked = Some(now);
5466 let cfg = match prepare(repo, opts) {
5467 Ok(cfg) => cfg,
5468 Err(e) => {
5469 tracing::warn!("cache check: no config: {e:#}");
5470 return;
5471 }
5472 };
5473 match clean::prune_cache_if_over_limit(&cfg, home) {
5474 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5475 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5476 pruned.files,
5477 pruned.freed
5478 ),
5479 Ok(_) => {}
5480 Err(e) => {
5481 tracing::warn!("housekeep: prune cache: {e:#}");
5482 notices::raise_in(
5483 home,
5484 Notice::warn(
5485 "housekeep:cache",
5486 format!(
5487 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5488 e.to_string()
5489 .lines()
5490 .next()
5491 .unwrap_or("unknown cleanup error")
5492 ),
5493 ),
5494 );
5495 }
5496 }
5497}
5498
5499fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5503 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5504}
5505
5506async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5525 let cfg = match prepare(repo, opts) {
5526 Ok(cfg) => cfg,
5527 Err(e) => {
5528 tracing::warn!("housekeep: no config: {e:#}");
5529 return;
5530 }
5531 };
5532 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5541 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5542 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5547 let mut extra = Vec::new();
5548 if out.unreadable > 0 {
5549 extra.push(format!("{} unreadable", out.unreadable));
5550 }
5551 if out.orphaned_worktrees > 0 {
5552 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5553 }
5554 let detail = if extra.is_empty() {
5555 String::new()
5556 } else {
5557 format!(" ({})", extra.join(", "))
5558 };
5559 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5560 }
5561 if out.external_merges_recorded > 0 {
5562 tracing::info!(
5563 "housekeep: recorded {} run(s) as merged externally",
5564 out.external_merges_recorded
5565 );
5566 }
5567 if out.stale_pr_states_repaired > 0 {
5568 tracing::info!(
5569 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5570 out.stale_pr_states_repaired
5571 );
5572 }
5573 if out.cache_files > 0 {
5574 tracing::info!(
5575 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5576 out.cache_files,
5577 out.cache_freed
5578 );
5579 }
5580 if out.questions_abandoned > 0 {
5581 tracing::info!(
5582 "housekeep: abandoned {} question(s) left open by a finished run",
5583 out.questions_abandoned
5584 );
5585 }
5586}
5587
5588async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5597 let questions = Questions::at(home.join("questions"));
5598 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5599 if report.is_empty() {
5600 return;
5601 }
5602 if !report.quarantined.is_empty() {
5603 tracing::info!(
5604 "triage: held {} blocked task(s) whose blocked-on task or \
5605 question no longer exists: {}",
5606 report.quarantined.len(),
5607 report.quarantined.join(", ")
5608 );
5609 }
5610 if !report.resumed.is_empty() {
5611 tracing::info!(
5612 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5613 report.resumed.len(),
5614 report.resumed.join(", ")
5615 );
5616 }
5617 if !report.asked.is_empty() {
5618 tracing::info!(
5619 "triage: asked about {} held task(s): {}",
5620 report.asked.len(),
5621 report.asked.join(", ")
5622 );
5623 }
5624 if !report.answered.is_empty() {
5625 tracing::info!(
5626 "triage: applied {} operator answer(s): {}",
5627 report.answered.len(),
5628 report.answered.join(", ")
5629 );
5630 }
5631}
5632
5633pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5640 disk_gate_with(repo, config, crate::disk::free_bytes)
5641}
5642
5643fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5647 repo: &Path,
5648 config: &Config,
5649 free_bytes: F,
5650) -> Option<String> {
5651 let min = config.disk.min_free_bytes;
5652 if min == 0 {
5653 return None;
5654 }
5655 let mut paths = vec![repo.to_owned()];
5656 if let Some(cache) = config.cache_dir() {
5657 paths.push(cache);
5658 }
5659 if let Some(root) = &config.graph.worktree_root {
5660 paths.push(root.clone());
5661 }
5662 for path in paths {
5663 let probe = if path == repo {
5664 path.as_path()
5665 } else {
5666 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5667 };
5668 let reason = match free_bytes(probe) {
5669 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5670 Err(e) => Some(crate::disk::unmeasured_in(
5671 probe,
5672 &e,
5673 &config.graph.language,
5674 )),
5675 };
5676 if reason.is_some() {
5677 return reason;
5678 }
5679 }
5680 None
5681}
5682
5683const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5690
5691const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5695
5696fn quota_wait(
5705 reset_at: Option<Timestamp>,
5706 now: Timestamp,
5707 fallback: Duration,
5708 cap: Duration,
5709) -> Duration {
5710 match reset_at {
5711 Some(at) if at > now => {
5712 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5713 Duration::from_secs(secs).min(cap)
5714 }
5715 _ => fallback,
5716 }
5717}
5718
5719fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5733 parse_reset_hint_zoned(text, now)
5734 .or_else(|| parse_reset_hint_dated(text))
5735 .or_else(|| parse_reset_hint_relative(text, recorded))
5736}
5737
5738fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5742 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5743 let mut rest = rest.trim();
5744 if rest.is_empty() {
5745 return None;
5746 }
5747 let mut total: i64 = 0;
5748 let mut matched = false;
5749 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5750 if let Some((digits, tail)) = rest.split_once(unit)
5751 && !digits.is_empty()
5752 && digits.bytes().all(|b| b.is_ascii_digit())
5753 {
5754 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5755 rest = tail;
5756 matched = true;
5757 }
5758 }
5759 if !rest.is_empty() || !matched {
5760 return None;
5761 }
5762 recorded
5763 .checked_add(jiff::SignedDuration::from_secs(total))
5764 .ok()
5765}
5766
5767fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5771 let clock = clock.trim().to_lowercase();
5772 let (digits, pm) = clock
5773 .strip_suffix("am")
5774 .map(|d| (d, false))
5775 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5776 let (h, m) = digits.trim().split_once(':')?;
5777 let mut hour: i8 = h.trim().parse().ok()?;
5778 let minute: i8 = m.trim().parse().ok()?;
5779 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5780 return None;
5781 }
5782 if pm && hour != 12 {
5783 hour += 12;
5784 } else if !pm && hour == 12 {
5785 hour = 0;
5786 }
5787 Some((hour, minute))
5788}
5789
5790fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5795 let open = text.find('(')?;
5796 let close = text.rfind(')')?;
5797 if close <= open {
5798 return None;
5799 }
5800 let zone = text[open + 1..close].trim();
5801 let (hour, minute) = parse_12h_clock(&text[..open])?;
5802 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5803 let candidate = now
5804 .to_zoned(tz)
5805 .with()
5806 .hour(hour)
5807 .minute(minute)
5808 .second(0)
5809 .millisecond(0)
5810 .microsecond(0)
5811 .nanosecond(0)
5812 .build()
5813 .ok()?;
5814 let mut at = candidate.timestamp();
5815 if at <= now {
5816 at += jiff::SignedDuration::from_hours(24);
5817 }
5818 Some(at)
5819}
5820
5821fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5830 let words: Vec<&str> = text.split_whitespace().collect();
5831 if words.len() < 5 {
5832 return None;
5833 }
5834 (0..=words.len() - 5)
5835 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5836}
5837
5838fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5844 if trailing.is_some_and(|next| next.starts_with('(')) {
5845 return None;
5846 }
5847 let month = month_number(window[0])?;
5848 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5849 let day_digits = ["st", "nd", "rd", "th"]
5850 .iter()
5851 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5852 let day: i8 = day_digits.parse().ok()?;
5853 let year_token = window[2];
5854 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5855 return None;
5856 }
5857 let year: i16 = year_token.parse().ok()?;
5858 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5862 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5863 let date = jiff::civil::Date::new(year, month, day).ok()?;
5864 let candidate = date
5865 .at(hour, minute, 0, 0)
5866 .to_zoned(jiff::tz::TimeZone::UTC)
5867 .ok()?;
5868 Some(candidate.timestamp())
5869}
5870
5871fn month_number(name: &str) -> Option<i8> {
5874 const NAMES: [&str; 12] = [
5875 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5876 ];
5877 let lower = name.to_lowercase();
5878 NAMES
5879 .iter()
5880 .position(|n| *n == lower.as_str())
5881 .map(|i| i as i8 + 1)
5882}
5883
5884#[cfg(test)]
5886fn exhausted_review_budget(state: &RunState) -> bool {
5887 !crate::handover::can_resume(state)
5890 && state.status == RunStatus::Blocked
5891 && !state.released()
5892 && state.retired_to.is_none()
5893}
5894
5895fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5932 unfinished_run_with(runs, short, RunState::load)
5933}
5934
5935pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5938where
5939 F: FnMut(&str) -> Result<RunState>,
5940{
5941 let mut id = runs.last()?.clone();
5942 let mut visited = std::collections::BTreeSet::new();
5943 let mut repo = None;
5944 loop {
5945 if !runs.contains(&id) || !visited.insert(id.clone()) {
5946 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5947 return None;
5948 }
5949 let state = match load(&id) {
5950 Ok(state) if state.id == id => state,
5951 Ok(_) => return None,
5952 Err(e) => {
5953 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5954 return None;
5955 }
5956 };
5957 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5958 return None;
5959 }
5960 repo = Some(state.repo.clone());
5961 if let Some(authority) = state.retired_to {
5962 id = authority;
5963 continue;
5964 }
5965 return (crate::handover::can_resume(&state)
5966 && state.liveness(false) != crate::run::Liveness::Live)
5967 .then_some(id);
5968 }
5969}
5970
5971fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5979where
5980 F: FnOnce(&str) -> Result<RunState>,
5981{
5982 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5984 || task.fresh_start
5985 || task.review_branch.is_some()
5986 {
5987 return false;
5988 }
5989 let Some(id) = task.execution_runs().last() else {
5990 return false;
5991 };
5992 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5993}
5994
5995#[derive(Debug, Clone, PartialEq, Eq)]
5998enum Starter {
5999 Review(String),
6002 Resume(String),
6004 Start,
6006}
6007
6008fn retained_attempt_hold(
6013 task: &Task,
6014 starter: &Starter,
6015 limit: usize,
6016 mut load: impl FnMut(&str) -> Result<RunState>,
6017) -> Option<String> {
6018 if limit == 0 || matches!(starter, Starter::Resume(_)) {
6019 return None;
6020 }
6021 let mut retained = 0;
6022 for id in task
6023 .runs
6024 .iter()
6025 .chain(task.startup_failures.iter().map(|failure| &failure.run))
6026 .collect::<std::collections::BTreeSet<_>>()
6027 {
6028 let state = match load(id) {
6029 Ok(state) if state.id == *id => state,
6030 _ => {
6031 return Some(format!(
6032 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
6033 ));
6034 }
6035 };
6036 if state.retired_to.is_some() || state.released() {
6037 continue;
6038 }
6039 retained += 1;
6042 }
6043 (retained >= limit).then(|| format!(
6044 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
6045 ))
6046}
6047
6048fn selected_starter(
6050 selected: &crate::pr_recovery::Selection,
6051 previous: Option<&RunState>,
6052 unfinished: Option<&str>,
6053) -> Starter {
6054 if let Some(id) = unfinished
6055 && previous.is_some_and(|r| {
6056 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
6057 })
6058 {
6059 Starter::Resume(id.to_owned())
6060 } else {
6061 Starter::Review(selected.branch.clone())
6062 }
6063}
6064
6065fn take_divergence_answer(
6069 branch: &str,
6070 remote: &str,
6071 task: &mut Task,
6072) -> Option<crate::reconcile::Choice> {
6073 let summary = crate::reconcile::summary_for(branch, remote);
6074 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
6075 (a.question == summary)
6076 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
6077 .flatten()
6078 .map(|c| (i, c))
6079 })?;
6080 task.answers.remove(idx);
6081 Some(choice)
6082}
6083
6084fn choose_starter(
6096 review_branch: Option<&str>,
6097 branch_exists: bool,
6098 unfinished: Option<&str>,
6099) -> Starter {
6100 match review_branch {
6101 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
6102 Some(_) => Starter::Start,
6103 None => match unfinished {
6104 Some(id) => Starter::Resume(id.to_owned()),
6105 None => Starter::Start,
6106 },
6107 }
6108}
6109
6110fn review_retry_starter(
6114 task: &Task,
6115 previous: Option<&RunState>,
6116 branch: Option<&str>,
6117 exists: bool,
6118 unfinished: Option<&str>,
6119) -> Starter {
6120 if let Some(run) = previous
6121 && unfinished == Some(run.id.as_str())
6122 && run.reviewed_commits.is_some()
6123 && branch.is_some_and(|branch| {
6124 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
6125 })
6126 && may_resume_run(run, task)
6127 {
6128 return Starter::Resume(run.id.clone());
6129 }
6130 match (branch, unfinished, &task.review_of) {
6131 (None, None, Some(branch)) => Starter::Review(branch.clone()),
6132 _ => choose_starter(branch, exists, unfinished),
6133 }
6134}
6135
6136fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
6138 let branch = task.review_branch.take();
6139 let instruction = task.review_instruction.take();
6140 branch.filter(|branch| match &instruction {
6141 Some(authored) => *authored == task.instruction,
6142 None => previous.map_or_else(
6145 || task.review_of.as_ref() == Some(branch),
6146 |run| compatible_run_instruction(run, task),
6147 ),
6148 })
6149}
6150
6151fn may_resume_run(run: &RunState, task: &Task) -> bool {
6154 if !task.assignment_accepts(run) {
6155 return false;
6156 }
6157 task.resume_override
6158 .as_ref()
6159 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
6160 || compatible_run_instruction(run, task)
6161}
6162
6163pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
6165 if let Some(authored) = &run.task_instruction {
6166 return compatible_instruction(authored, task);
6167 }
6168 if compatible_instruction(&run.instruction, task) {
6169 return true;
6170 }
6171 if run.reviewed_commits.is_some() {
6174 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
6175 return strip_answers_block(authored, task).trim() == task.instruction.trim();
6176 }
6177 if let Some(branch) = &task.review_of {
6178 let canonical = format!(
6179 "Review the branch `{branch}` (review-only: no implementation, no judging)."
6180 );
6181 let generated = format!(
6182 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
6183 );
6184 return task.instruction == canonical && run.instruction.starts_with(&generated);
6185 }
6186 }
6187 false
6188}
6189
6190pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
6192 strip_answers_block(old, task) == task.instruction
6193}
6194
6195fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
6198 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
6199 return fallback.to_path_buf();
6200 }
6201 task.repo.clone()
6202}
6203
6204const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
6208
6209fn answers_block(task: &Task, count: usize) -> String {
6211 let mut s = ANSWERS_HEADER.to_owned();
6212 for a in &task.answers[..count] {
6213 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
6214 }
6215 s
6216}
6217
6218fn resumption_block(r: &crate::queue::Resumption) -> String {
6221 format!(
6222 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
6223 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
6224 r.generation, r.at, r.instruction, r.answers_through,
6225 )
6226}
6227
6228fn append_answers(base: &str, task: &Task) -> String {
6230 let mut s = base.to_owned();
6231 if !task.answers.is_empty() {
6232 s.push_str(&answers_block(task, task.answers.len()));
6233 }
6234 if let Some(r) = &task.resumption {
6235 s.push_str(&resumption_block(r));
6236 }
6237 s
6238}
6239
6240fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
6244 let mut instruction = instruction;
6245 for approval in task
6246 .resumption
6247 .iter()
6248 .chain(task.resumption_history.iter().rev())
6249 {
6250 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
6251 instruction = base;
6252 break;
6253 }
6254 }
6255 for count in (1..=task.answers.len()).rev() {
6256 let block = answers_block(task, count);
6257 if let Some(base) = instruction.strip_suffix(&block) {
6258 return base;
6259 }
6260 }
6261 instruction
6262}
6263
6264fn instruction_for(task: &Task) -> String {
6272 append_answers(&task.instruction, task)
6273}
6274
6275fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
6287 append_answers(strip_answers_block(old_instruction, task), task)
6288}
6289
6290fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
6292 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
6293 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
6294 if let Some(cause) = task.hold_notice_cause.as_mut() {
6295 cause.target = failure.map(|f| f.target.clone());
6296 }
6297}
6298
6299#[derive(Debug, thiserror::Error)]
6300#[error("attachment `{name}` is recorded on the task but {path} is missing")]
6301struct MissingAttachment {
6302 name: String,
6303 path: PathBuf,
6304}
6305
6306fn task_attachments(
6308 queue: &Queue,
6309 task: &Task,
6310) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
6311 let paths = queue.attachment_paths(task);
6312 for (name, path) in task.attachments.iter().zip(&paths) {
6313 if !path.is_file() {
6314 return Err(MissingAttachment {
6315 name: name.clone(),
6316 path: path.clone(),
6317 });
6318 }
6319 }
6320 Ok(paths)
6321}
6322
6323fn prepare_instruction(
6334 starter: &Starter,
6335 old_instruction: Option<&str>,
6336 task: &Task,
6337) -> Option<String> {
6338 match starter {
6339 Starter::Start => Some(instruction_for(task)),
6340 Starter::Resume(_) => Some(resumed_instruction(
6341 old_instruction.expect("a resumed run always has a prior instruction"),
6342 task,
6343 )),
6344 Starter::Review(_) => None,
6345 }
6346}
6347
6348fn record(queue: &Queue, task: &mut Task) {
6352 if let Err(e) = queue.put(task) {
6353 tracing::error!("could not record task {}: {e:#}", task.short());
6354 let notice = Notice::error(
6355 "loop:record",
6356 format!(
6357 "Task {} state could not be saved: {}. Check the disk.",
6358 task.id,
6359 e.to_string().lines().next().unwrap_or("unknown save error")
6360 ),
6361 )
6362 .link(Link::Task {
6363 id: task.id.clone(),
6364 });
6365 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
6367 Some(home) => notices::raise_in(home, notice),
6368 None => notices::raise(notice),
6369 }
6370 }
6371}
6372
6373fn runnable(queue: &Queue) -> Vec<Task> {
6379 let mut tasks: Vec<Task> = queue
6380 .list()
6381 .into_iter()
6382 .filter(|t| t.status.runnable())
6383 .collect();
6384 tasks.sort_unstable_by(crate::queue::dispatch_order);
6385 tasks
6386}
6387
6388fn describe(state: &RunState) -> String {
6402 let p = phrases(&state.config.graph.language);
6403 let mut detail = if state.status == RunStatus::Stalled {
6404 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
6405 seats.sort_unstable();
6406 seats.dedup();
6407 if seats.is_empty() {
6408 p.quorum_lost.to_owned()
6409 } else {
6410 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
6411 }
6412 } else {
6413 format!("{}{}", p.run_ended, state.status.display_label())
6414 };
6415 if let Some(cause) = state.verification_blocking_cause() {
6416 detail.push_str(&format!(" ({cause})"));
6417 } else if let Some(last) = state.events.last() {
6418 let unanswered = state
6422 .reviews
6423 .last()
6424 .filter(|r| {
6425 state.status == RunStatus::Blocked
6426 && last.node == "review"
6427 && r.incomplete()
6428 && r.blocking == 0
6429 && r.round == state.config.graph.review_rounds
6430 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6431 })
6432 .map(|r| (r.expected - r.answered, r.round));
6433 match unanswered {
6434 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6435 detail.push_str(&format!(
6436 " ({}: {})",
6437 last.node,
6438 (p.reviewers_never_answered)(missing, rounds)
6439 ));
6440 }
6441 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6442 }
6443 }
6444 detail.push_str(&format!(" [run {}]", state.id));
6445 detail
6446}
6447
6448const DIAGNOSTIC_MAX: usize = 4_000;
6454
6455const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6460
6461fn diagnostic(state: &RunState) -> Option<String> {
6475 let mut parts: Vec<String> = Vec::new();
6476
6477 for o in state.gate.iter().filter(|o| !o.ok()) {
6479 parts.push(format!(
6480 "gate `{}` failed ({}):\n{}",
6481 o.command,
6482 notices::exit_status(o.code),
6483 crate::run::summarize_verify(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6484 ));
6485 }
6486
6487 if let Some(last) = state
6490 .events
6491 .iter()
6492 .rev()
6493 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6494 {
6495 parts.push(last.message.clone());
6496 }
6497
6498 if state.viable().is_empty() {
6505 for c in &state.candidates {
6506 if let Some(evidence) = &c.verified_noop {
6507 parts.push(format!(
6508 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6509 c.label
6510 ));
6511 } else if !c.summary.trim().is_empty() {
6512 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6513 } else if let Some(why) = &c.failed {
6514 parts.push(format!("candidate {}: {why}", c.label));
6515 }
6516 }
6517 }
6518
6519 if parts.is_empty() {
6520 return None;
6521 }
6522 Some(crate::run::tail(
6527 &parts.join("\n\n"),
6528 DIAGNOSTIC_MAX.saturating_sub(100),
6529 ))
6530}
6531
6532fn label(status: RunStatus) -> &'static str {
6540 status.as_str()
6541}
6542
6543pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6545 match mode {
6546 "none" => Ok(MergeMode::None),
6547 "local" => Ok(MergeMode::Local),
6548 "pr" => Ok(MergeMode::Pr),
6549 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6550 }
6551}
6552
6553fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6558 mutex
6559 .lock()
6560 .unwrap_or_else(std::sync::PoisonError::into_inner)
6561}
6562
6563#[cfg(test)]
6564mod tests {
6565 use super::*;
6566 use crate::queue::{Source, TaskStatus};
6567 use crate::run::{Candidate, CommandOutcome};
6568 use pretty_assertions::assert_eq;
6569
6570 fn task() -> Task {
6571 Task::new(
6572 "add retries".to_owned(),
6573 "add retries".to_owned(),
6574 PathBuf::from("/repo"),
6575 Source::Human,
6576 )
6577 }
6578
6579 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6580 let job = crate::worker::Job {
6581 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6582 sequence: 1,
6583 kind: "task".into(),
6584 key: format!("task:{}", request.task),
6585 payload: serde_json::to_value(request).unwrap(),
6586 generation: "a".repeat(64),
6587 version: None,
6588 status: crate::worker::Status::Starting,
6589 result: None,
6590 pid: None,
6591 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6592 finished_at: None,
6593 };
6594 let dir = home.join("workers/jobs").join(&job.id);
6595 std::fs::create_dir_all(&dir).unwrap();
6596 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6597 job
6598 }
6599
6600 #[test]
6601 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6602 let home = tempfile::tempdir().unwrap();
6603 let request = TaskWorker {
6604 opts: Opts::default(),
6605 queue: home.path().join("queue"),
6606 task: "startup".into(),
6607 lane: PermitKind::Ordinary,
6608 standalone: false,
6609 };
6610 let job = starting_task_reservation(home.path(), &request);
6611 let error = task_worker_spawn_error(
6612 home.path(),
6613 &request,
6614 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6615 )
6616 .expect_err("our own reservation must not turn a failed spawn into a resync");
6617 let active = crate::worker::active(home.path()).unwrap();
6618 assert_eq!(active.len(), 1);
6619 assert_eq!(active[0].id, job.id);
6620 assert_eq!(active[0].status, crate::worker::Status::Starting);
6621 assert!(
6622 crate::worker::control(home.path(), &job.key)
6623 .unwrap()
6624 .is_none()
6625 );
6626 let notices = notices::Notices::at(home.path().join("notifications")).list();
6627 assert_eq!(notices.len(), 1);
6628 assert_eq!(notices[0].key, "loop:worker-startup");
6629 assert!(!notices[0].message.contains("private-error-detail"));
6630 assert!(notices[0].message.contains(&request.task));
6631 assert!(
6632 notices[0]
6633 .message
6634 .contains("could not be started or acknowledged")
6635 );
6636 assert!(notices[0].message.contains("worker.log"));
6637 assert_eq!(
6638 notices[0].link,
6639 Some(Link::Task {
6640 id: request.task.clone()
6641 })
6642 );
6643 assert!(format!("{error:#}").contains("private-error-detail"));
6644 }
6645
6646 #[test]
6647 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6648 let home = tempfile::tempdir().unwrap();
6649 let mut request = TaskWorker {
6650 opts: Opts::default(),
6651 queue: home.path().join("queue"),
6652 task: "handoff".into(),
6653 lane: PermitKind::Ordinary,
6654 standalone: true,
6655 };
6656 let job = starting_task_reservation(home.path(), &request);
6657 request.standalone = false;
6658 task_worker_spawn_error(
6659 home.path(),
6660 &request,
6661 anyhow::anyhow!("operation key is already reserved for a different request"),
6662 )
6663 .expect("a different standalone request is the handoff race, not our startup failure");
6664 let active = crate::worker::active(home.path()).unwrap();
6665 assert_eq!(active.len(), 1);
6666 assert_eq!(active[0].id, job.id);
6667 assert_eq!(active[0].payload, job.payload);
6668 assert!(
6669 notices::Notices::at(home.path().join("notifications"))
6670 .list()
6671 .is_empty()
6672 );
6673 }
6674
6675 #[test]
6676 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6677 let home = tempfile::tempdir().unwrap();
6678 let queue = Queue::at(home.path().join("queue"));
6679 let mut task = task();
6680 task.runs = vec!["previous".into()];
6681 queue.put(&mut task).unwrap();
6682 let request = TaskWorker {
6683 opts: Opts::default(),
6684 queue: home.path().join("queue"),
6685 task: task.id.clone(),
6686 lane: PermitKind::Ordinary,
6687 standalone: false,
6688 };
6689 let job = starting_task_reservation(home.path(), &request);
6690 let claims = current_work(home.path(), Timestamp::now());
6691 assert_eq!(claims.len(), 1);
6692 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6693 assert_eq!(claims[0].run, "previous");
6694 assert!(other_claims_run(&claims, "previous", None));
6695 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6698 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6699 }
6700
6701 #[test]
6702 fn admission_excludes_only_the_callers_exact_reservation() {
6703 let own = Current {
6704 worker: Some("own".into()),
6705 task: "same-task".into(),
6706 run: "previous".into(),
6707 };
6708 assert!(!other_claims_run(
6709 std::slice::from_ref(&own),
6710 "previous",
6711 Some("own")
6712 ));
6713 assert!(other_claims_run(
6714 std::slice::from_ref(&own),
6715 "previous",
6716 None
6717 ));
6718 for worker in [None, Some("other".into())] {
6719 let other = Current {
6720 worker,
6721 ..own.clone()
6722 };
6723 assert!(other_claims_run(
6724 &[own.clone(), other],
6725 "previous",
6726 Some("own")
6727 ));
6728 }
6729 let legacy: Current =
6730 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6731 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6732 }
6733
6734 #[test]
6735 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6736 let mut task = task();
6737 let mut run = run_state(RunStatus::Reviewing);
6738 run.task_instruction = Some(task.instruction.clone());
6739 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6740 let branch = "magi/test/B";
6741 run.candidates.push(crate::run::Candidate {
6742 branch: branch.into(),
6743 ..candidate('B', "preserved work", false, None)
6744 });
6745 task.runs = vec![run.id.clone()];
6746 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6747 assert_eq!(
6748 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6749 Starter::Resume(run.id.clone())
6750 );
6751 assert!(matches!(
6752 review_retry_starter(
6753 &task,
6754 Some(&run),
6755 Some("different"),
6756 true,
6757 unfinished.as_deref()
6758 ),
6759 Starter::Review(_)
6760 ));
6761 assert_eq!(
6762 review_retry_starter(
6763 &task,
6764 Some(&run),
6765 Some(branch),
6766 false,
6767 unfinished.as_deref()
6768 ),
6769 Starter::Start
6770 );
6771 task.instruction.push_str(" changed criteria");
6772 assert!(matches!(
6773 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6774 Starter::Review(_)
6775 ));
6776 assert!(matches!(
6777 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6778 Starter::Review(_)
6779 ));
6780 }
6781
6782 #[test]
6783 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6784 let dir = tempfile::tempdir().unwrap();
6785 let queue = Queue::at(dir.path().join("queue"));
6786 let mut task = task();
6787 task.status = TaskStatus::Running;
6788 task.attempts = 1;
6789 queue.put(&mut task).unwrap();
6790 let live = std::collections::BTreeSet::from([task.id.clone()]);
6791 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6792 drop(queue.claim(&task.id).unwrap());
6793 let saved = queue.get(&task.id).unwrap();
6794 assert_eq!(saved.status, TaskStatus::Running);
6795 assert_eq!(saved.attempts, 1);
6796 assert!(saved.last_error.is_none());
6797 }
6798
6799 #[test]
6800 fn standalone_workers_do_not_consume_scheduler_lanes() {
6801 let request = |task: &str, lane, standalone| TaskWorker {
6802 opts: Opts::default(),
6803 queue: PathBuf::from("/queue"),
6804 task: task.to_owned(),
6805 lane,
6806 standalone,
6807 };
6808 let mut jobs = vec![
6809 request("ordinary", PermitKind::Ordinary, false),
6810 request("standalone-urgent", PermitKind::Urgent, true),
6811 request("standalone-ordinary", PermitKind::Ordinary, true),
6812 request("decision", PermitKind::None, false),
6813 ];
6814 assert_eq!(available_slots(&jobs, 2), (1, 1));
6815 jobs.push(request("urgent", PermitKind::Urgent, false));
6816 assert_eq!(available_slots(&jobs, 2), (1, 0));
6817 assert_eq!(available_slots(&jobs, 0), (0, 0));
6818 }
6819
6820 #[test]
6821 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6822 let now = Timestamp::now();
6823 let mut reading = Reading {
6824 pid: Some(std::process::id()),
6825 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6826 ..Reading::default()
6827 };
6828 assert!(!reading.running(now));
6829 assert!(legacy_scheduler_running(&reading, now, 0));
6830 reading.worker_protocol = Some(1);
6831 assert!(!legacy_scheduler_running(&reading, now, 0));
6832 }
6833
6834 #[test]
6835 fn selected_public_head_overrides_historical_resume_and_pin() {
6836 let mut t = task();
6837 let mut old = run_state(RunStatus::Blocked);
6838 let selection = crate::pr_recovery::Selection {
6839 url: "url".into(),
6840 branch: format!("magi/task-{}", t.id),
6841 head: "a".repeat(40),
6842 source_head: None,
6843 source_run: None,
6844 source_proof: None,
6845 disposition: Some(crate::pr_recovery::Disposition::Review),
6846 instruction: t.instruction.clone(),
6847 approved_at: Timestamp::now(),
6848 };
6849 t.resume_override = Some(crate::queue::OperatorResume {
6850 pinned_run: Some(old.id.clone()),
6851 question_id: "choice".into(),
6852 at: Timestamp::now(),
6853 conductor_rehold: None,
6854 forced: false,
6855 });
6856 assert_eq!(
6857 selected_starter(&selection, Some(&old), Some(&old.id)),
6858 Starter::Review(selection.branch.clone())
6859 );
6860 old.pr_recovery = Some(selection.clone());
6861 assert_eq!(
6862 selected_starter(&selection, Some(&old), Some(&old.id)),
6863 Starter::Review(selection.branch.clone())
6864 );
6865 old.reviewed_commits = Some(vec!["implementation".into()]);
6866 assert_eq!(
6867 selected_starter(&selection, Some(&old), Some(&old.id)),
6868 Starter::Resume(old.id.clone())
6869 );
6870 let mut changed = selection.clone();
6871 changed.head = "b".repeat(40);
6872 assert_eq!(
6873 selected_starter(&changed, Some(&old), Some(&old.id)),
6874 Starter::Review(changed.branch)
6875 );
6876 }
6877
6878 #[test]
6879 fn candidate_transplant_failure_is_one_actionable_hold() {
6880 let dir = tempfile::tempdir().unwrap();
6881 let q = Queue::at(dir.path().join("queue"));
6882 let mut t = task();
6883 let mut state = run_state(RunStatus::Blocked);
6884 state.event("prep", "cannot apply referenced commit: conflict");
6885 state.prep_failure = Some("conflict".into());
6886 t.start(state.id.clone());
6887 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6888 assert_eq!(t.attempts, 1);
6889 assert_eq!(t.status, TaskStatus::Held);
6890 assert!(t.operator_held());
6891 assert!(!t.status.runnable());
6892 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6893 let first = t.chat_notice_key().unwrap();
6894 t.chat_notices.push(first.clone());
6895 t.release_operator("test", None);
6896 state.id = "another-unprepared-run".into();
6897 t.start(state.id.clone());
6898 finish_attempt(
6899 3,
6900 &q,
6901 &mut t,
6902 &state,
6903 &[],
6904 Err(anyhow::anyhow!("still conflicts")),
6905 );
6906 assert_eq!(
6907 t.chat_notice_key().as_ref(),
6908 Some(&first),
6909 "no work happened"
6910 );
6911 t.release_operator("test", None);
6912 state.prep_failure = None;
6913 state.status = RunStatus::Failed;
6914 state.begin_pass();
6915 t.start(state.id.clone());
6916 finish_attempt(
6917 1,
6918 &q,
6919 &mut t,
6920 &state,
6921 &[],
6922 Err(anyhow::anyhow!("new gate failure")),
6923 );
6924 let failure = t.chat_notice_key().unwrap();
6925 assert_ne!(failure, first, "approved recovery reached a real failure");
6926 t.release_operator("test", None);
6927 state.begin_pass();
6928 t.start(state.id.clone());
6929 finish_attempt(
6930 1,
6931 &q,
6932 &mut t,
6933 &state,
6934 &[],
6935 Err(anyhow::anyhow!("new failure on resume")),
6936 );
6937 assert_ne!(
6938 t.chat_notice_key().as_ref(),
6939 Some(&failure),
6940 "a new settled pass"
6941 );
6942 }
6943
6944 #[test]
6945 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6946 for source in ["cli", "web"] {
6947 let mut t = task();
6948 let mut run = run_state(RunStatus::Blocked);
6949 run.instruction = "Run A original criteria".into();
6950 run.task_instruction = Some(run.instruction.clone());
6951 t.start(run.id.clone());
6952 t.release_operator("choice", Some("resume-question"));
6953 t.resume_override = Some(crate::queue::OperatorResume {
6954 question_id: "resume-question".into(),
6955 at: Timestamp::now(),
6956 conductor_rehold: None,
6957 forced: true,
6958 pinned_run: Some(run.id.clone()),
6959 });
6960 t.hold_machine(Some("not enough free space to start a run".into()));
6961 let mut unchanged = t.clone();
6962 unchanged
6963 .edit("title changed".into(), unchanged.instruction.clone())
6964 .unwrap();
6965 unchanged.release_operator(source, None);
6966 assert_eq!(
6967 unchanged
6968 .resume_override
6969 .as_ref()
6970 .unwrap()
6971 .pinned_run
6972 .as_deref(),
6973 Some(run.id.as_str())
6974 );
6975 assert!(!compatible_run_instruction(&run, &unchanged));
6976 assert!(may_resume_run(&run, &unchanged));
6977
6978 t.edit(
6979 "approved recovery".into(),
6980 "Reuse measured PR head B; fix Clippy; review".into(),
6981 )
6982 .unwrap();
6983 assert!(t.resume_override.is_none());
6984 t.release_operator(source, None);
6985 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6986 assert!(reloaded.resume_override.is_none());
6987 assert!(!may_resume_run(&run, &reloaded));
6988 let unfinished =
6989 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6990 .filter(|_| may_resume_run(&run, &reloaded));
6991 assert_eq!(
6992 choose_starter(None, false, unfinished.as_deref()),
6993 Starter::Start
6994 );
6995 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6996 assert_eq!(
6997 reloaded.resumption_history[0].question_id.as_deref(),
6998 Some("resume-question")
6999 );
7000
7001 unchanged.instruction = "Reuse measured PR head B".into();
7004 unchanged.hold_machine(Some("disk hold".into()));
7005 unchanged.release_operator(source, None);
7006 assert!(unchanged.resume_override.is_none());
7007 assert!(!may_resume_run(&run, &unchanged));
7008 }
7009 }
7010
7011 #[test]
7012 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
7013 let mut t = task();
7014 let mut run = run_state(RunStatus::Blocked);
7015 t.review_of = Some("approved-head".into());
7016 t.instruction =
7017 "Review the branch `approved-head` (review-only: no implementation, no judging)."
7018 .into();
7019 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
7020 run.reviewed_commits = Some(vec!["Commit log".into()]);
7021 t.start(run.id.clone());
7022 t.hold_machine(Some("interrupted review".into()));
7023 t.release_operator("cli", None);
7024 assert!(
7025 compatible_run_instruction(&run, &t),
7026 "legacy explicit review remains resumable"
7027 );
7028 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
7029 .filter(|_| compatible_run_instruction(&run, &t));
7030 assert_eq!(
7031 choose_starter(None, false, unfinished.as_deref()),
7032 Starter::Resume(run.id.clone())
7033 );
7034 run.task_instruction = Some(t.instruction.clone());
7035 assert!(compatible_run_instruction(&run, &t));
7036 let reloaded: RunState =
7037 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
7038 assert!(compatible_run_instruction(&reloaded, &t));
7039 t.edit(
7040 "new approved head".into(),
7041 "Reuse measured PR head B".into(),
7042 )
7043 .unwrap();
7044 assert!(!compatible_run_instruction(&run, &t));
7045
7046 t.review_of = None;
7048 t.instruction = "Fix Clippy on approved head".into();
7049 run.task_instruction = None;
7050 run.instruction = format!(
7051 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
7052 t.instruction
7053 );
7054 assert!(compatible_run_instruction(&run, &t));
7055 t.instruction.push('\n');
7056 assert!(
7057 compatible_run_instruction(&run, &t),
7058 "legacy generated review trimmed the original criteria"
7059 );
7060 t.hold_for_handover(Some("approved-head".into()), "refused".into());
7061 let mut unchanged = t.clone();
7062 assert_eq!(
7063 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
7064 Some("approved-head")
7065 );
7066 t.edit("new plan".into(), "Reuse measured PR head B".into())
7067 .unwrap();
7068 assert!(take_review_branch(&mut t, Some(&run)).is_none());
7069 assert!(!compatible_run_instruction(&run, &t));
7070 t.review_branch = Some("approved-head".into());
7071 t.review_instruction = None;
7072 assert!(
7073 take_review_branch(&mut t, Some(&run)).is_none(),
7074 "legacy retained candidate cannot replace a revised approved head"
7075 );
7076 }
7077
7078 #[test]
7079 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
7080 let home = crate::run::test_home();
7081 crate::run::pin_test_home();
7082 let dir = tempfile::tempdir().unwrap();
7083 let queue = Queue::at(dir.path().join("queue"));
7084 for resume in [true, false] {
7085 let state = run_state(RunStatus::Blocked);
7086 let mut t = task();
7087 t.start(state.id.clone());
7088 let action = if resume {
7089 ask::ChoiceAction::Resume {
7090 run: state.id.clone(),
7091 }
7092 } else {
7093 ask::ChoiceAction::Requeue
7094 };
7095 let mut q = action_question(&state.id, action);
7096 q.status = ask::QuestionStatus::Open;
7097 q.answer = None;
7098 q.answered_at = None;
7099 t.fail("run stopped with a question", 3);
7100 hold_if_runnable(&queue, &mut t);
7101 assert!(t.operator_decided_at.is_none());
7102 assert!(t.operator_held());
7103 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
7104 let expected = if resume {
7105 ActionDecision::Resume(state.id.clone())
7106 } else {
7107 ActionDecision::Requeue
7108 };
7109 assert_eq!(
7110 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
7111 expected
7112 );
7113 t.hold_manual(Some("later explicit hold".into()));
7114 assert_eq!(
7115 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
7116 ActionDecision::Stale
7117 );
7118 }
7119 }
7120
7121 #[test]
7122 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
7123 let mut t = task();
7124 t.answers.push(crate::queue::AnsweredQuestion {
7125 question: "Inspect the remote?".into(),
7126 answer: "Inspect then wait".into(),
7127 });
7128 let old = instruction_for(&t);
7129 assert!(compatible_instruction(&old, &t));
7130 t.instruction = "Reuse approved PR head and review".into();
7131 t.release_operator("cli", None);
7132 assert!(!compatible_instruction(&old, &t));
7133 let approved = instruction_for(&t);
7134 assert!(approved.contains("waiting premises are superseded"));
7135 assert!(compatible_instruction(&approved, &t));
7136 assert_eq!(resumed_instruction(&approved, &t), approved);
7137 t.release_operator("web", None);
7138 assert!(compatible_instruction(&approved, &t));
7139 assert_eq!(t.resumption_history.len(), 1);
7140 t.start("run".into());
7141 assert!(instruction_for(&t).contains("Explicit operator resumption"));
7142 }
7143
7144 #[test]
7148 fn unrelated_ci_refunds_once_per_pass_and_holds_for_the_owner() {
7149 let mut state = RunState::new(
7150 PathBuf::from("/repo"),
7151 "main".into(),
7152 "base".into(),
7153 "task".into(),
7154 Config::default(),
7155 );
7156 state.status = RunStatus::Landing;
7157 state.parked = true;
7158 state.pr_pushed_head = Some("head".into());
7159 state.land_unrelated = Some(
7160 serde_json::from_value(serde_json::json!({
7161 "head": "head", "question": "ci-question",
7162 "evidence": {"head": "head", "base": "base", "changed": [], "checks": []}
7163 }))
7164 .unwrap(),
7165 );
7166 let mut t = task();
7167 t.start(state.id.clone());
7168 let verdict = Verdict {
7169 status: RunStatus::Landing,
7170 left_pr: true,
7171 quota_hit: false,
7172 parked: true,
7173 no_viable_candidates: false,
7174 };
7175 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
7176 assert_eq!(t.attempts, 0);
7177 assert_eq!(t.status, TaskStatus::Held);
7178 assert!(!t.operator_held());
7179 assert!(
7180 t.hold_reason
7181 .as_deref()
7182 .unwrap()
7183 .starts_with("[ci-unrelated]")
7184 );
7185 t.attempts = 1;
7187 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
7188 assert_eq!(t.attempts, 1);
7189 assert_eq!(t.passes.len(), 1);
7190 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7191 state.pr_pushed_head = Some("new-head".into());
7192 state.parked = false;
7193 state.status = RunStatus::Blocked;
7194 let stale = Verdict {
7195 status: RunStatus::Blocked,
7196 parked: false,
7197 ..verdict
7198 };
7199 settle_and_diagnose(&mut t, stale, "stale CI hold", 2, &state);
7200 assert_eq!(t.attempts, 1);
7201 state.begin_pass();
7203 state.begin_pass();
7204 state.status = RunStatus::Landing;
7205 state.parked = true;
7206 state.pr_pushed_head = Some("head".into());
7207 settle_and_diagnose(&mut t, verdict, "CI hold on resume", 2, &state);
7208 assert_eq!(t.attempts, 0);
7209 assert_eq!(t.passes.len(), 2);
7210 }
7211
7212 #[test]
7213 fn orphan_recovery_does_not_refund_a_previous_pass_loss() {
7214 let mut state = run_state(RunStatus::Failed);
7215 state
7216 .quota
7217 .push(loss("advisor-1", "2026-10-22T14:38:00Z", None));
7218 state.begin_pass();
7219 let mut t = task();
7220 t.start(state.id.clone());
7221 reclaim(&mut t, Some(state), 2, "en");
7222 assert_eq!(t.attempts, 1);
7223 }
7224
7225 #[test]
7226 fn an_all_seats_unauthenticated_node_refunds_and_holds_without_retry() {
7227 let mut state = run_state(RunStatus::Blocked);
7228 let losses: Vec<_> = ["alpha", "beta"]
7229 .into_iter()
7230 .enumerate()
7231 .map(|(i, agent)| {
7232 QuotaLoss::from_report(
7233 format!("advisor-{i}"),
7234 "advise",
7235 Some(&crate::agent::Quota {
7236 cause: crate::agent::QuotaCause::Authentication,
7237 agent: agent.to_owned(),
7238 reset: None,
7239 }),
7240 )
7241 })
7242 .collect();
7243 crate::graph::block_machine_outage(&mut state, "advise", &losses);
7244 state.quota = losses;
7245 let mut t = task();
7246 t.start(state.id.clone());
7247 let verdict = Verdict {
7248 status: state.status,
7249 left_pr: false,
7250 quota_hit: true,
7251 parked: false,
7252 no_viable_candidates: true,
7253 };
7254 settle_and_diagnose(&mut t, verdict, "authentication unavailable", 2, &state);
7255 assert_eq!(t.attempts, 0);
7256 assert_eq!(t.status, TaskStatus::Held);
7257 assert!(!t.operator_held());
7258 let reason = t.hold_reason.as_deref().unwrap();
7259 assert!(reason.starts_with("[authentication]"));
7260 assert!(
7261 reason.contains("alpha")
7262 && reason.contains("beta")
7263 && reason.contains("re-authenticate")
7264 );
7265 assert!(cooldown_until(&state.quota, Timestamp::now()).is_none());
7266 let mut orphan = task();
7268 orphan.start(state.id.clone());
7269 reclaim(&mut orphan, Some(state), 2, "en");
7270 assert_eq!(orphan.attempts, 0);
7271 assert_eq!(orphan.status, TaskStatus::Held);
7272 }
7273
7274 #[test]
7275 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
7276 let mut state = RunState::new(
7277 PathBuf::from("/repo"),
7278 "main".to_owned(),
7279 "abc1234".to_owned(),
7280 "task".to_owned(),
7281 Config::default(),
7282 );
7283 state.status = RunStatus::Blocked;
7284 state.resource_block = Some(crate::run::ResourceBlock {
7285 cause: crate::run::ResourceBlockCause::Other,
7286 node: "graft".to_owned(),
7287 kind: "shared build cache".to_owned(),
7288 detail: "timed out".to_owned(),
7289 at: jiff::Timestamp::now(),
7290 });
7291 let mut t = task();
7292 t.start("20260902-000000-aaaa".to_owned());
7293 assert_eq!(t.attempts, 1);
7294 let verdict = Verdict {
7295 status: RunStatus::Blocked,
7296 left_pr: false,
7297 quota_hit: false,
7298 parked: false,
7299 no_viable_candidates: false,
7300 };
7301 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
7302 assert_eq!(t.status, TaskStatus::Held);
7303 assert_eq!(t.attempts, 0, "the attempt is refunded");
7304 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
7305 assert!(
7306 !t.operator_held(),
7307 "a machine hold, released without a human"
7308 );
7309
7310 let mut selected = task();
7311 selected.start("20260902-000000-cccc".into());
7312 selected.replacement = Some(crate::merge_decision::Replacement {
7313 question: "fixture".into(),
7314 run: "earlier".into(),
7315 pr: "https://example.invalid/pr/1".into(),
7316 agent: "selected".into(),
7317 context: "fixture".into(),
7318 });
7319 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
7320 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
7321 assert_eq!(selected.attempts, 0);
7322 assert!(
7323 !selected.operator_held(),
7324 "a selected agent also retains a machine resource hold"
7325 );
7326 assert!(
7327 selected
7328 .hold_reason
7329 .as_deref()
7330 .unwrap()
7331 .starts_with("[resource] ")
7332 );
7333
7334 let mut p = task();
7336 p.start("20260902-000000-bbbb".to_owned());
7337 settle_and_diagnose(
7338 &mut p,
7339 Verdict {
7340 parked: true,
7341 ..verdict
7342 },
7343 "parked",
7344 2,
7345 &state,
7346 );
7347 assert_eq!(p.status, TaskStatus::Parked);
7348 assert_eq!(
7349 p.attempts, 0,
7350 "parking refunds once rather than also settling the resource hold"
7351 );
7352 }
7353
7354 #[test]
7355 fn record_failure_notice_links_the_task() {
7356 let dir = tempfile::tempdir().unwrap();
7357 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
7360 let queue = Queue::at(dir.path().join("queue"));
7361 let mut t = interrupt_task("rec-fail-1");
7362 record(&queue, &mut t);
7363
7364 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
7365 assert_eq!(found.len(), 1);
7366 assert_eq!(found[0].key, "loop:record");
7367 assert_eq!(
7368 found[0].link,
7369 Some(Link::Task {
7370 id: "rec-fail-1".to_owned()
7371 })
7372 );
7373 }
7374
7375 fn interrupt_task(id: &str) -> Task {
7378 let mut t = task();
7379 t.id = id.to_owned();
7380 t.interrupt = true;
7381 t
7382 }
7383
7384 fn task_with_id(id: &str) -> Task {
7386 let mut t = task();
7387 t.id = id.to_owned();
7388 t
7389 }
7390
7391 fn urgent_task(id: &str) -> Task {
7393 let mut t = task();
7394 t.id = id.to_owned();
7395 t.urgent = true;
7396 t
7397 }
7398
7399 #[test]
7400 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
7401 let dir = tempfile::tempdir().unwrap();
7402 let queue = Queue::at(dir.path().join("queue"));
7403 let mut parked = task_with_id("parked");
7404 parked.start("run".into());
7405 parked.park("upgrade");
7406 parked.record_pass_at(
7407 "run",
7408 0,
7409 Some(Timestamp::now()),
7410 1,
7411 Some(crate::run::ParkCause::Upgrade {
7412 from: "old".into(),
7413 to: None,
7414 }),
7415 );
7416 let mut fresh = task_with_id("fresh");
7417 fresh.priority = 100;
7418 let mut urgent_task = urgent_task("urgent");
7419 for t in [&mut parked, &mut fresh, &mut urgent_task] {
7420 queue.put(t).unwrap();
7421 }
7422 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7423 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7424 let candidates = runnable(&queue);
7425 assert_eq!(candidates[0].id, parked.id);
7426 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
7427 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
7428 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
7429 drop(first);
7430 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
7431 let next = runnable(&queue)
7432 .into_iter()
7433 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
7434 .unwrap();
7435 assert_eq!(next.task.id, fresh.id);
7436 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
7437 drop(claimed_elsewhere);
7438 drop(next);
7439 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
7440 }
7441
7442 #[test]
7443 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
7444 let dir = tempfile::tempdir().unwrap();
7445 let queue = Queue::at(dir.path().join("queue"));
7446 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7447 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7448 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
7449 let mut first = task();
7450 queue.put(&mut first).unwrap();
7451 let stale_plain = queue.get(&first.id).unwrap();
7452 first.set_urgent(true).unwrap();
7453 queue.put(&mut first).unwrap();
7454 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
7455 let mut second = urgent_task("second");
7456 queue.put(&mut second).unwrap();
7457 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
7458 drop(queue.claim(&second.id).unwrap());
7459 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
7460 drop(dispatched);
7461 let stale_urgent = queue.get(&first.id).unwrap();
7462 first.set_urgent(false).unwrap();
7463 queue.put(&mut first).unwrap();
7464 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
7465 assert_eq!(urgent.available_permits(), 1);
7466 drop(queue.claim(&first.id).unwrap());
7467 drop(occupied);
7468 }
7469
7470 #[test]
7471 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
7472 let dir = tempfile::tempdir().unwrap();
7473 let queue = Queue::at(dir.path().join("queue"));
7474 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7475 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7476 let mut t = task();
7477 queue.put(&mut t).unwrap();
7478 for enabled in [true, false] {
7479 let stale = queue.get(&t.id).unwrap();
7480 t.set_urgent(enabled).unwrap();
7481 queue.put(&mut t).unwrap();
7482 assert_ne!(stale.urgent, enabled);
7483 let DispatchClaim {
7484 task: fresh,
7485 claim,
7486 permit,
7487 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
7488 assert_eq!(fresh.urgent, enabled);
7489 assert_eq!(ordinary.available_permits(), usize::from(enabled));
7490 assert_eq!(urgent.available_permits(), usize::from(!enabled));
7491 assert!(queue.claim(&t.id).is_err());
7492 assert_eq!(fresh.attempts, stale.attempts);
7493 drop((claim, permit));
7494 assert_eq!(ordinary.available_permits(), 1);
7495 assert_eq!(urgent.available_permits(), 1);
7496 }
7497 t.set_urgent(true).unwrap();
7498 queue.put(&mut t).unwrap();
7499 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
7500 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7501 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
7502 drop(queue.claim(&t.id).unwrap());
7503 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
7504 drop(occupied);
7505 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
7506 drop(queue.claim(&t.id).unwrap());
7507 let held = queue.claim(&t.id).unwrap();
7508 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7509 drop(held);
7510 t.status = TaskStatus::Held;
7511 queue.put(&mut t).unwrap();
7512 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7513 drop(queue.claim(&t.id).unwrap());
7514 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
7515 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7516 drop(queue.claim(&t.id).unwrap());
7517 assert_eq!(urgent.available_permits(), 1);
7518 }
7519
7520 #[test]
7525 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
7526 assert_eq!(permit_kind(true, false), PermitKind::None);
7527 assert_eq!(permit_kind(true, true), PermitKind::None);
7528 }
7529
7530 #[test]
7535 fn permit_kind_separates_urgent_from_ordinary() {
7536 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
7537 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
7538 }
7539
7540 #[test]
7547 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
7548 let cfg = Config::default();
7549 let repo = Path::new("/any/repo/path");
7550
7551 let reason =
7552 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
7553 assert!(reason.contains("1024"), "{reason}");
7554 assert!(
7555 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7556 "{reason}"
7557 );
7558
7559 assert_eq!(
7560 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
7561 None,
7562 "exactly at the floor is open"
7563 );
7564 assert_eq!(
7565 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
7566 None,
7567 "comfortably above the floor is open"
7568 );
7569 }
7570
7571 #[test]
7572 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
7573 let mut cfg = Config::default();
7574 cfg.disk.min_free_bytes = 0;
7575 let repo = Path::new("/any/repo/path");
7576 assert_eq!(
7577 disk_gate_with(repo, &cfg, |_| Ok(0)),
7578 None,
7579 "a zero floor never measures at all"
7580 );
7581 }
7582
7583 #[test]
7584 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7585 let cfg = Config::default();
7586 let repo = Path::new("/any/repo/path");
7587 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7588 .expect("a measurement failure must close the gate, not open it");
7589 assert!(reason.contains("could not measure"), "{reason}");
7590 }
7591
7592 #[test]
7593 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7594 let ordinary = task();
7595 let next = advance_interrupt(
7596 Interrupt::Idle,
7597 std::slice::from_ref(&ordinary.id),
7598 std::slice::from_ref(&ordinary),
7599 );
7600 assert_eq!(next, Interrupt::Idle);
7601 }
7602
7603 #[test]
7604 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7605 let marked = interrupt_task("marked");
7608 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7609 assert_eq!(next, Interrupt::Idle);
7610 }
7611
7612 #[test]
7613 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7614 let marked = interrupt_task("marked");
7615 let next = advance_interrupt(
7616 Interrupt::Idle,
7617 &["running".to_owned()],
7618 std::slice::from_ref(&marked),
7619 );
7620 assert_eq!(
7621 next,
7622 Interrupt::Parking {
7623 parked: vec!["running".to_owned()],
7624 interrupt_task: "marked".to_owned(),
7625 }
7626 );
7627 }
7628
7629 #[test]
7638 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7639 let marked = interrupt_task("marked");
7640
7641 let two = advance_interrupt(
7642 Interrupt::Idle,
7643 &["a".to_owned(), "b".to_owned()],
7644 std::slice::from_ref(&marked),
7645 );
7646 assert_eq!(two, Interrupt::Idle);
7647
7648 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7649 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7650 }
7651
7652 #[test]
7653 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7654 let state = Interrupt::Parking {
7655 parked: vec!["running".to_owned()],
7656 interrupt_task: "marked".to_owned(),
7657 };
7658 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7660 assert_eq!(still_going, state);
7661
7662 let stopped_but_not_yet_dispatched =
7666 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7667 assert_eq!(stopped_but_not_yet_dispatched, state);
7668
7669 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7671 assert_eq!(
7672 dispatched,
7673 Interrupt::Running {
7674 parked: vec!["running".to_owned()],
7675 interrupt_task: "marked".to_owned(),
7676 }
7677 );
7678 }
7679
7680 #[test]
7681 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7682 let state = Interrupt::Running {
7683 parked: vec!["running".to_owned()],
7684 interrupt_task: "marked".to_owned(),
7685 };
7686 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7687 assert_eq!(still_running, state);
7688
7689 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7696 assert_eq!(
7697 ended,
7698 Interrupt::Resuming {
7699 parked: vec!["running".to_owned()]
7700 }
7701 );
7702 }
7703
7704 #[test]
7705 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7706 let state = Interrupt::Resuming {
7707 parked: vec!["running".to_owned()],
7708 };
7709 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7710 assert_eq!(still_waiting, state);
7711
7712 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7713 assert_eq!(dispatched, Interrupt::Idle);
7714 }
7715
7716 #[test]
7722 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7723 {
7724 let state = Interrupt::Parking {
7725 parked: vec!["running".to_owned()],
7726 interrupt_task: "marked".to_owned(),
7727 };
7728 let next = advance_interrupt(state, &[], &[]);
7731 assert_eq!(
7732 next,
7733 Interrupt::Resuming {
7734 parked: vec!["running".to_owned()]
7735 },
7736 "abandoning the interrupt must not abandon the resume it owes"
7737 );
7738 }
7739
7740 #[test]
7743 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7744 let state = Interrupt::Resuming {
7745 parked: vec!["running".to_owned()],
7746 };
7747 let next = advance_interrupt(state, &[], &[]);
7748 assert_eq!(
7749 next,
7750 Interrupt::Idle,
7751 "nothing is left to wait for; the loop must not stay wedged"
7752 );
7753 }
7754
7755 #[test]
7756 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7757 let marked = interrupt_task("marked");
7758 let next = advance_interrupt_tick(
7759 false,
7760 Interrupt::Idle,
7761 &["running".to_owned()],
7762 std::slice::from_ref(&marked),
7763 );
7764 assert_eq!(
7765 next,
7766 Interrupt::Idle,
7767 "an unmarked, unconfigured daemon must behave exactly as before"
7768 );
7769 }
7770
7771 #[test]
7772 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7773 let state = Interrupt::Parking {
7774 parked: vec!["running".to_owned()],
7775 interrupt_task: "marked".to_owned(),
7776 };
7777 let candidates = vec![interrupt_task("marked"), task()];
7778 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7779 assert!(
7780 allowed.is_empty(),
7781 "nothing may dispatch - not even the interrupt task itself - \
7782 until the parked run has actually stopped"
7783 );
7784 }
7785
7786 #[test]
7800 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7801 for state in [
7802 Interrupt::Parking {
7803 parked: vec!["running".to_owned()],
7804 interrupt_task: "marked".to_owned(),
7805 },
7806 Interrupt::Running {
7807 parked: vec!["running".to_owned()],
7808 interrupt_task: "marked".to_owned(),
7809 },
7810 Interrupt::Resuming {
7811 parked: vec!["running".to_owned()],
7812 },
7813 ] {
7814 let candidates = vec![
7815 interrupt_task("marked"),
7816 urgent_task("hot"),
7817 task_with_id("ordinary"),
7818 ];
7819 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7820 assert!(
7821 !allowed.iter().any(|t| t.id == "hot"),
7822 "an urgent candidate must wait out the same gate as anything \
7823 else while the run it would run alongside has not actually \
7824 left flight, for state {state:?}: {allowed:?}"
7825 );
7826 }
7827 }
7828
7829 #[test]
7835 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7836 let state = Interrupt::Resuming {
7837 parked: vec!["hot".to_owned()],
7838 };
7839 let candidates = vec![urgent_task("hot"), task()];
7840 let allowed = interrupt_gate(&state, &[], candidates);
7841 assert_eq!(
7842 allowed.iter().filter(|t| t.id == "hot").count(),
7843 1,
7844 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7845 );
7846 }
7847
7848 #[test]
7849 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7850 let state = Interrupt::Parking {
7851 parked: vec!["running".to_owned()],
7852 interrupt_task: "marked".to_owned(),
7853 };
7854 let other = task();
7855 let candidates = vec![interrupt_task("marked"), other.clone()];
7856 let allowed = interrupt_gate(&state, &[], candidates);
7857 assert_eq!(allowed.len(), 1);
7858 assert_eq!(allowed[0].id, "marked");
7859 }
7860
7861 #[test]
7862 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7863 let state = Interrupt::Running {
7864 parked: vec!["running".to_owned()],
7865 interrupt_task: "marked".to_owned(),
7866 };
7867 let candidates = vec![task(), task()];
7868 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7869 assert!(allowed.is_empty());
7870 }
7871
7872 #[test]
7879 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7880 let state = Interrupt::Resuming {
7881 parked: vec!["a".to_owned(), "c".to_owned()],
7882 };
7883 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7884 let allowed = interrupt_gate(&state, &[], candidates);
7885 assert_eq!(
7886 allowed.len(),
7887 1,
7888 "at most one candidate may be offered while resuming: {allowed:?}"
7889 );
7890 assert_eq!(allowed[0].id, "a");
7891 }
7892
7893 #[test]
7894 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7895 let state = Interrupt::Resuming {
7896 parked: vec!["a".to_owned()],
7897 };
7898 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7899 assert!(allowed.is_empty());
7900 }
7901
7902 #[test]
7908 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7909 let running = task(); let marked = interrupt_task("marked");
7911
7912 let mut state = Interrupt::Idle;
7913 let in_flight = vec![running.id.clone()];
7915 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7916 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7917 assert!(gated.is_empty(), "still waiting on `running` to park");
7918
7919 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7921 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7922 assert_eq!(
7923 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7924 vec!["marked"],
7925 "only the interrupt task may be offered to the dispatcher now"
7926 );
7927
7928 state = advance_interrupt_tick(
7930 true,
7931 state,
7932 &["marked".to_owned()],
7933 std::slice::from_ref(&running),
7934 );
7935 let gated = interrupt_gate(
7936 &state,
7937 &["marked".to_owned()],
7938 vec![marked.clone(), running.clone()],
7939 );
7940 assert!(
7941 gated.is_empty(),
7942 "the parked run must not be offered back while the interrupt \
7943 task is still running"
7944 );
7945
7946 let other = task_with_id("other");
7950 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7951 assert_eq!(
7952 state,
7953 Interrupt::Resuming {
7954 parked: vec![running.id.clone()]
7955 }
7956 );
7957 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7958 assert_eq!(
7959 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7960 vec![running.id.as_str()],
7961 "exactly the parked run resumes - not the unrelated task, even \
7962 though it was offered first"
7963 );
7964
7965 state = advance_interrupt_tick(
7969 true,
7970 state,
7971 std::slice::from_ref(&running.id),
7972 std::slice::from_ref(&other),
7973 );
7974 assert_eq!(state, Interrupt::Idle);
7975 let gated = interrupt_gate(
7976 &state,
7977 std::slice::from_ref(&running.id),
7978 vec![other.clone()],
7979 );
7980 assert_eq!(
7981 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7982 vec![other.id.as_str()],
7983 "ordinary dispatch is unrestricted again"
7984 );
7985 }
7986
7987 #[test]
7988 fn every_run_status_settles_the_task_it_came_from() {
7989 let table = [
7991 (RunStatus::Merged, TaskStatus::Done, 1),
7992 (RunStatus::Ready, TaskStatus::Done, 1),
7993 (RunStatus::Stalled, TaskStatus::Failed, 0),
7994 (RunStatus::Blocked, TaskStatus::Failed, 1),
7995 (RunStatus::Failed, TaskStatus::Failed, 1),
7996 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7997 (RunStatus::Prep, TaskStatus::Failed, 1),
7998 (RunStatus::Implementing, TaskStatus::Failed, 1),
7999 (RunStatus::Judging, TaskStatus::Failed, 1),
8000 (RunStatus::Deliberating, TaskStatus::Failed, 1),
8001 (RunStatus::Voting, TaskStatus::Failed, 1),
8002 (RunStatus::Reviewing, TaskStatus::Failed, 1),
8003 (RunStatus::Gating, TaskStatus::Failed, 1),
8004 ];
8005 for (run, want, attempts) in table {
8006 let mut t = task();
8007 t.start("20260902-000000-aaaa".to_owned());
8008 settle(
8009 &mut t,
8010 Verdict {
8011 status: run,
8012 left_pr: false,
8013 parked: false,
8014 quota_hit: matches!(run, RunStatus::Stalled),
8015 no_viable_candidates: false,
8016 },
8017 "why",
8018 2,
8019 );
8020 assert_eq!(t.status, want, "task status after {}", label(run));
8021 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
8022 }
8023 }
8024
8025 #[test]
8026 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
8027 let mut stalled = task();
8028 stalled.start("20260902-000000-aaaa".to_owned());
8029 settle(
8030 &mut stalled,
8031 Verdict {
8032 status: RunStatus::Stalled,
8033 left_pr: false,
8034 parked: false,
8035 quota_hit: true,
8036 no_viable_candidates: false,
8037 },
8038 "quota",
8039 1,
8040 );
8041 assert_eq!(stalled.attempts, 0);
8042 assert!(
8043 stalled.status.runnable(),
8044 "a machine problem must leave the task in line"
8045 );
8046
8047 let mut blocked = task();
8048 blocked.start("20260902-000000-aaaa".to_owned());
8049 settle(
8050 &mut blocked,
8051 Verdict {
8052 status: RunStatus::Blocked,
8053 left_pr: false,
8054 parked: false,
8055 quota_hit: false,
8056 no_viable_candidates: false,
8057 },
8058 "findings open",
8059 1,
8060 );
8061 assert_eq!(blocked.attempts, 1);
8062 assert_eq!(
8063 blocked.status,
8064 TaskStatus::Held,
8065 "the last attempt hands the task to a human"
8066 );
8067 }
8068
8069 #[test]
8070 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
8071 let mut delivered = task();
8074 delivered.start("20260903-080619-01c2".to_owned());
8075 settle(
8076 &mut delivered,
8077 Verdict {
8078 status: RunStatus::Blocked,
8079 left_pr: true,
8080 parked: false,
8081 quota_hit: false,
8082 no_viable_candidates: false,
8083 },
8084 "no check status",
8085 4,
8086 );
8087 assert_eq!(
8088 delivered.status,
8089 TaskStatus::Held,
8090 "a pull request waiting on CI or a person is not a retryable failure"
8091 );
8092 assert!(
8093 !delivered.status.runnable(),
8094 "the loop must not pick this task up again"
8095 );
8096 assert_eq!(
8097 delivered.last_error.as_deref(),
8098 Some("no check status"),
8099 "the operator needs to be told what the gate was waiting for"
8100 );
8101
8102 let mut empty_handed = task();
8105 empty_handed.start("20260903-080619-01c2".to_owned());
8106 settle(
8107 &mut empty_handed,
8108 Verdict {
8109 status: RunStatus::Blocked,
8110 left_pr: false,
8111 parked: false,
8112 quota_hit: false,
8113 no_viable_candidates: false,
8114 },
8115 "findings open",
8116 4,
8117 );
8118 assert_eq!(empty_handed.status, TaskStatus::Failed);
8119 assert!(empty_handed.status.runnable());
8120 }
8121
8122 #[test]
8123 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
8124 let mut noop = task();
8130 noop.start("20260912-131304-391f".to_owned());
8131 settle(
8132 &mut noop,
8133 Verdict {
8134 status: RunStatus::VerifiedNoop,
8135 left_pr: false,
8136 parked: false,
8137 quota_hit: false,
8138 no_viable_candidates: true,
8139 },
8140 "candidate A: already fixed by b32cfc4, on main",
8141 4,
8142 );
8143 assert_eq!(
8144 noop.status,
8145 TaskStatus::Held,
8146 "an unverified claim is a request for a human, not a failure"
8147 );
8148 assert!(
8149 !noop.status.runnable(),
8150 "the loop must not requeue this on the same unverified claim"
8151 );
8152 assert_eq!(noop.attempts, 1);
8157 }
8158
8159 #[test]
8160 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
8161 let mut parked = task();
8166 parked.start("20260903-183634-2d98".to_owned());
8167 settle(
8168 &mut parked,
8169 Verdict {
8170 status: RunStatus::Implementing,
8171 left_pr: false,
8172 quota_hit: false,
8173 parked: true,
8174 no_viable_candidates: false,
8175 },
8176 "parked after `implementing`",
8177 2,
8178 );
8179 assert_eq!(parked.attempts, 0, "a park is refunded");
8180 assert!(
8181 parked.status.runnable(),
8182 "and the task stays in line so the next loop resumes its run"
8183 );
8184 assert_eq!(parked.status, TaskStatus::Parked);
8185 assert_eq!(
8186 parked.park_reason.as_deref(),
8187 Some("parked after `implementing`"),
8188 "the card says where it stopped"
8189 );
8190 assert_eq!(parked.last_error, None, "nothing failed");
8191
8192 let mut broken = task();
8196 broken.start("20260903-183634-2d98".to_owned());
8197 settle(
8198 &mut broken,
8199 Verdict {
8200 status: RunStatus::Implementing,
8201 left_pr: false,
8202 quota_hit: false,
8203 parked: false,
8204 no_viable_candidates: false,
8205 },
8206 "returned mid-flight",
8207 2,
8208 );
8209 assert_eq!(broken.attempts, 1);
8210 }
8211
8212 #[test]
8213 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
8214 let mut flaky = task();
8219 flaky.start("20260903-123023-e633".to_owned());
8220 settle(
8221 &mut flaky,
8222 Verdict {
8223 status: RunStatus::Stalled,
8224 left_pr: false,
8225 parked: false,
8226 quota_hit: false,
8227 no_viable_candidates: false,
8228 },
8229 "verdict rests on 1 of 3 judges",
8230 2,
8231 );
8232 assert_eq!(
8233 flaky.attempts, 1,
8234 "flakiness spends an attempt, so `max_attempts` still bounds it"
8235 );
8236 assert!(flaky.status.runnable(), "and it is still worth retrying");
8237
8238 let mut limited = task();
8240 limited.start("20260903-123023-e633".to_owned());
8241 settle(
8242 &mut limited,
8243 Verdict {
8244 status: RunStatus::Stalled,
8245 left_pr: false,
8246 parked: false,
8247 quota_hit: true,
8248 no_viable_candidates: false,
8249 },
8250 "judge-2, judge-3 out of quota",
8251 2,
8252 );
8253 assert_eq!(limited.attempts, 0, "a quota window is refunded");
8254 assert!(limited.status.runnable());
8255
8256 let mut worn = task();
8259 for _ in 0..2 {
8260 worn.release();
8261 }
8262 worn.start("20260903-123023-e633".to_owned());
8263 worn.attempts = 2;
8264 settle(
8265 &mut worn,
8266 Verdict {
8267 status: RunStatus::Stalled,
8268 left_pr: false,
8269 parked: false,
8270 quota_hit: false,
8271 no_viable_candidates: false,
8272 },
8273 "no quorum again",
8274 2,
8275 );
8276 assert_eq!(worn.status, TaskStatus::Held);
8277 assert!(!worn.status.runnable());
8278 }
8279
8280 #[test]
8281 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
8282 let mut wiped_out = task();
8289 wiped_out.start("20260907-025000-a1b2".to_owned());
8290 settle(
8291 &mut wiped_out,
8292 Verdict {
8293 status: RunStatus::Failed,
8294 left_pr: false,
8295 parked: false,
8296 quota_hit: true,
8297 no_viable_candidates: true,
8298 },
8299 "no candidate produced a change; nothing to judge",
8300 2,
8301 );
8302 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
8303 assert!(
8304 wiped_out.status.runnable(),
8305 "a machine problem must leave the task in line"
8306 );
8307
8308 let mut partial_progress = task();
8314 partial_progress.start("20260907-025500-c3d4".to_owned());
8315 settle(
8316 &mut partial_progress,
8317 Verdict {
8318 status: RunStatus::Failed,
8319 left_pr: false,
8320 parked: false,
8321 quota_hit: true,
8322 no_viable_candidates: false,
8323 },
8324 "gate failed on the winning candidate",
8325 2,
8326 );
8327 assert_eq!(
8328 partial_progress.attempts, 1,
8329 "a candidate that actually produced a change spends the attempt \
8330 even though some other seat hit its quota"
8331 );
8332 assert!(partial_progress.status.runnable());
8333 }
8334
8335 #[test]
8336 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
8337 let mut t = task();
8343 t.start("20260907-025000-a1b2".to_owned());
8344 let mut state = run_state(RunStatus::Failed);
8345 state.quota.push(QuotaLoss {
8346 cause: crate::agent::QuotaCause::UsageLimit,
8347 agent: String::new(),
8348 seat: "cand-a".to_owned(),
8349 node: "implement".to_owned(),
8350 at: Timestamp::now(),
8351 reset: None,
8352 });
8353 assert!(
8354 state.viable().is_empty(),
8355 "no candidate was added, so nothing is viable"
8356 );
8357 reclaim(&mut t, Some(state), 2, "en");
8358 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
8359 assert!(t.status.runnable());
8360 }
8361
8362 #[test]
8363 fn a_held_task_is_never_offered_to_the_loop() {
8364 let dir = tempfile::tempdir().unwrap();
8365 let queue = Queue::at(dir.path().to_path_buf());
8366 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
8367 let mut t = task();
8368 t.id = format!("2026090{n}-000000-000{n}");
8369 t.priority = priority;
8370 queue.put(&mut t).unwrap();
8371 }
8372 let mut held = task();
8373 held.id = "20260909-000000-9999".to_owned();
8374 held.priority = 99;
8375 held.hold_machine(None);
8376 queue.put(&mut held).unwrap();
8377
8378 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
8379 assert_eq!(order.len(), 3);
8380 assert!(!order.contains(&held.id));
8381 assert_eq!(
8382 order.first().cloned(),
8383 queue.next_runnable().map(|t| t.id),
8384 "the loop's first candidate is exactly what the queue offers"
8385 );
8386 assert_eq!(
8387 order,
8388 vec![
8389 "20260902-000000-0002".to_owned(),
8390 "20260903-000000-0003".to_owned(),
8391 "20260901-000000-0001".to_owned(),
8392 ],
8393 "priority first, then oldest, so nothing starves"
8394 );
8395 }
8396
8397 #[test]
8398 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
8399 let dir = tempfile::tempdir().unwrap();
8400 let queue = Queue::at(dir.path().to_path_buf());
8401 let mut old = task();
8402 old.id = "20260101-000000-old0".to_owned();
8403 queue.put(&mut old).unwrap();
8404 let mut fresh = task();
8405 fresh.id = "20260101-000000-new0".to_owned();
8406 queue.put(&mut fresh).unwrap();
8407
8408 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
8412 std::thread::sleep(Duration::from_millis(60));
8413 let live = queue.claim(&fresh.id).unwrap();
8414
8415 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8416 assert_eq!(swept, vec![old.id.clone()]);
8417 assert!(
8418 queue.claim(&old.id).is_ok(),
8419 "an unparseable lock older than the threshold is swept"
8420 );
8421 assert!(
8422 queue.claim(&fresh.id).is_err(),
8423 "a live pid protects its lock regardless of age"
8424 );
8425 drop(live);
8426 }
8427
8428 #[test]
8429 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
8430 let dir = tempfile::tempdir().unwrap();
8433 let queue = Queue::at(dir.path().to_path_buf());
8434 let mut t = task();
8435 t.id = "20260101-000000-live".to_owned();
8436 queue.put(&mut t).unwrap();
8437
8438 let claim = queue.claim(&t.id).unwrap();
8439 std::thread::sleep(Duration::from_millis(60));
8440
8441 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8442 assert!(
8443 swept.is_empty(),
8444 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
8445 );
8446 assert!(
8447 queue.claim(&t.id).is_err(),
8448 "the lock still protects its task"
8449 );
8450 drop(claim);
8451 }
8452
8453 fn injected_dead_pid() -> u32 {
8456 std::process::id().checked_add(1).unwrap_or(1)
8457 }
8458
8459 #[test]
8460 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
8461 let dir = tempfile::tempdir().unwrap();
8462 let queue = Queue::at(dir.path().to_path_buf());
8463 let mut t = task();
8464 t.id = "20260101-000000-dead".to_owned();
8465 queue.put(&mut t).unwrap();
8466 let dead_pid = injected_dead_pid();
8467
8468 std::fs::write(
8473 dir.path().join(format!("{}.lock", t.id)),
8474 dead_pid.to_string(),
8475 )
8476 .unwrap();
8477
8478 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8479 pid != dead_pid
8480 });
8481 assert_eq!(
8482 swept,
8483 vec![t.id.clone()],
8484 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
8485 );
8486 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
8487 }
8488
8489 #[test]
8490 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
8491 let dir = tempfile::tempdir().unwrap();
8492 let queue = Queue::at(dir.path().to_path_buf());
8493 let mut t = task();
8494 t.id = "20260101-000000-late".to_owned();
8495 queue.put(&mut t).unwrap();
8496 let dead_pid = injected_dead_pid();
8497
8498 assert!(
8501 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
8502 "nothing has claimed the task yet"
8503 );
8504
8505 std::fs::write(
8508 dir.path().join(format!("{}.lock", t.id)),
8509 dead_pid.to_string(),
8510 )
8511 .unwrap();
8512
8513 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8517 pid != dead_pid
8518 });
8519 assert_eq!(swept, vec![t.id.clone()]);
8520 }
8521
8522 #[test]
8523 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
8524 crate::run::pin_test_home();
8529 let dir = tempfile::tempdir().unwrap();
8530 let queue = Queue::at(dir.path().to_path_buf());
8531 let mut t = task();
8532 t.id = "20260101-000000-crsh".to_owned();
8533 t.status = TaskStatus::Running;
8534 t.attempts = 1;
8535 t.runs.push("20260904-000000-4043".to_owned());
8539 queue.put(&mut t).unwrap();
8540 let dead_pid = injected_dead_pid();
8541
8542 std::fs::write(
8545 dir.path().join(format!("{}.lock", t.id)),
8546 dead_pid.to_string(),
8547 )
8548 .unwrap();
8549
8550 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
8556 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8557
8558 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8559 pid != dead_pid
8560 });
8561 assert_eq!(swept, vec![t.id.clone()]);
8562
8563 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8564 assert_eq!(reclaimed, vec![t.id.clone()]);
8565 let after = queue.get(&t.id).unwrap();
8566 assert_eq!(
8567 after.status,
8568 TaskStatus::Held,
8569 "no run.json to recover from, so a human is asked"
8570 );
8571 assert_eq!(
8572 after.runs,
8573 vec!["20260904-000000-4043".to_owned()],
8574 "the crashed run's id is kept as evidence, not discarded"
8575 );
8576 }
8577
8578 #[test]
8579 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
8580 let dir = tempfile::tempdir().unwrap();
8581 let queue = Queue::at(dir.path().to_path_buf());
8582 let mut t = task();
8583 t.id = "20260101-000000-unknown".to_owned();
8584 queue.put(&mut t).unwrap();
8585 let dead_pid = injected_dead_pid();
8586 std::fs::write(
8587 dir.path().join(format!("{}.lock", t.id)),
8588 dead_pid.to_string(),
8589 )
8590 .unwrap();
8591
8592 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8593 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8594 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8595 }
8596
8597 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8598 let mut s = run_state(status);
8599 s.config.graph.language = language.to_owned();
8600 s
8601 }
8602
8603 fn unstarted_verdict(status: RunStatus) -> Verdict {
8604 Verdict {
8605 status,
8606 left_pr: false,
8607 quota_hit: false,
8608 parked: false,
8609 no_viable_candidates: false,
8610 }
8611 }
8612
8613 #[test]
8614 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8615 let mut t = task();
8616 t.attempts = 1;
8617 settle_in(
8618 &mut t,
8619 unstarted_verdict(RunStatus::AlreadyInBase),
8620 "already in main",
8621 1,
8622 phrases("en"),
8623 );
8624 assert_eq!(t.status, TaskStatus::Done);
8625 assert_eq!(t.attempts, 0);
8626 }
8627
8628 #[test]
8629 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8630 let reason = |language: &str| {
8631 let mut t = task();
8632 settle_in(
8633 &mut t,
8634 unstarted_verdict(RunStatus::Judging),
8635 "boom",
8636 1,
8637 phrases(language),
8638 );
8639 t.last_error.or(t.hold_reason).unwrap_or_default()
8640 };
8641 assert!(
8642 reason("en").starts_with("the graph stopped at `"),
8643 "{}",
8644 reason("en")
8645 );
8646 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8647 assert!(reason("日本語").contains("boom"));
8648 assert_eq!(reason("fr"), reason("en"));
8649 }
8650
8651 #[test]
8652 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8653 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8654 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8655 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8656 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8657 assert!(ja.contains("[run "), "{ja}");
8658 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8659 assert!(ended.starts_with("run 終了: "), "{ended}");
8660 let mut de = run_state_in(RunStatus::Failed, "de");
8661 let mut en = run_state_in(RunStatus::Failed, "en");
8662 de.id = "same".to_owned();
8663 en.id = "same".to_owned();
8664 assert_eq!(describe(&de), describe(&en));
8665 }
8666
8667 #[test]
8668 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8669 use crate::handover::Refused;
8670 let cases = [
8671 Refused::Foreign {
8672 branch: "b".into(),
8673 path: "/w/x".into(),
8674 why: "made by hand".into(),
8675 },
8676 Refused::Unsafe {
8677 branch: "b".into(),
8678 path: "/w/x".into(),
8679 why: "its worktree has uncommitted changes (a.rs)".into(),
8680 },
8681 Refused::ReleaseFailed {
8682 branch: "b".into(),
8683 path: "/w/x".into(),
8684 run: "ab12".into(),
8685 },
8686 ];
8687 for r in &cases {
8688 let en = (phrases("en").handover_refused)(r);
8689 assert_eq!(en, r.to_string());
8690 let ja = (phrases("ja").handover_refused)(r);
8691 assert!(
8692 !ja.contains("is checked out") && !ja.contains("try again"),
8693 "{ja}"
8694 );
8695 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8696 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8697 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8698 }
8699 }
8700 assert!(phrases("en").handover_hint.contains("release the task"));
8701 assert!(phrases("ja").handover_hint.contains("解放"));
8702 }
8703
8704 #[test]
8705 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8706 let build = |lang: &str| {
8707 let mut s = run_state_in(RunStatus::Blocked, lang);
8708 s.id = "same".to_owned();
8709 s.config.graph.review_rounds = 3;
8710 let mut r = review_round(3);
8711 r.expected = 3;
8712 r.answered = 1;
8713 s.reviews.push(r);
8714 s.event(
8715 "review",
8716 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8717 );
8718 s
8719 };
8720 let en = describe(&build("en"));
8721 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8722 let ja = describe(&build("ja"));
8723 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8724 assert!(!ja.contains("never answered"), "{ja}");
8725 assert!(ja.contains("[run "), "{ja}");
8726
8727 let mut failed = build("ja");
8730 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8731 command: "cargo test".to_owned(),
8732 code: Some(1),
8733 output_tail: String::new(),
8734 duration_ms: 0,
8735 resource_blocked: false,
8736 });
8737 failed.event("review", "stopped; e2e failed: cargo test");
8738 let ja = describe(&failed);
8739 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8740 assert!(!ja.contains("席のレビュアー"), "{ja}");
8741 }
8742
8743 #[test]
8744 fn refusals_and_recovery_prose_follow_the_language() {
8745 let home = crate::run::test_home();
8746 let t = held_task_with("r1");
8747 let q = action_question(
8748 "r1",
8749 ask::ChoiceAction::Resume {
8750 run: "r1".to_owned(),
8751 },
8752 );
8753 let refuse = |p: &Phrases| match decide_action(&home, &t, &q, p, |_: &str| bail!("gone")) {
8754 ActionDecision::Refuse(s) => s,
8755 other => panic!("{other:?}"),
8756 };
8757 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8758 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8759 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8760
8761 let mut held = task();
8762 reclaim(&mut held, None, 2, "ja");
8763 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8764 let mut held = task();
8765 reclaim(&mut held, None, 2, "xx");
8766 assert!(held.hold_reason.unwrap().contains("held for a human"));
8767 }
8768
8769 fn run_state(status: RunStatus) -> RunState {
8770 let mut state = RunState::new(
8771 PathBuf::from("/repo"),
8772 "main".to_owned(),
8773 "abc1234def".to_owned(),
8774 "add retries".to_owned(),
8775 Config::default(),
8776 );
8777 state.status = status;
8778 state
8779 }
8780
8781 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8782 Candidate {
8783 index: 0,
8784 label,
8785 agent: "claude".to_owned(),
8786 branch: format!("magi/x/{label}"),
8787 worktree: PathBuf::from("/repo"),
8788 summary: summary.to_owned(),
8789 stat: String::new(),
8790 files: 0,
8791 commits: usize::from(!empty),
8792 empty,
8793 failed: failed.map(str::to_owned),
8794 verified_noop: None,
8795 duration_ms: 0,
8796 folded: false,
8797 }
8798 }
8799
8800 #[test]
8801 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8802 let mut state = run_state(RunStatus::Blocked);
8803 state.gate = vec![
8804 CommandOutcome {
8805 command: "cargo make check".to_owned(),
8806 code: Some(0),
8807 output_tail: "ok".to_owned(),
8808 duration_ms: 0,
8809 resource_blocked: false,
8810 },
8811 CommandOutcome {
8812 command: "cargo test".to_owned(),
8813 code: Some(101),
8814 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8815 duration_ms: 0,
8816 resource_blocked: false,
8817 },
8818 ];
8819 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8820 assert!(d.contains("cargo test"), "{d}");
8821 assert!(
8822 !d.contains("cargo make check"),
8823 "a passing check is not a diagnostic: {d}"
8824 );
8825 assert!(d.contains("assertion failed"), "{d}");
8826 }
8827
8828 #[test]
8829 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8830 let mut state = run_state(RunStatus::Blocked);
8831 state.event(
8832 "land",
8833 "stopped: the fixer produced no commit while 2 check(s) were failing \
8834 (build, lint); stopping instead of looping on an unchanged tree",
8835 );
8836 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8837 assert!(d.contains("build"), "{d}");
8838 assert!(d.contains("lint"), "{d}");
8839 assert!(d.contains("fixer produced no commit"), "{d}");
8840 }
8841
8842 #[test]
8843 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8844 let state = run_state(RunStatus::VerifiedNoop);
8849 let d = describe(&state);
8850 assert!(
8851 d.contains("agent-verified no-op"),
8852 "expected the display label, not the wire spelling: {d}"
8853 );
8854 assert!(!d.contains("verified_noop"), "{d}");
8855 }
8856
8857 #[test]
8858 fn diagnostic_keeps_gate_panic_before_large_backtrace() {
8859 let mut state = run_state(RunStatus::Failed);
8860 state.gate.push(crate::run::CommandOutcome {
8861 command: "cargo test".into(),
8862 code: Some(101),
8863 output_tail: format!(
8864 "---- widget stdout ----\nwidget log\nthread 'widget' panicked at widget.rs:12:\nassertion failed: widget missing\n{}test result: FAILED\n",
8865 " 41: 0x1234 - backtrace frame\n".repeat(10000)
8866 ),
8867 duration_ms: 1,
8868 resource_blocked: false,
8869 });
8870 let d = diagnostic(&state).unwrap();
8871 assert!(d.contains("panicked at widget.rs:12"), "{d}");
8872 assert!(d.contains("assertion failed: widget missing"));
8873 assert!(d.contains("widget log"));
8874 }
8875
8876 #[test]
8877 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8878 let mut state = run_state(RunStatus::Failed);
8884 state.candidates = vec![candidate(
8885 'A',
8886 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8887 true,
8888 None,
8889 )];
8890 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8891 assert!(d.contains("candidate A"), "{d}");
8892 assert!(d.contains("tagged v1.2.3"), "{d}");
8893 }
8894
8895 #[test]
8896 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8897 let mut state = run_state(RunStatus::Failed);
8898 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8899 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8900 assert!(d.contains("candidate A"), "{d}");
8901 assert!(d.contains("agent timed out"), "{d}");
8902 }
8903
8904 #[test]
8905 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8906 let mut state = run_state(RunStatus::Failed);
8909 state.candidates = vec![candidate('A', "did the work", false, None)];
8910 assert!(diagnostic(&state).is_none());
8911 }
8912
8913 #[test]
8914 fn diagnostic_is_bounded_however_much_a_run_printed() {
8915 let mut state = run_state(RunStatus::Blocked);
8916 state.gate = vec![
8917 CommandOutcome {
8918 command: "cargo test".to_owned(),
8919 code: Some(101),
8920 output_tail: "x".repeat(50_000),
8921 duration_ms: 0,
8922 resource_blocked: false,
8923 },
8924 CommandOutcome {
8925 command: "cargo clippy".to_owned(),
8926 code: Some(1),
8927 output_tail: "y".repeat(50_000),
8928 duration_ms: 0,
8929 resource_blocked: false,
8930 },
8931 ];
8932 state.candidates = vec![
8933 candidate('A', &"z".repeat(50_000), true, None),
8934 candidate('B', &"w".repeat(50_000), true, None),
8935 ];
8936 let d = diagnostic(&state).expect("plenty here to diagnose");
8937 assert!(
8938 d.len() <= DIAGNOSTIC_MAX,
8939 "diagnostic grew to {} bytes, unbounded",
8940 d.len()
8941 );
8942 }
8943
8944 #[test]
8945 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8946 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8947 let mut t = task();
8948 let mut state = run_state(status);
8949 t.pr_recovery = Some(crate::pr_recovery::Selection {
8950 url: "url".into(),
8951 branch: format!("magi/task-{}", t.id),
8952 head: "a".repeat(40),
8953 source_head: None,
8954 source_run: None,
8955 source_proof: None,
8956 disposition: Some(crate::pr_recovery::Disposition::Land),
8957 instruction: t.instruction.clone(),
8958 approved_at: Timestamp::now(),
8959 });
8960 state.pr_recovery = t.pr_recovery.clone();
8961 t.start(state.id.clone());
8962 settle_and_diagnose(
8963 &mut t,
8964 Verdict {
8965 status,
8966 left_pr: false,
8967 quota_hit: false,
8968 parked: false,
8969 no_viable_candidates: false,
8970 },
8971 "delivery stopped",
8972 5,
8973 &state,
8974 );
8975 assert_eq!(t.status, TaskStatus::Held);
8976 let reason = t.hold_reason.as_deref().unwrap();
8977 if status == RunStatus::Ready {
8978 assert!(reason.contains("Selected PR delivery is incomplete"));
8979 assert!(t.recovery_budget.is_none());
8980 } else {
8981 assert!(reason.contains("Selected recovery stopped"));
8982 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8983 }
8984 assert!(t.pr_recovery.is_some());
8985 }
8986 }
8987
8988 #[test]
8989 fn base_recovery_settlement_holds_without_replacement_or_duplicate_notice_cause() {
8990 let mut state = run_state(RunStatus::Voting);
8991 state.base_sync = Some(crate::run::BaseSync {
8992 tip: "base-tip".into(),
8993 behind: 3,
8994 attempts: 4,
8995 conflict: Some("base stopped".into()),
8996 already_in: None,
8997 });
8998 state.record_base_sync_stop("candidate-head");
8999 let mut t = task();
9000 t.start(state.id.clone());
9001 let verdict = Verdict {
9002 status: RunStatus::Blocked,
9003 left_pr: false,
9004 quota_hit: false,
9005 parked: false,
9006 no_viable_candidates: false,
9007 };
9008 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
9009 assert_eq!(
9010 t.status,
9011 TaskStatus::Held,
9012 "unspent task attempts cannot start another competition"
9013 );
9014 let at = t.held_at;
9015 let cause = t.hold_notice_cause.clone();
9016 state.begin_pass();
9017 t.release_operator("test-resume", None);
9018 t.start(state.id.clone());
9019 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
9020 assert_eq!(t.status, TaskStatus::Held);
9021 assert_eq!(t.held_at, at);
9022 assert_eq!(t.hold_notice_cause, cause);
9023 }
9024
9025 #[test]
9026 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
9027 let mut state = run_state(RunStatus::Blocked);
9028 state.gate = vec![CommandOutcome {
9029 command: "cargo test".to_owned(),
9030 code: Some(101),
9031 output_tail: "assertion failed".to_owned(),
9032 duration_ms: 0,
9033 resource_blocked: false,
9034 }];
9035 let verdict = Verdict {
9036 status: RunStatus::Blocked,
9037 left_pr: false,
9038 quota_hit: false,
9039 parked: false,
9040 no_viable_candidates: false,
9041 };
9042
9043 let mut t = task();
9046 t.start("run-1".to_owned());
9047 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
9048 assert_eq!(t.status, TaskStatus::Failed);
9049 assert!(t.diagnostic.is_none());
9050
9051 t.start("run-2".to_owned());
9054 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
9055 assert_eq!(t.status, TaskStatus::Held);
9056 let d = t.diagnostic.expect("a held task must carry its diagnostic");
9057 assert!(d.contains("cargo test"), "{d}");
9058 }
9059
9060 #[test]
9061 fn a_held_task_names_the_open_question_it_is_waiting_on() {
9062 crate::run::pin_test_home();
9067 let home = crate::run::home();
9068 let state = run_state(RunStatus::VerifiedNoop);
9069 let mut q = ask::Question::new(
9070 state.id.clone(),
9071 "implement".to_owned(),
9072 "impl-A".to_owned(),
9073 "is this really a no-op?".to_owned(),
9074 String::new(),
9075 Vec::new(),
9076 );
9077 Questions::at(home.join("questions")).put(&mut q).unwrap();
9078
9079 let verdict = Verdict {
9080 status: RunStatus::VerifiedNoop,
9081 left_pr: false,
9082 quota_hit: false,
9083 parked: false,
9084 no_viable_candidates: false,
9085 };
9086 let mut t = task();
9087 t.start(state.id.clone());
9088 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
9089
9090 assert_eq!(t.status, TaskStatus::Held);
9091 let reason = t.hold_reason.expect("a held task must record why");
9092 assert!(
9093 reason.starts_with("run ended agent-verified no-op"),
9094 "the original settle reason must survive unchanged: {reason}"
9095 );
9096 assert!(
9097 reason.contains(q.short()),
9098 "the open question's id must be named so the notice is actionable: {reason}"
9099 );
9100 }
9101
9102 #[test]
9103 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
9104 crate::run::pin_test_home();
9105 let state = run_state(RunStatus::VerifiedNoop);
9106
9107 let verdict = Verdict {
9108 status: RunStatus::VerifiedNoop,
9109 left_pr: false,
9110 quota_hit: false,
9111 parked: false,
9112 no_viable_candidates: false,
9113 };
9114 let mut t = task();
9115 t.start(state.id.clone());
9116 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
9117
9118 assert_eq!(t.status, TaskStatus::Held);
9119 assert_eq!(
9120 t.hold_reason.as_deref(),
9121 Some("run ended agent-verified no-op"),
9122 "nothing to append when the question was already answered or never asked"
9123 );
9124 }
9125
9126 #[test]
9127 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
9128 crate::run::pin_test_home();
9129 let mut first = run_state(RunStatus::Blocked);
9130 first.id = "20260101-000000-sup1".to_owned();
9131 first.save().unwrap();
9132 let mut second = run_state(RunStatus::Merged);
9133 second.id = "20260101-000000-sup2".to_owned();
9134 second.save().unwrap();
9135
9136 let mut t = task();
9137 t.runs = vec![first.id.clone(), second.id.clone()];
9138 t.status = TaskStatus::Done;
9139
9140 supersede_prior_runs(&t, &crate::run::home());
9141
9142 assert_eq!(
9143 RunState::load(&first.id).unwrap().status,
9144 RunStatus::Superseded,
9145 "the first attempt's Blocked no longer needs anyone's attention"
9146 );
9147 assert_eq!(
9148 RunState::load(&second.id).unwrap().status,
9149 RunStatus::Merged,
9150 "the run that actually succeeded is left exactly as it was"
9151 );
9152 }
9153
9154 #[test]
9155 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
9156 crate::run::pin_test_home();
9162 let mut first = run_state(RunStatus::Blocked);
9163 first.id = "20260101-000000-sup9".to_owned();
9164 first.driver_pid = Some(std::process::id());
9167 first.driver_started_at = Some(
9168 crate::proc::process_started_at(std::process::id())
9169 .expect("this test process's own start time must be queryable"),
9170 );
9171 first.save().unwrap();
9172 let mut second = run_state(RunStatus::Merged);
9173 second.id = "20260101-000000-supa".to_owned();
9174 second.save().unwrap();
9175
9176 let mut t = task();
9177 t.runs = vec![first.id.clone(), second.id.clone()];
9178 t.status = TaskStatus::Done;
9179
9180 supersede_prior_runs(&t, &crate::run::home());
9181
9182 assert_eq!(
9183 RunState::load(&first.id).unwrap().status,
9184 RunStatus::Blocked,
9185 "a live driver_pid means something is still actually working this run, \
9186 even though no daemon claims it - rewriting under it would just be \
9187 undone the next time that process saves"
9188 );
9189 }
9190
9191 #[test]
9192 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
9193 let dir = tempfile::tempdir().unwrap();
9199 let home = dir.path().to_path_buf();
9200 let queue = Queue::at(dir.path().join("queue"));
9201
9202 let mut first = run_state(RunStatus::Blocked);
9203 first.id = "20260101-000000-supd".to_owned();
9204 first.driver_pid = Some(std::process::id());
9205 first.driver_started_at = Some(
9206 crate::proc::process_started_at(std::process::id())
9207 .expect("this test process's own start time must be queryable"),
9208 );
9209 first.save_under(&home).unwrap();
9210 let mut second = run_state(RunStatus::Merged);
9211 second.id = "20260101-000000-supe".to_owned();
9212 second.save_under(&home).unwrap();
9213
9214 let mut t = task();
9215 t.runs = vec![first.id.clone(), second.id.clone()];
9216 t.status = TaskStatus::Done;
9217 queue.put(&mut t).unwrap();
9218
9219 resweep_superseded_attempts(&queue, &home);
9220 assert_eq!(
9221 RunState::load_under(&first.id, &home).unwrap().status,
9222 RunStatus::Blocked,
9223 "still live on the first pass, so still untouched"
9224 );
9225
9226 let mut stale = RunState::load_under(&first.id, &home).unwrap();
9232 stale.driver_started_at = Some("1".to_owned());
9233 stale.save_under(&home).unwrap();
9234
9235 resweep_superseded_attempts(&queue, &home);
9236 assert_eq!(
9237 RunState::load_under(&first.id, &home).unwrap().status,
9238 RunStatus::Superseded,
9239 "the second pass catches up what the first one correctly skipped"
9240 );
9241 }
9242
9243 #[test]
9244 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
9245 crate::run::pin_test_home();
9246 let mut first = run_state(RunStatus::Blocked);
9247 first.id = "20260101-000000-sup3".to_owned();
9248 first.save().unwrap();
9249 let mut second = run_state(RunStatus::Blocked);
9250 second.id = "20260101-000000-sup4".to_owned();
9251 second.save().unwrap();
9252
9253 let mut t = task();
9254 t.runs = vec![first.id.clone(), second.id.clone()];
9255 t.status = TaskStatus::Failed;
9259
9260 supersede_prior_runs(&t, &crate::run::home());
9261
9262 assert_eq!(
9263 RunState::load(&first.id).unwrap().status,
9264 RunStatus::Blocked
9265 );
9266 assert_eq!(
9267 RunState::load(&second.id).unwrap().status,
9268 RunStatus::Blocked
9269 );
9270 }
9271
9272 #[test]
9273 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
9274 crate::run::pin_test_home();
9278 let mut first = run_state(RunStatus::Blocked);
9279 first.id = "20260101-000000-sup5".to_owned();
9280 first.save().unwrap();
9281
9282 let mut t = task();
9283 t.runs = vec![first.id.clone()];
9284 t.status = TaskStatus::Done;
9285
9286 supersede_prior_runs(&t, &crate::run::home());
9287
9288 assert_eq!(
9289 RunState::load(&first.id).unwrap().status,
9290 RunStatus::Blocked,
9291 "a single-attempt task has no earlier run to supersede"
9292 );
9293 }
9294
9295 #[test]
9296 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
9297 crate::run::pin_test_home();
9304 let mut first = run_state(RunStatus::Blocked);
9305 first.id = "20260101-000000-supb".to_owned();
9306 first.save().unwrap();
9307 let mut second = run_state(RunStatus::Failed);
9308 second.id = "20260101-000000-supc".to_owned();
9309 second.save().unwrap();
9310
9311 let mut t = task();
9312 t.runs = vec![first.id.clone(), second.id.clone()];
9313 t.status = TaskStatus::Done;
9314
9315 supersede_prior_runs(&t, &crate::run::home());
9316
9317 assert_eq!(
9318 RunState::load(&first.id).unwrap().status,
9319 RunStatus::Blocked,
9320 "the task's last attempt never landed, so there is nothing here \
9321 actually superseding it"
9322 );
9323 }
9324
9325 #[test]
9326 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
9327 crate::run::pin_test_home();
9331 let mut failed = run_state(RunStatus::Failed);
9332 failed.id = "20260101-000000-sup6".to_owned();
9333 failed.save().unwrap();
9334 let mut noop = run_state(RunStatus::VerifiedNoop);
9335 noop.id = "20260101-000000-sup7".to_owned();
9336 noop.save().unwrap();
9337 let mut winner = run_state(RunStatus::Ready);
9338 winner.id = "20260101-000000-sup8".to_owned();
9339 winner.save().unwrap();
9340
9341 let mut t = task();
9342 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
9343 t.status = TaskStatus::Done;
9344
9345 supersede_prior_runs(&t, &crate::run::home());
9346
9347 assert_eq!(
9348 RunState::load(&failed.id).unwrap().status,
9349 RunStatus::Failed
9350 );
9351 assert_eq!(
9352 RunState::load(&noop.id).unwrap().status,
9353 RunStatus::VerifiedNoop
9354 );
9355 }
9356
9357 fn approval_question(run: &str) -> ask::Question {
9358 ask::Question::new(
9359 run.to_owned(),
9360 land::APPROVAL_NODE.to_owned(),
9361 "land".to_owned(),
9362 "merge?".to_owned(),
9363 String::new(),
9364 vec!["merge".to_owned(), "hold".to_owned()],
9365 )
9366 }
9367
9368 #[test]
9369 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
9370 crate::run::pin_test_home();
9371 let mut state = run_state(RunStatus::Landing);
9372 state.id = "20260101-000000-fre1".to_owned();
9373 state.parked = true;
9374 state.save().unwrap();
9375 ask::Questions::open()
9376 .put(&mut approval_question(&state.id))
9377 .unwrap();
9378
9379 let mut t = task();
9380 t.runs.push(state.id.clone());
9381 assert_eq!(
9382 land_resume_state(&t),
9383 LandResume::StillWaiting,
9384 "nobody has answered and the timeout has not passed"
9385 );
9386 }
9387
9388 #[test]
9389 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
9390 crate::run::pin_test_home();
9391 let mut state = run_state(RunStatus::Gating);
9392 state.id = "20260101-000000-gt01".to_owned();
9393 state.parked = true;
9394 state.config.graph.answer_timeout = 60;
9395 state.github_text = Some(crate::run::GithubTextAsk {
9396 rewrite: None,
9397 fingerprint: "f".into(),
9398 title: true,
9399 body: false,
9400 categories: vec!["title-language".into()],
9401 question: None,
9402 asks: 1,
9403 resolved: false,
9404 chosen_title: None,
9405 choice: None,
9406 publication: None,
9407 });
9408 state.save().unwrap();
9409 let store = ask::Questions::open();
9410 let mut q = ask::Question::new(
9411 state.id.clone(),
9412 crate::github_text::ASK_NODE.to_owned(),
9413 crate::github_text::ASK_SEAT.to_owned(),
9414 "s".into(),
9415 String::new(),
9416 vec!["use fallback".into()],
9417 );
9418 store.put(&mut q).unwrap();
9419 let mut t = task();
9420 t.runs.push(state.id.clone());
9421 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
9422 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9423 store.put(&mut q).unwrap();
9424 assert_eq!(land_resume_state(&t), LandResume::Ready);
9425 assert!(!store.get(&q.id).unwrap().status.open());
9426 state.github_text.as_mut().unwrap().resolved = true;
9428 state.save().unwrap();
9429 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
9430 }
9431
9432 #[test]
9433 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
9434 crate::run::pin_test_home();
9439 let mut state = run_state(RunStatus::Landing);
9440 state.id = "20260101-000000-exp1".to_owned();
9441 state.parked = true;
9442 state.config.graph.answer_timeout = 60;
9443 state.save().unwrap();
9444
9445 let store = ask::Questions::open();
9446 let mut q = approval_question(&state.id);
9447 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9448 store.put(&mut q).unwrap();
9449
9450 let mut t = task();
9451 t.runs.push(state.id.clone());
9452 assert_eq!(
9453 land_resume_state(&t),
9454 LandResume::Ready,
9455 "an expired question must not be waited on forever"
9456 );
9457
9458 let after = store.get(&q.id).unwrap();
9459 assert!(
9460 !after.status.open(),
9461 "the question is abandoned, not silently ignored"
9462 );
9463 assert!(
9464 after.resolution().is_none(),
9465 "an abandoned question is not read as a decision"
9466 );
9467 }
9468
9469 #[test]
9470 fn reclaim_settles_a_running_task_against_its_last_run() {
9471 let mut t = task();
9472 t.start("20260904-000000-4043".to_owned());
9473 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
9474 assert_eq!(
9475 t.status,
9476 TaskStatus::Done,
9477 "a run that actually finished must not stay `running` forever"
9478 );
9479 }
9480
9481 #[test]
9482 fn reclaim_preserves_dispatched_inquiries_until_a_committed_pass() {
9483 let dir = tempfile::tempdir().unwrap();
9484 let queue = Queue::at(dir.path().join("queue"));
9485 let mut task = task();
9486 let mut run = run_state(RunStatus::Blocked);
9487 run.reserve_under(dir.path()).unwrap();
9488 run.driver_exited = true;
9489 run.save_under(dir.path()).unwrap();
9490 queue.put(&mut task).unwrap();
9491 let stale = task.clone();
9492 task.resume_override = Some(crate::queue::OperatorResume {
9493 question_id: "pending-resume".into(),
9494 at: Timestamp::now(),
9495 conductor_rehold: None,
9496 forced: true,
9497 pinned_run: Some(run.id.clone()),
9498 });
9499 task.start(run.id.clone());
9500 queue.put(&mut task).unwrap();
9501 let mut recovered = queue.get(&task.id).unwrap();
9503 let disk = RunState::load_under(&run.id, dir.path()).unwrap();
9504 reclaim(&mut recovered, Some(disk), 2, "en");
9505 queue.put(&mut recovered).unwrap();
9506 assert!(recovered.resume_override.is_none());
9507 assert_eq!(
9508 recovered.dispatched_resumes.get("pending-resume"),
9509 Some(&run.id)
9510 );
9511 let mut stale = stale;
9512 queue.put(&mut stale).unwrap();
9513 assert_eq!(
9514 stale.dispatched_resumes, recovered.dispatched_resumes,
9515 "a stale task writer cannot erase dispatch ownership"
9516 );
9517 let mut resumed = RunState::load_under(&run.id, dir.path()).unwrap();
9518 assert!(resumed.decision_starts.is_empty());
9519 resumed.carry_dispatched_resumes(&queue.get(&task.id).unwrap());
9520 assert!(
9521 resumed.decision_starts.is_empty(),
9522 "handoff is not execution proof"
9523 );
9524 resumed.begin_pass();
9525 resumed.save_under(dir.path()).unwrap();
9526 let mut disk = RunState::load_under(&run.id, dir.path()).unwrap();
9527 assert_eq!(disk.decision_starts.get("pending-resume"), Some(&0));
9528 disk.carry_dispatched_resumes(&recovered);
9529 disk.begin_pass();
9530 assert_eq!(
9531 disk.decision_starts.get("pending-resume"),
9532 Some(&0),
9533 "later execution must preserve the original pass receipt"
9534 );
9535 }
9536
9537 #[test]
9538 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
9539 let mut t = task();
9540 t.start("20260904-000000-4043".to_owned());
9541 t.last_error = Some("earlier trouble".to_owned());
9542 let mut state = run_state(RunStatus::Judging);
9543 state.parked = true;
9544 reclaim(&mut t, Some(state), 1, "en");
9545 assert_eq!(t.status, TaskStatus::Parked);
9546 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
9547 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9548 assert!(t.park_reason.is_some());
9549 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
9550 assert!(!t.fresh_start);
9551 assert!(t.status.runnable());
9552 }
9553
9554 #[test]
9555 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
9556 let mut t = task();
9557 t.start("20260904-000000-4043".to_owned());
9558 t.last_error = Some("earlier trouble".to_owned());
9559 settle(
9560 &mut t,
9561 Verdict {
9562 status: RunStatus::Judging,
9563 left_pr: false,
9564 quota_hit: false,
9565 parked: true,
9566 no_viable_candidates: false,
9567 },
9568 "parked after `judging`",
9569 1,
9570 );
9571 assert_eq!(t.status, TaskStatus::Parked);
9572 assert_eq!(t.attempts, 0);
9573 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9574 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
9575 assert_eq!(t.schema, crate::queue::SCHEMA);
9576 assert!(
9577 crate::notices::task_held(&t).is_none(),
9578 "a park is not news"
9579 );
9580 t.start("20260904-000000-4043".to_owned());
9581 assert_eq!(t.park_reason, None);
9582 }
9583
9584 #[test]
9585 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
9586 let mut t = task();
9590 t.start("20260904-000000-4043".to_owned());
9591 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
9592 assert_eq!(t.status, TaskStatus::Failed);
9593 assert!(t.status.runnable());
9594 }
9595
9596 #[test]
9597 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
9598 let mut t = task();
9599 t.start("20260904-000000-4043".to_owned());
9600 assert_eq!(t.attempts, 1);
9601 let mut run = run_state(RunStatus::Implementing);
9602 run.id = "20260904-000000-4043".to_owned();
9603 run.begin_pass();
9604 let cause = crate::run::ParkCause::Upgrade {
9605 from: "0.121.0".to_owned(),
9606 to: Some("0.121.1".to_owned()),
9607 };
9608 run.end_pass_parked(cause.clone());
9609 run.parked = true;
9610 let verdict = Verdict {
9611 status: run.status,
9612 left_pr: false,
9613 quota_hit: false,
9614 parked: true,
9615 no_viable_candidates: false,
9616 };
9617 let mut orphan = t.clone();
9620 reclaim(&mut orphan, Some(run.clone()), 2, "en");
9621 assert_eq!(orphan.status, TaskStatus::Parked);
9622 assert_eq!(orphan.attempts, 0);
9623 assert_eq!(
9624 orphan.upgrade_resume_key(),
9625 Some((run.passes[0].started_at, orphan.id.as_str()))
9626 );
9627 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
9628 assert_eq!(t.attempts, 0);
9629 assert_eq!(t.passes.len(), 1);
9630 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9631 assert_eq!(t.passes[0].parked, Some(cause));
9632 assert_eq!(t.passes[0].pass_index, 0);
9633 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
9634 assert!(t.upgrade_resume_key().is_some());
9635
9636 t.start("20260904-000000-4043".to_owned());
9639 run.begin_pass();
9640 run.status = RunStatus::Failed;
9641 let verdict = Verdict {
9642 status: RunStatus::Failed,
9643 left_pr: false,
9644 quota_hit: false,
9645 parked: false,
9646 no_viable_candidates: false,
9647 };
9648 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
9649 assert_eq!(t.passes.len(), 2);
9650 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9651 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
9652 assert_eq!(t.passes[1].pass_index, 1);
9653 assert_eq!(t.attempts, 1);
9654 }
9655
9656 #[test]
9657 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
9658 let mut t = task();
9659 t.start("20260904-000000-4043".to_owned());
9660 reclaim(&mut t, None, 2, "en");
9661 assert_eq!(t.status, TaskStatus::Held);
9662 assert!(
9663 t.last_error
9664 .as_deref()
9665 .is_some_and(|e| e.contains("running")),
9666 "the operator needs to know why this task was held"
9667 );
9668 }
9669
9670 #[test]
9671 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
9672 let dir = tempfile::tempdir().unwrap();
9673 let queue = Queue::at(dir.path().to_path_buf());
9674
9675 let mut orphaned = task();
9677 orphaned.id = "20260904-000000-orph".to_owned();
9678 orphaned.status = TaskStatus::Running;
9679 orphaned.attempts = 1;
9680 queue.put(&mut orphaned).unwrap();
9681
9682 let mut alive = task();
9683 alive.id = "20260904-000000-live".to_owned();
9684 alive.status = TaskStatus::Running;
9685 alive.attempts = 1;
9686 queue.put(&mut alive).unwrap();
9687 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
9688
9689 let mut queued = task();
9690 queued.id = "20260904-000000-wait".to_owned();
9691 queue.put(&mut queued).unwrap();
9692
9693 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9694 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
9695
9696 assert_eq!(
9697 queue.get(&orphaned.id).unwrap().status,
9698 TaskStatus::Held,
9699 "nothing was driving it and there was no run to recover"
9700 );
9701 assert_eq!(
9702 queue.get(&alive.id).unwrap().status,
9703 TaskStatus::Running,
9704 "a live claim must protect the task it belongs to"
9705 );
9706 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9707 }
9708
9709 fn read_run_under(home: &Path, id: &str) -> RunState {
9715 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9716 serde_json::from_str(&body).unwrap()
9717 }
9718
9719 #[test]
9720 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9721 let dir = tempfile::tempdir().unwrap();
9722 let home = dir.path().to_path_buf();
9723 let now = Timestamp::now();
9724 let overrun_seat = || crate::run::ActiveSeat {
9725 node: "implement".to_owned(),
9726 started_at: now - jiff::SignedDuration::new(21_000, 0),
9727 timeout_secs: 3_600,
9728 attempt: 0,
9729 task: None,
9730 command: None,
9731 index: None,
9732 total: None,
9733 };
9734
9735 let mut dead = run_state(RunStatus::Implementing);
9736 dead.id = "20260101-000000-dead".to_owned();
9737 dead.active.insert("impl-A".to_owned(), overrun_seat());
9738 dead.driver_pid = Some(4242);
9741 dead.save_under(&home).unwrap();
9742
9743 let mut alive = run_state(RunStatus::Implementing);
9746 alive.id = "20260101-000000-aliv".to_owned();
9747 alive.active.insert("impl-A".to_owned(), overrun_seat());
9748 alive.save_under(&home).unwrap();
9749 let mut status = Status::new();
9750 status.current = vec![Current {
9751 worker: None,
9752 task: "20260101-000000-task".to_owned(),
9753 run: alive.id.clone(),
9754 }];
9755 write_status_to(&home.join("daemon.json"), &status).unwrap();
9756
9757 let questions = Questions::at(home.join("questions"));
9761 let mut q = ask::Question::new(
9762 dead.id.clone(),
9763 "implement".to_owned(),
9764 "impl-A".to_owned(),
9765 "Which storage backend?".to_owned(),
9766 String::new(),
9767 vec!["SQLite".to_owned(), "Redis".to_owned()],
9768 );
9769 questions.put(&mut q).unwrap();
9770
9771 let abandoned = reclaim_abandoned_runs_with(
9772 &home,
9773 now,
9774 |pid| if pid == 4242 { Some(false) } else { None },
9775 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9776 );
9777 assert_eq!(abandoned, vec![dead.id.clone()]);
9778
9779 let reloaded = read_run_under(&home, &dead.id);
9780 assert_eq!(reloaded.status, RunStatus::Failed);
9781 assert!(reloaded.active.is_empty());
9782 assert!(
9783 !questions.get(&q.id).unwrap().status.open(),
9784 "the failed run's own open question must be settled in the same pass"
9785 );
9786
9787 let still_alive = read_run_under(&home, &alive.id);
9788 assert_eq!(
9789 still_alive.status,
9790 RunStatus::Implementing,
9791 "a live daemon's claim protects it"
9792 );
9793 assert!(!still_alive.active.is_empty());
9794 }
9795
9796 #[test]
9806 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9807 let dir = tempfile::tempdir().unwrap();
9808 let home = dir.path().to_path_buf();
9809 let now = Timestamp::now();
9810
9811 let mut manual = run_state(RunStatus::Reviewing);
9812 manual.id = "20260101-000000-manl".to_owned();
9813 manual.active.insert(
9814 "review-1".to_owned(),
9815 crate::run::ActiveSeat {
9816 node: "review".to_owned(),
9817 started_at: now - jiff::SignedDuration::new(21_000, 0),
9818 timeout_secs: 3_600,
9819 attempt: 0,
9820 task: None,
9821 command: None,
9822 index: None,
9823 total: None,
9824 },
9825 );
9826 manual.driver_pid = Some(4242);
9830 manual.driver_started_at = Some("1790000000".to_owned());
9831 manual.save_under(&home).unwrap();
9832
9833 let abandoned = reclaim_abandoned_runs_with(
9834 &home,
9835 now,
9836 |pid| if pid == 4242 { Some(true) } else { None },
9837 |pid| {
9838 if pid == 4242 {
9839 Some("1790000000".to_owned())
9840 } else {
9841 None
9842 }
9843 },
9844 );
9845 assert!(
9846 abandoned.is_empty(),
9847 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9848 );
9849
9850 let reloaded = read_run_under(&home, &manual.id);
9851 assert_eq!(reloaded.status, RunStatus::Reviewing);
9852 assert!(!reloaded.active.is_empty());
9853 }
9854
9855 #[test]
9856 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9857 let dir = tempfile::tempdir().unwrap();
9858 let queue = Queue::at(dir.path().to_path_buf());
9859 let mut only = task();
9860 queue.put(&mut only).unwrap();
9861
9862 let _elsewhere = queue.claim(&only.id).unwrap();
9863 let candidates = runnable(&queue);
9864 assert_eq!(candidates.len(), 1, "the task is still runnable");
9865 assert!(
9866 queue.claim(&candidates[0].id).is_err(),
9867 "the loop cannot take a claim somebody else holds"
9868 );
9869
9870 let after = queue.get(&only.id).unwrap();
9871 assert_eq!(after.status, TaskStatus::Queued);
9872 assert_eq!(
9873 after.attempts, 0,
9874 "losing the race is not an attempt at the task"
9875 );
9876 assert_eq!(after.last_error, None);
9877 }
9878
9879 #[test]
9880 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9881 let dir = tempfile::tempdir().unwrap();
9882 let path = dir.path().join("daemon.json");
9883
9884 let mut status = Status::new();
9885 status.idle = false;
9886 status.completed = 7;
9887 status.current = vec![Current {
9888 worker: None,
9889 task: "20260902-000000-t111".to_owned(),
9890 run: "20260902-000001-r111".to_owned(),
9891 }];
9892 write_status_to(&path, &status).unwrap();
9893 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9894 assert_eq!(first.schema, SCHEMA);
9895 assert_eq!(first.pid, std::process::id());
9896 assert!(!first.idle);
9897 assert_eq!(first.completed, 7);
9898 assert_eq!(first.current, status.current);
9899 assert!(
9900 !path.with_extension("json.tmp").exists(),
9901 "the temp file is renamed, not left behind"
9902 );
9903
9904 std::thread::sleep(Duration::from_millis(5));
9905 status.updated_at = Timestamp::now();
9906 status.polls = 3;
9907 write_status_to(&path, &status).unwrap();
9908 let second: Status =
9909 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9910 assert!(
9911 second.updated_at > first.updated_at,
9912 "a reader can only detect staleness if the heartbeat moves"
9913 );
9914 assert_eq!(
9915 second.started_at, first.started_at,
9916 "the start time is not a heartbeat"
9917 );
9918 assert_eq!(second.polls, 3);
9919 }
9920
9921 #[test]
9922 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9923 let dir = tempfile::tempdir().unwrap();
9924
9925 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9926
9927 let mut status = Status::new();
9928 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9929 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9930 let stale = read_status(dir.path()).unwrap();
9931 assert!(
9932 !stale.running(Timestamp::now()),
9933 "a minute without a heartbeat is a dead daemon, not a busy one"
9934 );
9935 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9936
9937 status.updated_at = Timestamp::now();
9938 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9939 let fresh = read_status(dir.path()).unwrap();
9940 assert!(fresh.running(Timestamp::now()));
9941 }
9942
9943 #[test]
9944 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9945 let dir = tempfile::tempdir().unwrap();
9946 let now = Timestamp::now();
9947 let mine = "20260903-080619-01c2";
9948
9949 assert!(
9950 !is_working_on(dir.path(), mine, now),
9951 "no status file means nobody is working on anything"
9952 );
9953
9954 let mut status = Status::new();
9955 status.current = vec![Current {
9956 worker: None,
9957 task: "20260903-080340-0167".to_owned(),
9958 run: mine.to_owned(),
9959 }];
9960 status.updated_at = now;
9961 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9962 assert!(is_working_on(dir.path(), mine, now));
9963 assert!(
9964 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9965 "a daemon busy with one run is not working on another"
9966 );
9967
9968 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9971 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9972 assert!(
9973 !is_working_on(dir.path(), mine, now),
9974 "a stale heartbeat is a dead daemon, so its run is a leftover"
9975 );
9976 }
9977
9978 #[test]
9979 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9980 let dir = tempfile::tempdir().unwrap();
9981 let now = Timestamp::now();
9982
9983 assert!(
9984 !is_working_on_short(dir.path(), "01c2", now),
9985 "no status file means nobody is working on anything"
9986 );
9987
9988 let mut status = Status::new();
9989 status.current = vec![Current {
9990 worker: None,
9991 task: "20260903-080340-0167".to_owned(),
9992 run: "20260903-080619-01c2".to_owned(),
9993 }];
9994 status.updated_at = now;
9995 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9996 assert!(
9997 is_working_on_short(dir.path(), "01c2", now),
9998 "the run's short id is the last block of its full id"
9999 );
10000 assert!(
10001 !is_working_on_short(dir.path(), "3cbf", now),
10002 "a daemon busy with one worktree bay is not working on another"
10003 );
10004 }
10005
10006 #[test]
10007 fn a_newer_status_file_still_yields_a_reading() {
10008 let dir = tempfile::tempdir().unwrap();
10009 std::fs::write(
10012 dir.path().join("daemon.json"),
10013 serde_json::json!({
10014 "schema": 2,
10015 "updated_at": Timestamp::now().to_string(),
10016 "idle": true,
10017 "surprise": { "nested": [1, 2, 3] },
10018 })
10019 .to_string(),
10020 )
10021 .unwrap();
10022
10023 let reading = read_status(dir.path()).expect("a forward-compatible read");
10024 assert!(reading.running(Timestamp::now()));
10025 assert!(reading.idle);
10026 assert!(reading.current.is_empty());
10027 }
10028
10029 #[test]
10030 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
10031 let dir = tempfile::tempdir().unwrap();
10037 std::fs::write(
10038 dir.path().join("daemon.json"),
10039 serde_json::json!({
10040 "schema": 1,
10041 "pid": 4242,
10042 "updated_at": Timestamp::now().to_string(),
10043 "idle": false,
10044 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
10045 "completed": 3,
10046 "polls": 9,
10047 })
10048 .to_string(),
10049 )
10050 .unwrap();
10051
10052 let reading = read_status(dir.path()).expect("an older shape must still parse");
10053 assert!(reading.running(Timestamp::now()));
10054 assert_eq!(
10055 reading.current,
10056 vec![Current {
10057 worker: None,
10058 task: "20260902-140501-aaaa".to_owned(),
10059 run: "20260902-140502-bbbb".to_owned(),
10060 }]
10061 );
10062 }
10063
10064 #[test]
10065 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
10066 let dir = tempfile::tempdir().unwrap();
10067 std::fs::write(
10068 dir.path().join("daemon.json"),
10069 serde_json::json!({
10070 "schema": 1,
10071 "updated_at": Timestamp::now().to_string(),
10072 "idle": true,
10073 "current": null,
10074 })
10075 .to_string(),
10076 )
10077 .unwrap();
10078 let with_null = read_status(dir.path()).expect("null must still parse");
10079 assert!(with_null.current.is_empty());
10080
10081 std::fs::write(
10082 dir.path().join("daemon.json"),
10083 serde_json::json!({
10084 "schema": 1,
10085 "updated_at": Timestamp::now().to_string(),
10086 "idle": true,
10087 })
10088 .to_string(),
10089 )
10090 .unwrap();
10091 let absent = read_status(dir.path()).expect("a missing field must still parse");
10092 assert!(absent.current.is_empty());
10093 }
10094
10095 #[test]
10096 fn a_task_without_a_repository_runs_in_the_daemons_default() {
10097 let fallback = Path::new("/default");
10098 let mut blank = task();
10099 blank.repo = PathBuf::new();
10100 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
10101 let mut dot = task();
10102 dot.repo = PathBuf::from(".");
10103 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
10104 assert_eq!(
10105 repo_for(&task(), fallback),
10106 PathBuf::from("/repo"),
10107 "a task that names a repository keeps it"
10108 );
10109 }
10110
10111 #[test]
10112 fn task_implementers_edit_inherits_daemon_config_and_merge() {
10113 let dir = tempfile::tempdir().unwrap();
10114 let path = dir.path().join("explicit.toml");
10115 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
10116 let opts = Opts {
10117 config: Some(path),
10118 merge: Some("none".into()),
10119 ..Default::default()
10120 };
10121 let mut t = task();
10122 t.set_implementers(
10123 2,
10124 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10125 .unwrap(),
10126 )
10127 .unwrap();
10128 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
10129 assert_eq!(cfg.graph.implementers, 2);
10130 assert_eq!(cfg.graph.judges, 1);
10131 assert_eq!(cfg.merge.mode, MergeMode::None);
10132 t.set_implementers(1, &cfg).unwrap();
10133 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
10134 apply_task_implementers(&mut cfg, &t);
10135 assert_eq!(cfg.graph.implementers, 1);
10136 assert_eq!(cfg.graph.judges, 1);
10137 assert_eq!(cfg.merge.mode, MergeMode::None);
10138 }
10139
10140 #[test]
10141 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
10142 let dir = tempfile::tempdir().unwrap();
10143 let repo = dir.path().join("repo");
10144 std::fs::create_dir(&repo).unwrap();
10145 std::fs::write(
10146 repo.join("magi.toml"),
10147 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
10148 )
10149 .unwrap();
10150 let daemon_config = dir.path().join("daemon.toml");
10151 std::fs::write(
10152 &daemon_config,
10153 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
10154 )
10155 .unwrap();
10156 let opts = Opts {
10157 config: Some(daemon_config),
10158 merge: Some("pr".into()),
10159 ..Default::default()
10160 };
10161 let q = Queue::at(dir.path().join("queue"));
10162 let validation: Config =
10163 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10164 .unwrap();
10165 for record in ["{}", r#"{"candidates":2}"#] {
10167 let overrides = serde_json::from_str(record).unwrap();
10168 let filing = crate::direct::Filing {
10169 instruction: "work".into(),
10170 title: "direct".into(),
10171 repo: repo.clone(),
10172 source: crate::queue::Source::Human,
10173 solo: false,
10174 overrides,
10175 review_of: None,
10176 sources: Default::default(),
10177 };
10178 let filed =
10179 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
10180 let crate::direct::Filed::Standalone {
10181 mut task,
10182 claim: _claim,
10183 } = filed
10184 else {
10185 panic!("local filing")
10186 };
10187 let before = prepare_for(&repo, &opts, &task).unwrap();
10188 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
10189 assert_eq!(
10190 before.merge.mode,
10191 MergeMode::None,
10192 "daemon --merge must not leak"
10193 );
10194 for n in [2, 1, 2] {
10195 task.set_implementers(n, &validation).unwrap();
10196 q.put(&mut task).unwrap();
10197 task = q.get(&task.id).unwrap();
10198 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
10199 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
10200 apply_task_implementers(&mut cfg, &task);
10201 assert_eq!(cfg.graph.implementers, usize::from(n));
10202 assert_eq!(cfg.graph.judges, 2);
10203 assert_eq!(cfg.merge.mode, MergeMode::None);
10204 }
10205 }
10206 }
10207
10208 #[test]
10209 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
10210 let cfg: Config =
10211 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10212 .unwrap();
10213 let mut t = task();
10214 t.review_of = Some("review-target".into());
10215 t.hold_manual(Some("retry".into()));
10216 let mut previous = RunState::new(
10217 PathBuf::from("/repo"),
10218 "main".into(),
10219 "abc1234".into(),
10220 "review".into(),
10221 cfg.clone(),
10222 );
10223 previous.config.graph.implementers = 2;
10224 for count in [1, 3] {
10225 t.set_implementers(count, &cfg).unwrap();
10226 t.release();
10227 assert_eq!(t.review_of.as_deref(), Some("review-target"));
10228 for proposed in [
10229 Starter::Resume(previous.id.clone()),
10230 Starter::Review("recovery-target".into()),
10231 ] {
10232 assert_eq!(
10233 starter_with_implementers(&t, Some(&previous), proposed),
10234 Starter::Review("review-target".into())
10235 );
10236 }
10237 let mut configured = cfg.clone();
10238 apply_task_implementers(&mut configured, &t);
10239 assert_eq!(configured.graph.implementers, usize::from(count));
10240 assert_eq!(previous.config.graph.implementers, 2);
10241 }
10242 }
10243
10244 #[test]
10245 fn task_implementers_dispatch_and_resume_agree() {
10246 let mut t = task();
10247 let cfg: crate::config::Config =
10248 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10249 .unwrap();
10250 let mut run = RunState::new(
10251 PathBuf::from("/repo"),
10252 "main".into(),
10253 "abc1234".into(),
10254 "work".into(),
10255 cfg.clone(),
10256 );
10257 run.config.graph.implementers = 2;
10258 for n in [2, 1, 3, 2] {
10259 t.set_implementers(n, &cfg).unwrap();
10260 let mut dispatched = cfg.clone();
10261 apply_task_implementers(&mut dispatched, &t);
10262 assert_eq!(dispatched.graph.implementers, usize::from(n));
10263 assert_eq!(implementers_match(&t, &run), n == 2);
10264 for proposed in [
10265 Starter::Resume(run.id.clone()),
10266 Starter::Review("old-branch".into()),
10267 ] {
10268 let expected = if n == 2 {
10269 proposed.clone()
10270 } else {
10271 Starter::Start
10272 };
10273 assert_eq!(
10274 starter_with_implementers(&t, Some(&run), proposed),
10275 expected
10276 );
10277 }
10278 assert_eq!(
10279 run.config.graph.implementers, 2,
10280 "history must not be rewritten"
10281 );
10282 }
10283 t.solo = false;
10284 t.overrides = None;
10285 assert!(implementers_match(&t, &run));
10286 }
10287
10288 #[test]
10289 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
10290 let mut solo_cfg = Config::default();
10296 solo_cfg.graph.implementers = 3;
10297 let mut solo_task = task();
10298 solo_task.solo = true;
10299 apply_task_implementers(&mut solo_cfg, &solo_task);
10300 assert_eq!(solo_cfg.graph.implementers, 1);
10301
10302 let mut plain_cfg = Config::default();
10303 plain_cfg.graph.implementers = 3;
10304 let plain_task = task();
10305 assert!(!plain_task.solo);
10306 apply_task_implementers(&mut plain_cfg, &plain_task);
10307 assert_eq!(
10308 plain_cfg.graph.implementers, 3,
10309 "a task that did not ask to run alone keeps the config's candidates"
10310 );
10311 }
10312
10313 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
10314 QuotaLoss {
10315 cause: crate::agent::QuotaCause::UsageLimit,
10316 agent: String::new(),
10317 seat: seat.into(),
10318 node: "judge".into(),
10319 at: at.parse().unwrap(),
10320 reset: reset.map(str::to_string),
10321 }
10322 }
10323
10324 #[test]
10325 fn authentication_losses_never_arm_a_cooldown_even_with_reset_hints() {
10326 let now = Timestamp::now();
10327 for reset in [None, Some("in 1h")] {
10328 let auth = crate::agent::Quota {
10329 cause: crate::agent::QuotaCause::Authentication,
10330 agent: "alpha".into(),
10331 reset: reset.map(str::to_owned),
10332 };
10333 let loss = QuotaLoss::from_report("advisor-1".into(), "advise", Some(&auth));
10334 assert_eq!(cooldown_until(&[loss], now), None);
10335 }
10336 }
10337
10338 #[test]
10339 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
10340 let old: Vec<QuotaLoss> = (1..=4)
10341 .map(|i| {
10342 loss(
10343 &format!("judge-{i}"),
10344 "2026-09-23T05:23:00Z",
10345 Some("2:40pm (Asia/Tokyo)"),
10346 )
10347 })
10348 .collect();
10349 let fresh = losses_this_attempt(&old, &old);
10350 assert!(fresh.is_empty());
10351 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
10352 }
10354
10355 #[test]
10356 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
10357 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
10358 let now = Timestamp::now();
10359 let mut after = old.clone();
10360 after.push(loss("judge-2", &now.to_string(), None));
10361 let fresh = losses_this_attempt(&old, &after);
10362 assert_eq!(fresh, vec![after[1].clone()]);
10363 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
10364 assert_eq!(
10365 until,
10366 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
10367 );
10368 }
10369
10370 #[test]
10371 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
10372 let before = vec![
10375 loss("judge-1", "2026-09-23T05:23:00Z", None),
10376 loss("judge-2", "2026-09-23T05:24:00Z", None),
10377 ];
10378 let after = vec![
10379 loss("judge-2", "2026-09-23T05:24:00Z", None),
10380 loss("judge-1", "2026-09-24T01:00:00Z", None),
10381 ];
10382 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
10383 }
10384
10385 #[test]
10386 fn merge_overrides_are_parsed_or_refused() {
10387 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
10388 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
10389 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
10390 assert!(merge_mode("squash").is_err());
10391 }
10392
10393 #[test]
10394 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
10395 let now = Timestamp::now();
10396 let fallback = Duration::from_secs(300);
10397 let cap = Duration::from_secs(1800);
10398
10399 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
10401
10402 let soon = now + jiff::SignedDuration::from_secs(600);
10404 assert_eq!(
10405 quota_wait(Some(soon), now, fallback, cap),
10406 Duration::from_secs(600)
10407 );
10408
10409 let past = now - jiff::SignedDuration::from_secs(60);
10412 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
10413
10414 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
10417 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
10418 }
10419
10420 #[test]
10421 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
10422 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10423
10424 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
10425 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
10426
10427 let already_past =
10431 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
10432 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
10433
10434 assert!(
10435 parse_reset_hint("session limit reached", now, now).is_none(),
10436 "free text with no recognised shape is not guessed at"
10437 );
10438 assert!(
10439 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
10440 "an unresolvable zone name is not guessed at either"
10441 );
10442 }
10443
10444 #[test]
10445 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
10446 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10447
10448 let at = parse_reset_hint(
10449 "You've hit your usage limit. Visit \
10450 https://chatgpt.com/codex/settings/usage to purchase more \
10451 credits or try again at Sep 19th, 2026 5:10 PM.",
10452 now,
10453 now,
10454 )
10455 .expect("the codex reset wording is a recognised shape");
10456 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
10457
10458 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
10463 .expect("an explicit year needs no rollover");
10464 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
10465
10466 assert!(
10467 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
10468 "a two-digit year is not the documented shape and is not guessed at"
10469 );
10470 assert!(
10471 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
10472 "a four-letter month name is not the documented three-letter abbreviation"
10473 );
10474 assert!(
10475 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
10476 "an explicit zone on the dated shape is a format nobody has \
10477 documented, and is refused rather than guessed at as UTC"
10478 );
10479 }
10480
10481 #[test]
10482 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
10483 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
10484 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
10485
10486 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
10487 assert_eq!(at.as_second() - recorded.as_second(), 3769);
10488
10489 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
10490 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
10491
10492 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
10493 assert!(
10494 parse_reset_hint(bad, now, recorded).is_none(),
10495 "{bad:?} must not be guessed at"
10496 );
10497 }
10498 }
10499
10500 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
10504 let config = dir.join("magi.toml");
10505 std::fs::write(
10506 &config,
10507 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
10508 )
10509 .unwrap();
10510 let opts = Opts {
10511 poll: Duration::from_secs(30),
10512 config: Some(config),
10513 repo: dir.join("repo"),
10517 ..Opts::default()
10518 };
10519 let home = dir.join("home");
10528 let stamps = home.join("service-ticks");
10531 std::fs::create_dir_all(&stamps).unwrap();
10532 for service in ["waiter", "deputies", "release-watch"] {
10533 std::fs::write(
10534 stamps.join(service),
10535 Timestamp::now().as_second().to_string(),
10536 )
10537 .unwrap();
10538 }
10539 for operation in [
10540 "conductor",
10541 "maintenance",
10542 "housekeeping",
10543 "disk-pressure",
10544 "cache",
10545 "fetch",
10546 ] {
10547 std::fs::write(
10548 stamps.join(format!("daemon-{operation}")),
10549 serde_json::to_vec(&Timestamp::now()).unwrap(),
10550 )
10551 .unwrap();
10552 }
10553 let worktrees = dir.join("wt");
10554 (
10555 opts,
10556 Queue::at(dir.join("queue")),
10557 home.join("daemon.json"),
10558 home,
10559 worktrees,
10560 )
10561 }
10562
10563 #[tokio::test]
10564 async fn source_eviction_service_receipts_explain_idle_and_active_skips() {
10565 let dir = tempfile::tempdir().unwrap();
10566 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10567 let cfg = Config::default();
10568 start_service(
10569 &home,
10570 &opts,
10571 &queue,
10572 &worktrees,
10573 "housekeeping",
10574 Some(&cfg),
10575 true,
10576 )
10577 .await
10578 .unwrap();
10579 let receipt = home.join("source-eviction-housekeeping-dispatch.json");
10580 let read = |path: &Path| -> serde_json::Value {
10581 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
10582 };
10583 assert_eq!(read(&receipt)["detail"], "cadence not due");
10584 std::fs::remove_file(home.join("service-ticks/daemon-housekeeping")).unwrap();
10585 start_service(
10586 &home,
10587 &opts,
10588 &queue,
10589 &worktrees,
10590 "housekeeping",
10591 Some(&cfg),
10592 true,
10593 )
10594 .await
10595 .unwrap();
10596 assert_eq!(
10597 read(&receipt)["detail"],
10598 "active work or runnable task queue"
10599 );
10600 assert_eq!(read(&receipt)["version"], env!("CARGO_PKG_VERSION"));
10601 assert_eq!(read(&receipt)["auto_fold"], cfg.disk.auto_fold);
10602 std::fs::remove_file(home.join("service-ticks/daemon-disk-pressure")).unwrap();
10603 start_service(
10604 &home,
10605 &opts,
10606 &queue,
10607 &worktrees,
10608 "disk-pressure",
10609 Some(&cfg),
10610 false,
10611 )
10612 .await
10613 .unwrap();
10614 assert_eq!(
10615 read(&home.join("source-eviction-disk-pressure-dispatch.json"))["detail"],
10616 "no active work; idle housekeeping owns scan"
10617 );
10618 assert!(crate::worker::active(&home).unwrap().is_empty());
10619 }
10620
10621 #[test]
10622 fn a_stop_is_idempotent_and_once_set_stays_set() {
10623 let stop = Stop::new();
10624 assert!(!stop.stopped());
10625
10626 stop.stop();
10627 assert!(stop.stopped());
10628 stop.stop();
10629 assert!(stop.stopped(), "a second stop is not a toggle");
10630
10631 let shared = stop.clone();
10632 assert!(
10633 shared.stopped(),
10634 "a clone is the same stop; that is how the loop and its caller share one"
10635 );
10636 }
10637
10638 #[tokio::test]
10639 async fn once_drain_rechecks_answers_after_queue_services_release_their_claims() {
10640 for operation in ["conductor", "maintenance", "housekeeping"] {
10641 let dir = tempfile::tempdir().unwrap();
10642 let (mut opts, queue, _, home, worktrees) = idle_loop(dir.path());
10643 opts.once = true;
10644 opts.poll = Duration::from_millis(1);
10645 let mut task = task();
10646 task.runs.push("fixture-run".into());
10647 queue.put(&mut task).unwrap();
10648 let claim = queue.claim(&task.id).unwrap();
10649
10650 let directory = home.join("workers/jobs/abcdef");
10653 std::fs::create_dir_all(&directory).unwrap();
10654 let lease = std::fs::File::create(directory.join("live.lock")).unwrap();
10655 crate::pr_head::try_lock_file(&lease).unwrap();
10656 let mut job = crate::worker::Job {
10657 id: "abcdef".into(),
10658 sequence: 1,
10659 kind: "daemon-service".into(),
10660 key: format!("service:daemon-{operation}"),
10661 payload: serde_json::Value::Null,
10662 generation: "fixture".into(),
10663 version: None,
10664 status: crate::worker::Status::Running,
10665 result: None,
10666 pid: Some(std::process::id()),
10667 created_at: Timestamp::now().as_second() as u64,
10668 finished_at: None,
10669 };
10670 let record = directory.join("job.json");
10671 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10672
10673 let status = Arc::new(Mutex::new(Status::new()));
10674 let stop = Stop::new();
10675 let mut drain = Box::pin(poll(
10676 &opts,
10677 &queue,
10678 &status,
10679 &home,
10680 &worktrees,
10681 &stop,
10682 DispatchLimits {
10683 max_concurrent: 1,
10684 pause_for_interrupts: false,
10685 recovery_cfg: None,
10686 },
10687 ));
10688 std::future::poll_fn(|cx| {
10689 use std::future::Future;
10690 assert!(
10691 drain.as_mut().poll(cx).is_pending(),
10692 "--once must wait for {operation} to release its queue claim"
10693 );
10694 std::task::Poll::Ready(())
10695 })
10696 .await;
10697 assert!(lock(&status).polls > 0, "the dispatch scan was exercised");
10698
10699 let questions = Questions::at(home.join("questions"));
10702 let mut q = action_question("fixture-run", ask::ChoiceAction::Done);
10703 questions.put(&mut q).unwrap();
10704 drop(claim);
10705 let expired =
10709 serde_json::to_vec(&"2000-01-01T00:00:00Z".parse::<Timestamp>().unwrap()).unwrap();
10710 for service in [operation, "housekeeping"] {
10711 std::fs::write(
10712 home.join("service-ticks").join(format!("daemon-{service}")),
10713 &expired,
10714 )
10715 .unwrap();
10716 }
10717 job.status = crate::worker::Status::Finished;
10718 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10719 drop(lease);
10720 tokio::time::timeout(Duration::from_secs(2), drain)
10721 .await
10722 .expect("the settled service must allow the drain to finish")
10723 .unwrap();
10724 let after = queue.get(&task.id).unwrap();
10725 assert_eq!(after.status, TaskStatus::Done, "answer after {operation}");
10726 assert!(after.action_applied(&q.id));
10727 for service in [operation, "housekeeping"] {
10728 assert_eq!(
10729 std::fs::read(home.join("service-ticks").join(format!("daemon-{service}")))
10730 .unwrap(),
10731 expired,
10732 "the final once rescan must not restart {service}"
10733 );
10734 }
10735 }
10736 }
10737
10738 #[test]
10739 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
10740 let stop = Stop::new();
10741 stop.busy.store(1, Ordering::SeqCst);
10742 assert!(
10743 !stop.finishing(),
10744 "a busy loop nobody has asked to stop is just running"
10745 );
10746
10747 stop.stop();
10748 assert!(
10749 stop.finishing(),
10750 "a stop asked for mid-run has not landed until the run is settled"
10751 );
10752
10753 stop.busy.store(0, Ordering::SeqCst);
10754 assert!(
10755 !stop.finishing(),
10756 "once the run is settled the stop has landed and there is nothing to finish"
10757 );
10758 }
10759
10760 #[test]
10761 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
10762 let stop = Stop::new();
10763 stop.busy.store(2, Ordering::SeqCst);
10764 stop.stop();
10765 assert!(stop.finishing(), "two runs still in flight");
10766
10767 stop.busy.store(1, Ordering::SeqCst);
10768 assert!(
10769 stop.finishing(),
10770 "one run finished, but a sibling is still working"
10771 );
10772
10773 stop.busy.store(0, Ordering::SeqCst);
10774 assert!(
10775 !stop.finishing(),
10776 "the last run out is what actually lands the stop"
10777 );
10778 }
10779
10780 #[tokio::test]
10781 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
10782 let dir = tempfile::tempdir().unwrap();
10783 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10784 let stop = Stop::new();
10785 stop.stop();
10786
10787 let began = std::time::Instant::now();
10788 tokio::time::timeout(
10789 Duration::from_secs(2),
10790 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10791 )
10792 .await
10793 .expect("a stopped loop must return, not sit out its poll interval")
10794 .expect("the loop's own setup and teardown must not fail");
10795 assert!(
10796 began.elapsed() < opts.poll,
10797 "returned only after {:?}, which is a poll interval, not a stop",
10798 began.elapsed()
10799 );
10800 }
10801
10802 #[tokio::test]
10803 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
10804 let dir = tempfile::tempdir().unwrap();
10805 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10806 let stop = Stop::new();
10807
10808 let asker = {
10811 let stop = stop.clone();
10812 tokio::spawn(async move {
10813 tokio::time::sleep(Duration::from_millis(20)).await;
10814 stop.stop();
10815 })
10816 };
10817
10818 let began = std::time::Instant::now();
10819 tokio::time::timeout(
10820 Duration::from_secs(2),
10821 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10822 )
10823 .await
10824 .expect("a stop asked for while idle must wake the wait")
10825 .expect("the loop's own setup and teardown must not fail");
10826 asker.await.unwrap();
10827 assert!(
10828 began.elapsed() < opts.poll,
10829 "returned only after {:?}, so the stop waited on the sleep",
10830 began.elapsed()
10831 );
10832 }
10833
10834 #[tokio::test]
10835 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
10836 let dir = tempfile::tempdir().unwrap();
10837 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10838 let stop = Stop::new();
10839 stop.stop();
10840
10841 tokio::time::timeout(
10842 Duration::from_secs(2),
10843 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10844 )
10845 .await
10846 .expect("a stopped loop must return")
10847 .expect("the loop's own setup and teardown must not fail");
10848
10849 assert!(
10850 home.is_dir(),
10851 "the loop did publish a status file, so its removal is the teardown and not an absence"
10852 );
10853 assert!(
10854 !status_file.exists(),
10855 "a stopped loop clears its status file"
10856 );
10857 assert!(
10858 read_status(&home).is_none(),
10859 "a reader must see no daemon at all, not a heartbeat that merely stopped"
10860 );
10861 }
10862
10863 #[tokio::test]
10864 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
10865 let dir = tempfile::tempdir().unwrap();
10866 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10867
10868 let mut settled = RunState::new(
10869 dir.path().join("repo"),
10870 "main".to_owned(),
10871 "abc1234".to_owned(),
10872 "fixture".to_owned(),
10873 Config::default(),
10874 );
10875 settled.status = RunStatus::Ready;
10876 let run_dir = home.join("runs").join(&settled.id);
10877 std::fs::create_dir_all(&run_dir).unwrap();
10878 std::fs::write(
10879 run_dir.join("run.json"),
10880 serde_json::to_string_pretty(&settled).unwrap(),
10881 )
10882 .unwrap();
10883 let questions = Questions::at(home.join("questions"));
10884 let mut question = ask::Question::new(
10885 settled.id.clone(),
10886 "review".to_owned(),
10887 "reviewer-1".to_owned(),
10888 "Continue?".to_owned(),
10889 String::new(),
10890 Vec::new(),
10891 );
10892 questions.put(&mut question).unwrap();
10893
10894 run_service_worker(
10895 &home,
10896 serde_json::to_value(ServiceWorker {
10897 opts,
10898 queue: queue.root().to_path_buf(),
10899 worktrees_root: worktrees,
10900 operation: "housekeeping".to_owned(),
10901 })
10902 .unwrap(),
10903 )
10904 .await
10905 .unwrap();
10906
10907 assert_eq!(
10908 questions.get(&question.id).unwrap().status,
10909 ask::QuestionStatus::Abandoned,
10910 "the independent housekeeping worker settles ended-run questions"
10911 );
10912 }
10913
10914 #[test]
10915 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10916 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10917
10918 assert!(
10919 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10920 "never checked before: due at once"
10921 );
10922
10923 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10924 assert!(
10925 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10926 "well inside the interval: not due yet"
10927 );
10928
10929 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10930 assert!(
10931 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10932 "exactly at the edge: not yet due, same convention as `clean::due`"
10933 );
10934
10935 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10936 assert!(
10937 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10938 "past the interval: due again"
10939 );
10940 }
10941
10942 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10947 let config = dir.join("magi.toml");
10948 std::fs::write(
10954 &config,
10955 format!(
10956 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10957 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10958 cache_dir.display()
10959 ),
10960 )
10961 .unwrap();
10962 Opts {
10963 config: Some(config),
10964 repo: dir.join("repo"),
10965 ..Opts::default()
10966 }
10967 }
10968
10969 #[tokio::test]
10970 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10971 let dir = tempfile::tempdir().unwrap();
10972 let home = dir.path().join("home");
10973 let cache_dir = dir.path().join("cache");
10974 std::fs::create_dir_all(&cache_dir).unwrap();
10975 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10976 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10977
10978 let running = Stop::new();
10981 let mut last_checked = None;
10982 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10983 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10984 .await;
10985 assert_eq!(
10986 crate::disk::dir_size(&cache_dir),
10987 0,
10988 "over the cap on the first check ever: pruned at once, no idle queue required"
10989 );
10990 assert_eq!(last_checked, Some(t0));
10991
10992 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10994 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10995 maybe_prune_cache_between_runs(
10996 &opts.repo,
10997 &opts,
10998 &home,
10999 &running,
11000 &mut last_checked,
11001 too_soon,
11002 )
11003 .await;
11004 assert_eq!(
11005 crate::disk::dir_size(&cache_dir),
11006 10,
11007 "too soon since the last check: left alone rather than rescanned every call"
11008 );
11009 assert_eq!(
11010 last_checked,
11011 Some(t0),
11012 "an idle check does not reset the clock"
11013 );
11014
11015 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
11017 maybe_prune_cache_between_runs(
11018 &opts.repo,
11019 &opts,
11020 &home,
11021 &running,
11022 &mut last_checked,
11023 due_again,
11024 )
11025 .await;
11026 assert_eq!(
11027 crate::disk::dir_size(&cache_dir),
11028 0,
11029 "due again: pruned back under the cap"
11030 );
11031 }
11032
11033 #[tokio::test]
11041 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
11042 let dir = tempfile::tempdir().unwrap();
11043 let home = dir.path().join("home");
11044 let cache_dir = dir.path().join("cache");
11045 std::fs::create_dir_all(&cache_dir).unwrap();
11046 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
11047 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
11048
11049 let stop = Stop::new();
11050 stop.stop();
11051 assert!(
11052 !stop.finishing(),
11053 "no run is in flight at a between-runs boundary, so nothing else \
11054 would tell the operator this stop had not taken effect yet"
11055 );
11056
11057 let mut last_checked = None;
11058 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
11059 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
11060 .await;
11061 assert_eq!(
11062 crate::disk::dir_size(&cache_dir),
11063 10,
11064 "over its cap, and due for the first check ever, but a stop outranks \
11065 it: the cap is a standing policy the next start measures again"
11066 );
11067 assert_eq!(
11068 last_checked, None,
11069 "a check that never happened must not claim the interval"
11070 );
11071 }
11072
11073 #[tokio::test]
11076 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
11077 let dir = tempfile::tempdir().unwrap();
11078 let cache_dir = dir.path().join("cache");
11079 std::fs::create_dir_all(&cache_dir).unwrap();
11080 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
11081
11082 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
11083 opts.poll = Duration::from_millis(20);
11084 opts.max_attempts = 1_000;
11085
11086 let queue = Queue::at(dir.path().join("queue"));
11087 for i in 0..128 {
11088 let mut task = Task::new(
11089 format!("queued work {i}"),
11090 format!("queued work {i}"),
11091 opts.repo.clone(),
11092 Source::Human,
11093 );
11094 queue.put(&mut task).unwrap();
11095 }
11096
11097 let home = dir.path().join("home");
11098 let worktrees = dir.path().join("wt");
11099 run_service_worker(
11100 &home,
11101 serde_json::to_value(ServiceWorker {
11102 opts,
11103 queue: queue.root().to_path_buf(),
11104 worktrees_root: worktrees,
11105 operation: "cache".to_owned(),
11106 })
11107 .unwrap(),
11108 )
11109 .await
11110 .unwrap();
11111 let after = queue.list();
11112 assert_eq!(after.len(), 128);
11113 assert!(
11114 after
11115 .iter()
11116 .all(|task| task.status.runnable() && task.attempts == 0)
11117 );
11118
11119 assert_eq!(
11120 crate::disk::dir_size(&cache_dir),
11121 0,
11122 "an oversized cache must not be left to grow unboundedly just because the \
11123 queue kept the loop busy the whole time"
11124 );
11125 }
11126
11127 #[test]
11128 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
11129 let dir = tempfile::tempdir().unwrap();
11130 let queue = Queue::at(dir.path().join("queue"));
11131 let questions = Questions::at(dir.path().join("questions"));
11132 let mut task = task();
11133 queue.put(&mut task).unwrap();
11134
11135 let mut task_question = ask::Question::new(
11136 task.id.clone(),
11137 crate::conduct::NODE.to_owned(),
11138 "conduct".to_owned(),
11139 "Which backend?".to_owned(),
11140 String::new(),
11141 Vec::new(),
11142 );
11143 questions.put(&mut task_question).unwrap();
11144 task.block(vec![task_question.id.clone()], None);
11145 queue.put(&mut task).unwrap();
11146
11147 let mut run_question = ask::Question::new(
11148 "20260101-000000-run1".to_owned(),
11149 "review".to_owned(),
11150 "reviewer-1".to_owned(),
11151 "Run question".to_owned(),
11152 String::new(),
11153 Vec::new(),
11154 );
11155 questions.put(&mut run_question).unwrap();
11156
11157 let mut coincidental = ask::Question::new(
11162 task.id.clone(),
11163 "review".to_owned(),
11164 "reviewer-1".to_owned(),
11165 "Unrelated review question".to_owned(),
11166 String::new(),
11167 Vec::new(),
11168 );
11169 questions.put(&mut coincidental).unwrap();
11170
11171 reconcile_task_questions(&queue, &questions);
11172 assert!(questions.get(&task_question.id).unwrap().status.open());
11173 assert!(questions.get(&run_question.id).unwrap().status.open());
11174 assert!(questions.get(&coincidental.id).unwrap().status.open());
11175
11176 task.release();
11177 queue.put(&mut task).unwrap();
11178 reconcile_task_questions(&queue, &questions);
11179 assert_eq!(
11180 questions.get(&task_question.id).unwrap().status,
11181 ask::QuestionStatus::Abandoned
11182 );
11183 assert!(
11184 questions.get(&run_question.id).unwrap().status.open(),
11185 "run questions remain the run janitor's responsibility"
11186 );
11187 assert!(
11188 questions.get(&coincidental.id).unwrap().status.open(),
11189 "a non-conductor question must not be abandoned just because its \
11190 run id coincides with a task id"
11191 );
11192 }
11193
11194 #[test]
11195 fn a_freshly_started_running_task_is_never_stalled() {
11196 let dir = tempfile::tempdir().unwrap();
11197 let mut t = task();
11198 t.start("run-1".to_owned());
11199 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
11202 }
11203
11204 #[test]
11205 fn a_long_running_task_with_no_live_daemon_is_stalled() {
11206 let dir = tempfile::tempdir().unwrap();
11207 let mut t = task();
11208 t.start("run-1".to_owned());
11209 t.updated_at = Timestamp::now()
11210 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
11211 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
11212 assert_eq!(
11213 stalled_tasks(
11214 &Queue::at(dir.path().join("q")),
11215 dir.path(),
11216 Timestamp::now()
11217 )
11218 .len(),
11219 0,
11220 "the task was never written to this queue"
11221 );
11222 }
11223
11224 #[test]
11225 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
11226 let dir = tempfile::tempdir().unwrap();
11227 let mut t = task();
11228 t.id = "20260903-080340-0167".to_owned();
11229 t.start("20260903-080619-01c2".to_owned());
11230 t.updated_at = Timestamp::now()
11231 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
11232
11233 let mut status = Status::new();
11234 status.current = vec![Current {
11235 worker: None,
11236 task: t.id.clone(),
11237 run: "20260903-080619-01c2".to_owned(),
11238 }];
11239 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
11240
11241 assert!(
11242 !is_stalled(&t, dir.path(), Timestamp::now()),
11243 "a live daemon's own heartbeat rules out stalled, however long the task has run"
11244 );
11245 }
11246
11247 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
11251 let path = queue.path_of(id);
11252 let body = std::fs::read_to_string(&path).unwrap();
11253 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
11254 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
11255 v["updated_at"] = serde_json::Value::String(old.to_string());
11256 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
11257 }
11258
11259 #[test]
11260 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
11261 let dir = tempfile::tempdir().unwrap();
11274 let queue = Queue::at(dir.path().join("queue"));
11275 let home = dir.path().join("home");
11276
11277 let mut t = task();
11278 t.id = "20260101-000001-lock".to_owned();
11279 t.start("run-1".to_owned());
11280 queue.put(&mut t).unwrap();
11281 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
11282 std::fs::write(
11283 dir.path().join("queue").join(format!("{}.lock", t.id)),
11284 "not a pid",
11285 )
11286 .unwrap();
11287
11288 let now = Timestamp::now();
11289 assert!(
11290 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
11291 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
11292 and reclaim must leave the task alone"
11293 );
11294 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
11295
11296 let stalled = stalled_tasks(&queue, &home, now);
11297 assert_eq!(
11298 stalled.len(),
11299 1,
11300 "reclaim's inability to claim it yet must not hide it from the conductor"
11301 );
11302 assert_eq!(stalled[0].id, t.id);
11303 }
11304
11305 #[test]
11306 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
11307 let dir = tempfile::tempdir().unwrap();
11308 crate::run::set_home(dir.path().join("run-home"));
11309 let queue = Queue::at(dir.path().join("queue"));
11310 let home = dir.path().join("home");
11311 let questions = Questions::at(dir.path().join("questions"));
11312
11313 let mut t = task();
11314 t.id = "20260101-000003-dead".to_owned();
11315 t.start("missing-run".to_owned());
11316 queue.put(&mut t).unwrap();
11317 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
11318
11319 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
11322 assert_eq!(
11323 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
11324 [&t.id]
11325 );
11326 assert_eq!(
11327 reclaim_orphaned_running(&queue, 2, &Default::default()),
11328 [t.id.clone()]
11329 );
11330 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11331
11332 crate::conduct::apply(
11335 &queue,
11336 &questions,
11337 &crate::conduct::Verdict {
11338 decisions: vec![crate::conduct::Decision {
11339 id: t.id.clone(),
11340 recovery: Some(crate::conduct::Recovery::Requeue),
11341 ..crate::conduct::Decision::default()
11342 }],
11343 },
11344 )
11345 .unwrap();
11346 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11347 }
11348
11349 #[test]
11350 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
11351 let dir = tempfile::tempdir().unwrap();
11352 let queue = Queue::at(dir.path().join("queue"));
11353 let home = dir.path().join("home");
11354
11355 let mut fresh = task();
11356 fresh.id = "20260101-000001-aaaa".to_owned();
11357 fresh.start("run-1".to_owned());
11358 queue.put(&mut fresh).unwrap();
11359
11360 let mut old = task();
11361 old.id = "20260101-000002-bbbb".to_owned();
11362 old.start("run-2".to_owned());
11363 queue.put(&mut old).unwrap();
11364 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
11365
11366 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
11367 assert_eq!(stalled.len(), 1);
11368 assert_eq!(stalled[0].id, old.id);
11369 }
11370
11371 #[test]
11372 fn queued_and_finished_task_views_partition_by_status() {
11373 let dir = tempfile::tempdir().unwrap();
11374 let queue = Queue::at(dir.path().join("queue"));
11375
11376 let mut queued = task();
11377 queued.id = "20260101-000001-aaaa".to_owned();
11378 queue.put(&mut queued).unwrap();
11379
11380 let mut failed = task();
11381 failed.id = "20260101-000002-bbbb".to_owned();
11382 failed.start("run-1".to_owned());
11383 failed.fail("gate red", 5);
11384 queue.put(&mut failed).unwrap();
11385
11386 let mut held = task();
11387 held.id = "20260101-000003-cccc".to_owned();
11388 held.hold_machine(None);
11389 queue.put(&mut held).unwrap();
11390
11391 let mut running = task();
11392 running.id = "20260101-000004-dddd".to_owned();
11393 running.start("run-2".to_owned());
11394 queue.put(&mut running).unwrap();
11395
11396 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
11397 assert_eq!(queued_ids, [queued.id.clone()]);
11398
11399 let mut finished_ids: Vec<String> =
11400 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
11401 finished_ids.sort_unstable();
11402 let mut want = vec![failed.id.clone(), held.id.clone()];
11403 want.sort_unstable();
11404 assert_eq!(finished_ids, want);
11405 }
11406
11407 #[test]
11408 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
11409 for short in [false, true] {
11410 let dir = tempfile::tempdir().unwrap();
11411 let queue = Queue::at(dir.path().join("queue"));
11412 let questions = ask::Questions::at(dir.path().join("questions"));
11413 let mut running = task();
11414 running.id = "20261013-192213-feb2".into();
11415 running.start("existing-run".into());
11416 queue.put(&mut running).unwrap();
11417 let mut middle = task();
11418 middle.id = "20261013-193611-ca5b".into();
11419 middle.block(
11420 vec![if short {
11421 "feb2".into()
11422 } else {
11423 running.id.clone()
11424 }],
11425 None,
11426 );
11427 queue.put(&mut middle).unwrap();
11428 let mut waiting = task();
11429 waiting.id = "20261013-202849-ee37".into();
11430 waiting.block(
11431 vec![if short {
11432 "ca5b".into()
11433 } else {
11434 middle.id.clone()
11435 }],
11436 Some("wait for implementation".into()),
11437 );
11438 queue.put(&mut waiting).unwrap();
11439 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
11440 let inv = crate::blockers::Inventory::load(&queue, &questions);
11441 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
11442 assert!(inv.stuck().is_empty());
11443 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11444 assert!(report.quarantined.is_empty());
11445 assert!(report.asked.is_empty());
11446 resolve_blockers(&queue, &questions);
11447 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11448 assert!(questions.list().is_empty());
11449 running.succeed();
11452 queue.put(&mut running).unwrap();
11453 resolve_blockers(&queue, &questions);
11454 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
11455 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11456 middle = queue.get(&middle.id).unwrap();
11457 middle.succeed();
11458 queue.put(&mut middle).unwrap();
11459 resolve_blockers(&queue, &questions);
11460 let after = queue.get(&waiting.id).unwrap();
11461 assert_eq!(after.status, TaskStatus::Queued);
11462 assert!(after.blocked_by.is_empty());
11463 assert_eq!(after.instruction, waiting.instruction);
11464 }
11465 }
11466
11467 #[test]
11468 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
11469 for future_schema in [false, true] {
11470 let name = "20261013-193611-ca5b";
11471 let mut future = task();
11472 future.id = name.into();
11473 future.schema = crate::queue::SCHEMA + 1;
11474 let contents = if future_schema {
11475 serde_json::to_string(&future).unwrap()
11476 } else {
11477 "corrupt".into()
11478 };
11479 let dir = tempfile::tempdir().unwrap();
11480 let queue = Queue::at(dir.path().join("queue"));
11481 let questions = ask::Questions::at(dir.path().join("questions"));
11482 let mut waiting = task();
11483 waiting.id = "20261013-202849-ee37".into();
11484 waiting.block(vec!["ca5b".into()], None);
11485 queue.put(&mut waiting).unwrap();
11486 std::fs::write(queue.path_of(name), contents).unwrap();
11487 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
11488 assert!(
11489 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
11490 );
11491 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11492 assert!(report.asked.is_empty());
11493 resolve_blockers(&queue, &questions);
11494 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11495 assert!(questions.list().is_empty());
11496 }
11497 let dir = tempfile::tempdir().unwrap();
11498 let queue = Queue::at(dir.path().join("queue"));
11499 let questions = ask::Questions::at(dir.path().join("questions"));
11500 let mut waiting = task();
11501 waiting.id = "20261013-202849-ee37".into();
11502 waiting.block(vec!["ca5b".into()], None);
11503 queue.put(&mut waiting).unwrap();
11504 let mut done = task();
11505 done.id = "20261013-193611-ca5b".into();
11506 done.succeed();
11507 queue.put(&mut done).unwrap();
11508 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
11511 assert!(queue.get("ca5b").is_err());
11512 assert_eq!(
11513 crate::queue::dependency(&queue, &questions, "ca5b"),
11514 crate::queue::Dependency::Ambiguous
11515 );
11516 resolve_blockers(&queue, &questions);
11517 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11518 assert!(report.asked.is_empty());
11519 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11520 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
11521 }
11522
11523 #[test]
11524 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
11525 for held in [false, true] {
11526 for question in [false, true] {
11527 for reference in ["ca5b", "20261013-193611-ca5b"] {
11528 let dir = tempfile::tempdir().unwrap();
11529 let queue = Queue::at(dir.path().join("queue"));
11530 let questions = ask::Questions::at(dir.path().join("questions"));
11531 let mut prerequisite = task();
11532 prerequisite.id = "20261013-193611-ca5b".into();
11533 prerequisite.start("existing-run".into());
11534 queue.put(&mut prerequisite).unwrap();
11535 let mut waiting = task();
11536 waiting.id = "20261013-202849-ee37".into();
11537 if held {
11538 waiting.hold_machine(Some("waiting for separate consent".into()));
11539 }
11540 queue.put(&mut waiting).unwrap();
11541 let reason = "Deleting saved data requires separate owner consent";
11542 crate::conduct::apply(
11543 &queue,
11544 &questions,
11545 &crate::conduct::Verdict {
11546 decisions: vec![crate::conduct::Decision {
11547 id: waiting.id.clone(),
11548 blocked_by: vec![reference.into()],
11549 question: question.then(|| "May the saved data be deleted?".into()),
11550 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
11551 reason: Some(reason.into()),
11552 ..crate::conduct::Decision::default()
11553 }],
11554 },
11555 )
11556 .unwrap();
11557 let before = queue.get(&waiting.id).unwrap();
11558 if question {
11559 assert_eq!(before.status, TaskStatus::Blocked);
11560 let q = questions.get(&before.blocked_by[0]).unwrap();
11561 assert_eq!(q.summary, "May the saved data be deleted?");
11562 assert!(q.status.open());
11563 } else {
11564 assert_eq!(before.status, TaskStatus::Held);
11565 let recorded_reason = before.hold_reason.as_deref().unwrap();
11566 assert!(recorded_reason.starts_with(reason));
11567 if held {
11568 assert!(recorded_reason.contains("waiting for separate consent"));
11569 }
11570 }
11571 prerequisite.succeed();
11572 queue.put(&mut prerequisite).unwrap();
11573 resolve_blockers(&queue, &questions);
11574 let after = queue.get(&waiting.id).unwrap();
11575 assert_eq!(after.status, before.status);
11576 assert_eq!(after.blocked_by, before.blocked_by);
11577 assert_eq!(after.hold_reason, before.hold_reason);
11578 assert_eq!(after.instruction, waiting.instruction);
11579 if question {
11580 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
11581 }
11582 assert!(
11583 queue.next_runnable().is_none(),
11584 "dependency completion is not consent"
11585 );
11586 }
11587 }
11588 }
11589 }
11590
11591 #[test]
11592 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
11593 let dir = tempfile::tempdir().unwrap();
11594 let queue = Queue::at(dir.path().join("queue"));
11595 let questions = ask::Questions::at(dir.path().join("questions"));
11596
11597 let mut dep = task();
11598 dep.id = "20260101-000001-dep0".to_owned();
11599 dep.succeed();
11600 queue.put(&mut dep).unwrap();
11601
11602 let mut still_going = task();
11603 still_going.id = "20260101-000002-dep1".to_owned();
11604 queue.put(&mut still_going).unwrap();
11605
11606 let mut blocked = task();
11607 blocked.id = "20260101-000003-main".to_owned();
11608 blocked.block(
11609 vec![dep.id.clone(), still_going.id.clone()],
11610 Some("waits on both".to_owned()),
11611 );
11612 queue.put(&mut blocked).unwrap();
11613
11614 resolve_blockers(&queue, &questions);
11615
11616 let after = queue.get(&blocked.id).unwrap();
11617 assert_eq!(
11618 after.status,
11619 TaskStatus::Blocked,
11620 "one dependency is still outstanding"
11621 );
11622 assert_eq!(after.blocked_by, [still_going.id.clone()]);
11623 }
11624
11625 #[test]
11626 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
11627 let dir = tempfile::tempdir().unwrap();
11628 let queue = Queue::at(dir.path().join("queue"));
11629 let questions = ask::Questions::at(dir.path().join("questions"));
11630
11631 let mut q = crate::ask::Question::new(
11632 "20260101-000001-main".to_owned(),
11633 crate::conduct::NODE.to_owned(),
11634 "conduct".to_owned(),
11635 "Which backend?".to_owned(),
11636 String::new(),
11637 Vec::new(),
11638 );
11639 questions.put(&mut q).unwrap();
11640 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
11641 .unwrap();
11642 questions.put(&mut q).unwrap();
11643
11644 let mut blocked = task();
11645 blocked.id = "20260101-000001-main".to_owned();
11646 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
11647 queue.put(&mut blocked).unwrap();
11648
11649 resolve_blockers(&queue, &questions);
11650
11651 let after = queue.get(&blocked.id).unwrap();
11652 assert_eq!(
11653 after.status,
11654 TaskStatus::Queued,
11655 "the only blocker resolved"
11656 );
11657 assert_eq!(after.answers.len(), 1);
11658 assert_eq!(after.answers[0].question, "Which backend?");
11659 assert_eq!(after.answers[0].answer, "SQLite");
11660
11661 let instruction = instruction_for(&after);
11663 assert!(instruction.contains("Which backend?"));
11664 assert!(instruction.contains("SQLite"));
11665 }
11666
11667 #[test]
11668 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
11669 let dir = tempfile::tempdir().unwrap();
11670 let queue = Queue::at(dir.path().join("queue"));
11671 let questions = ask::Questions::at(dir.path().join("questions"));
11672
11673 let mut q = crate::ask::Question::new(
11674 "20260101-000001-main".to_owned(),
11675 crate::conduct::NODE.to_owned(),
11676 "conduct".to_owned(),
11677 "Is the setup done?".to_owned(),
11678 String::new(),
11679 Vec::new(),
11680 );
11681 q.abandon("no answer within 60s of asking");
11682 questions.put(&mut q).unwrap();
11683
11684 let mut blocked = task();
11685 blocked.id = "20260101-000001-main".to_owned();
11686 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
11687 queue.put(&mut blocked).unwrap();
11688
11689 resolve_blockers(&queue, &questions);
11690
11691 let after = queue.get(&blocked.id).unwrap();
11692 assert_eq!(
11693 after.status,
11694 TaskStatus::Held,
11695 "never left blocked on nothing"
11696 );
11697 assert!(!after.operator_held(), "a machine hold, for triage");
11698 assert!(
11699 after
11700 .hold_reason
11701 .as_deref()
11702 .unwrap_or_default()
11703 .contains("went unanswered")
11704 );
11705 }
11706
11707 #[test]
11708 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
11709 let dir = tempfile::tempdir().unwrap();
11715 let queue = Queue::at(dir.path().join("queue"));
11716 let questions = ask::Questions::at(dir.path().join("questions"));
11717
11718 let mut q = crate::ask::Question::new(
11719 "20260101-000001-main".to_owned(),
11720 crate::conduct::NODE.to_owned(),
11721 "conduct".to_owned(),
11722 "How should this be handled?".to_owned(),
11723 String::new(),
11724 Vec::new(),
11725 );
11726 questions.put(&mut q).unwrap();
11727 q.answer(crate::ask::Answer::Text(
11728 "leave it held, a human will look at it later".to_owned(),
11729 ))
11730 .unwrap();
11731 questions.put(&mut q).unwrap();
11732
11733 let mut held = task();
11734 held.id = "20260101-000001-main".to_owned();
11735 held.hold_machine(Some("out of attempts".to_owned()));
11736 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
11737 queue.put(&mut held).unwrap();
11738
11739 resolve_blockers(&queue, &questions);
11740
11741 let after = queue.get(&held.id).unwrap();
11742 assert_eq!(after.status, TaskStatus::Held);
11743 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
11744 assert_eq!(
11745 after.answers[0].answer,
11746 "leave it held, a human will look at it later"
11747 );
11748 }
11749
11750 #[test]
11751 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
11752 let dir = tempfile::tempdir().unwrap();
11753 let queue = Queue::at(dir.path().join("queue"));
11754 let questions = ask::Questions::at(dir.path().join("questions"));
11755 let mut dep = task();
11756 dep.id = "20260101-000001-gone".to_owned();
11757 queue.put(&mut dep).unwrap();
11758 let mut blocked = task();
11759 blocked.id = "20260101-000003-main".to_owned();
11760 blocked.block(vec![dep.id.clone()], None);
11761 queue.put(&mut blocked).unwrap();
11762
11763 let claim = queue.claim(&blocked.id).unwrap();
11765 queue.remove(&dep.id, false, &questions).unwrap();
11766 drop(claim);
11767 resolve_blockers(&queue, &questions);
11768
11769 let after = queue.get(&blocked.id).unwrap();
11770 assert_eq!(after.status, TaskStatus::Queued);
11771 assert!(after.blocked_by.is_empty());
11772 }
11773
11774 #[test]
11775 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
11776 let dir = tempfile::tempdir().unwrap();
11782 let queue = Queue::at(dir.path().join("queue"));
11783 let questions = ask::Questions::at(dir.path().join("questions"));
11784
11785 let mut still_going = task();
11786 still_going.id = "20260101-000002-dep1".to_owned();
11787 queue.put(&mut still_going).unwrap();
11788
11789 let mut blocked = task();
11790 blocked.id = "20260101-000003-main".to_owned();
11791 blocked.block(
11792 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
11793 Some("waits on both".to_owned()),
11794 );
11795 queue.put(&mut blocked).unwrap();
11796
11797 resolve_blockers(&queue, &questions);
11798
11799 let after = queue.get(&blocked.id).unwrap();
11800 assert_eq!(
11801 after.status,
11802 TaskStatus::Held,
11803 "a missing dependency must not leave the task blocked forever"
11804 );
11805 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
11806 assert!(after.blocked_by.is_empty());
11807 let reason = after.hold_reason.as_deref().unwrap_or_default();
11808 assert!(
11809 reason.contains("20260101-000001-gone"),
11810 "the missing id must be named so an operator can tell what happened: {reason}"
11811 );
11812 assert!(
11813 reason.contains(&still_going.id),
11814 "the still-valid dependency must not silently vanish from the record: {reason}"
11815 );
11816 }
11817
11818 #[test]
11819 fn instruction_for_is_unchanged_without_any_answers() {
11820 let t = task();
11821 assert_eq!(instruction_for(&t), t.instruction);
11822 }
11823
11824 #[test]
11825 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
11826 let dir = tempfile::tempdir().unwrap();
11827 let q = Queue::at(dir.path().join("queue"));
11828 let src = dir.path().join("shot.png");
11829 std::fs::write(&src, "x").unwrap();
11830 let mut t = task();
11831 q.attach(&mut t, &[src]).unwrap();
11832 let paths = task_attachments(&q, &t).unwrap();
11833 assert_eq!(paths.len(), 1);
11834 assert!(paths[0].is_absolute() && paths[0].is_file());
11835 std::fs::remove_file(&paths[0]).unwrap();
11836 let err = task_attachments(&q, &t).unwrap_err().to_string();
11837 assert!(err.contains("shot.png"), "{err}");
11838 }
11839
11840 #[tokio::test]
11841 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
11842 let dir = tempfile::tempdir().unwrap();
11843 let queue_path = dir.path().join("queue");
11844 let queue = Queue::at(queue_path.clone());
11845 let mut t = task();
11846 t.source = Source::Agent {
11847 run: "originating-chat".into(),
11848 node: crate::queue::CHAT_NODE.into(),
11849 };
11850 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
11851 stage: "config-fetch",
11852 target: "origin/main".into(),
11853 });
11854 record_config_failure(&mut t, &fetch);
11855 t.hold_machine(Some(format!("{fetch:#}")));
11856 let fetch_key = t.chat_notice_due().unwrap();
11857 t.chat_notices.push(fetch_key.clone());
11858 queue.put(&mut t).unwrap();
11859
11860 let queue = Queue::at(queue_path);
11861 let mut t = queue.get(&t.id).unwrap();
11862 let a = dir.path().join("a.toml");
11863 let b = dir.path().join("b.toml");
11864 for (path, text, new_cause) in [
11865 (&a, "[graph", true),
11866 (&a, "[verify", false),
11867 (&a, "[graph]\nimplementers = 'invalid-type'", true),
11868 (&b, "[graph]\nimplementers = 'invalid-type'", true),
11869 ] {
11870 t.release_operator("test", None);
11871 std::fs::write(path, text).unwrap();
11872 let error = Config::discover_fetched(dir.path(), Some(path))
11873 .await
11874 .unwrap_err();
11875 record_config_failure(&mut t, &error);
11876 t.hold_machine(Some(format!("{error:#}")));
11877 let due = t.chat_notice_due();
11878 assert_eq!(due.is_some(), new_cause, "{error:#}");
11879 if let Some(key) = due {
11880 assert_ne!(key, fetch_key);
11881 t.chat_notices.push(key);
11882 }
11883 }
11884 t.succeed();
11885 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11886 }
11887
11888 #[test]
11889 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11890 let dir = tempfile::tempdir().unwrap();
11891 let queue_path = dir.path().join("queue");
11892 let queue = Queue::at(queue_path.clone());
11893 let mut t = task();
11894 t.source = crate::queue::Source::Agent {
11895 run: "originating-chat".into(),
11896 node: crate::queue::CHAT_NODE.into(),
11897 };
11898 let a = dir.path().join("a.png");
11899 let b = dir.path().join("b.png");
11900 std::fs::write(&a, "a").unwrap();
11901 std::fs::write(&b, "b").unwrap();
11902 queue.attach(&mut t, &[a, b]).unwrap();
11903 let paths = task_attachments(&queue, &t).unwrap();
11904 for path in &paths {
11905 std::fs::remove_file(path).unwrap();
11906 }
11907 let missing = task_attachments(&queue, &t).unwrap_err();
11908 t.attempts += 1;
11909 let reason = missing.to_string();
11910 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11911 let first = t.chat_notice_due().expect("first attachment notice");
11912 t.chat_notices.push(first.clone());
11913 queue.put(&mut t).unwrap();
11914
11915 let queue = Queue::at(queue_path.clone());
11916 let mut t = queue.get(&t.id).unwrap();
11917 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11918 t.release_operator("test", None);
11919 t.attempts += 1;
11920 let missing = task_attachments(&queue, &t).unwrap_err();
11921 t.fail_startup_target(
11922 "startup-attachments",
11923 missing.name,
11924 "same missing attachment with regenerated wording",
11925 1,
11926 );
11927 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11928
11929 std::fs::write(&paths[0], "restored").unwrap();
11930 t.release_operator("test", None);
11931 t.attempts += 1;
11932 let missing = task_attachments(&queue, &t).unwrap_err();
11933 let reason = missing.to_string();
11934 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11935 let second = t
11936 .chat_notice_due()
11937 .expect("different attachment must notify");
11938 assert_ne!(first, second);
11939 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11940 }
11941
11942 #[test]
11943 fn resumed_instruction_is_unchanged_without_any_answers() {
11944 let t = task();
11945 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11946 }
11947
11948 #[test]
11949 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11950 let mut t = task();
11951 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11952 let old = t.instruction.clone();
11956
11957 let refreshed = resumed_instruction(&old, &t);
11958 assert!(refreshed.starts_with(&old), "the original text is kept");
11959 assert!(refreshed.contains("Which backend?"));
11960 assert!(refreshed.contains("SQLite"));
11961 }
11962
11963 #[test]
11964 fn resumed_instruction_keeps_an_original_answers_heading() {
11965 let mut t = task();
11966 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11967 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11968
11969 let refreshed = resumed_instruction(&t.instruction, &t);
11970
11971 assert!(
11972 refreshed.starts_with(&t.instruction),
11973 "an answers heading in the original instruction is not the appended block"
11974 );
11975 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11976 assert!(refreshed.contains("Which backend?"));
11977 assert!(refreshed.contains("SQLite"));
11978
11979 let repeated = resumed_instruction(&refreshed, &t);
11980 assert_eq!(
11981 repeated, refreshed,
11982 "only the final appended block is refreshed"
11983 );
11984 }
11985
11986 #[test]
11987 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11988 let mut t = task();
11989 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11990
11991 let once = resumed_instruction(&t.instruction, &t);
11995 let twice = resumed_instruction(&once, &t);
11996 assert_eq!(once, twice);
11997 assert_eq!(once.matches("Which backend?").count(), 1);
11998
11999 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
12001 let refreshed = resumed_instruction(&once, &t);
12002 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
12003 assert!(refreshed.contains("Which backend?"));
12004 assert!(refreshed.contains("Which cache?"));
12005 }
12006
12007 #[test]
12008 fn prepare_instruction_covers_all_three_starters() {
12009 let mut t = task();
12010 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
12011
12012 assert_eq!(
12015 prepare_instruction(&Starter::Start, None, &t),
12016 Some(instruction_for(&t))
12017 );
12018
12019 let old = t.instruction.clone();
12022 assert_eq!(
12023 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
12024 Some(resumed_instruction(&old, &t))
12025 );
12026
12027 assert_eq!(
12031 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
12032 None
12033 );
12034 }
12035
12036 #[test]
12037 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
12038 assert_eq!(
12039 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
12040 Starter::Review("magi/eba2/A".to_owned())
12041 );
12042 }
12043
12044 #[test]
12045 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
12046 assert_eq!(
12047 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
12048 Starter::Start,
12049 "a vanished review branch must not fall back to resuming the old run either"
12050 );
12051 }
12052
12053 #[test]
12054 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
12055 let mut t = task();
12056 t.start("old-run".to_owned());
12057 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
12058 t.release();
12059 let branch = t.review_branch.take();
12060 assert_eq!(
12061 choose_starter(branch.as_deref(), true, Some("old-run")),
12062 Starter::Review("magi/eba2/A".to_owned()),
12063 "a review wins over resuming the old run"
12064 );
12065 }
12066
12067 #[test]
12068 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
12069 assert_eq!(
12070 choose_starter(None, false, Some("some-run")),
12071 Starter::Resume("some-run".to_owned())
12072 );
12073 assert_eq!(choose_starter(None, false, None), Starter::Start);
12074 }
12075
12076 #[test]
12077 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
12078 let mut released = task();
12079 released.start("stalled-run".to_owned());
12080 released.requeue();
12081 let unfinished = (!released.fresh_start)
12082 .then(|| Some("stalled-run".to_owned()))
12083 .flatten();
12084 assert_eq!(
12085 choose_starter(None, false, unfinished.as_deref()),
12086 Starter::Start,
12087 "release keeps run history but must not resume it"
12088 );
12089 assert_eq!(released.runs, ["stalled-run"]);
12090 }
12091
12092 #[test]
12093 fn landing_hold_receipt_survives_release_retry_and_checkpoint_removal() {
12094 let dir = tempfile::tempdir().unwrap();
12095 let queue = Queue::at(dir.path().join("queue"));
12096 let mut state = run_state(RunStatus::Blocked);
12097 state.merge_decision = Some(crate::merge_decision::Decision {
12098 question: "hold-inquiry".into(),
12099 target: crate::merge_decision::Target {
12100 pr: "https://example.test/pull/1".into(),
12101 head: "reviewed-head".into(),
12102 language: "en".into(),
12103 },
12104 choice: land::HOLD.into(),
12105 closed: false,
12106 replacement_run: None,
12107 replacement_pr: None,
12108 });
12109 let mut t = task();
12110 t.start(state.id.clone());
12111 settle_and_diagnose(
12112 &mut t,
12113 Verdict {
12114 status: RunStatus::Blocked,
12115 left_pr: true,
12116 quota_hit: false,
12117 parked: false,
12118 no_viable_candidates: false,
12119 },
12120 "held",
12121 2,
12122 &state,
12123 );
12124 queue.put(&mut t).unwrap();
12125 t.release_operator("chat", None);
12126 t.edit("revised".into(), "revised instruction".into())
12127 .unwrap();
12128 queue.put(&mut t).unwrap();
12129 drop(state);
12130 assert_eq!(
12131 queue
12132 .get(&t.id)
12133 .unwrap()
12134 .decision_effects
12135 .get("hold-inquiry"),
12136 Some(&format!("decision: {}", land::HOLD))
12137 );
12138 }
12139
12140 fn stopped_public_run(status: RunStatus) -> RunState {
12141 let mut state = run_state(status);
12142 state.driver_exited = true;
12143 state.config.graph.land = true;
12144 state.config.graph.review_rounds = 1;
12145 state.reviews = vec![review_round(1)];
12146 state.merge = Some(crate::run::MergeOutcome {
12147 mode: MergeMode::Pr,
12148 ok: false,
12149 detail: "landing stopped".into(),
12150 empty: false,
12151 });
12152 state.pr = Some(crate::run::PrRecord {
12153 url: "https://example.test/pull/1".into(),
12154 number: 1,
12155 state: "open".into(),
12156 checks: "red".into(),
12157 round: 1,
12158 rounds: 1,
12159 red_at_merge: Vec::new(),
12160 });
12161 state
12162 }
12163
12164 #[test]
12165 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
12166 for status in [RunStatus::Blocked, RunStatus::Gating] {
12167 let home = tempfile::tempdir().unwrap();
12168 let mut state = stopped_public_run(status);
12169 state.save_under(home.path()).unwrap();
12170 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
12171 assert!(!exhausted_review_budget(&state));
12172 let mut task = held_task_with(&state.id);
12173 task.release();
12174 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
12175 assert_eq!(
12176 choose_starter(None, false, unfinished.as_deref()),
12177 Starter::Resume(state.id.clone())
12178 );
12179 let question = action_question(&state.id, resume_action(&state.id));
12180 assert_eq!(
12181 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
12182 state.clone()
12183 )),
12184 ActionDecision::Resume(state.id.clone())
12185 );
12186 assert_eq!(
12188 crate::resume::validate(home.path(), &state.id).unwrap().id,
12189 state.id
12190 );
12191 let queue = Queue::at(home.path().join("queue"));
12192 let mut controlled = Task::new(
12193 "resume land".into(),
12194 state.instruction.clone(),
12195 state.repo.clone(),
12196 Source::Agent {
12197 run: "talk".into(),
12198 node: crate::queue::CHAT_NODE.into(),
12199 },
12200 );
12201 controlled.runs.push(state.id.clone());
12202 controlled.hold_machine(Some("land stopped".into()));
12203 queue.put(&mut controlled).unwrap();
12204 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
12205 assert_eq!(resumed.status, TaskStatus::Queued);
12206 assert_eq!(
12207 resumed.resume_override.unwrap().pinned_run,
12208 Some(state.id.clone())
12209 );
12210 assert_eq!(resumed.attempts, 0);
12211 }
12212 }
12213
12214 #[test]
12215 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
12216 for review_rounds in [0, 1] {
12217 let home = tempfile::tempdir().unwrap();
12218 let mut state = run_state(RunStatus::Blocked);
12219 state.config.graph.review_rounds = review_rounds;
12220 if review_rounds > 0 {
12221 let mut review = review_round(1);
12222 review.clean = true;
12223 state.reviews = vec![review];
12224 }
12225 state.gate_ran = true;
12226 state.gate = vec![crate::run::CommandOutcome {
12227 command: "verify".into(),
12228 code: Some(1),
12229 output_tail: "gate failed after repair budget exhausted".into(),
12230 duration_ms: 0,
12231 resource_blocked: false,
12232 }];
12233 state.save_under(home.path()).unwrap();
12234 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
12235 assert!(exhausted_review_budget(&state));
12236 let mut task = held_task_with(&state.id);
12237 task.release();
12238 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
12239 let question = action_question(&state.id, resume_action(&state.id));
12240 assert_ne!(
12241 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
12242 state.clone()
12243 )),
12244 ActionDecision::Resume(state.id.clone())
12245 );
12246 let error = crate::resume::validate(home.path(), &state.id)
12247 .unwrap_err()
12248 .to_string();
12249 assert!(!error.contains("review budget exhausted"), "{error}");
12250 assert!(
12251 error.contains(if review_rounds == 0 {
12252 "review is disabled"
12253 } else {
12254 "gate already ran"
12255 }),
12256 "{error}"
12257 );
12258
12259 state.gate_ran = false;
12261 state.gate.clear();
12262 assert!(!exhausted_review_budget(&state));
12263 assert_eq!(
12264 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
12265 Some(state.id)
12266 );
12267 }
12268 }
12269
12270 #[test]
12271 fn resource_blocked_verification_resumes_the_same_run() {
12272 for (kind, stage) in [
12273 ("cache", "clean"),
12274 ("coordination", "clean"),
12275 ("cache", "review-e2e"),
12276 ("coordination", "no-reviews"),
12277 ("coordination", "handoff"),
12278 ] {
12279 let home = tempfile::tempdir().unwrap();
12280 let mut state = run_state(RunStatus::Blocked);
12281 state.config.graph.review_rounds = 1;
12282 let mut review = review_round(1);
12283 review.clean = true;
12284 state.reviews = vec![review];
12285 if stage == "review-e2e" {
12286 let review = &mut state.reviews[0];
12287 review.clean = false;
12288 review.blocking = 1;
12289 review.e2e = vec![crate::run::CommandOutcome {
12290 command: "verify".into(),
12291 code: None,
12292 output_tail: "resource wait expired".into(),
12293 duration_ms: 0,
12294 resource_blocked: true,
12295 }];
12296 } else if stage == "no-reviews" {
12297 state.config.graph.review_rounds = 0;
12298 state.reviews.clear();
12299 } else if stage == "handoff" {
12300 let review = &mut state.reviews[0];
12303 review.clean = false;
12304 review.blocking = 1;
12305 review.e2e = vec![crate::run::CommandOutcome {
12306 command: "verify".into(),
12307 code: Some(0),
12308 output_tail: String::new(),
12309 duration_ms: 0,
12310 resource_blocked: false,
12311 }];
12312 }
12313 state.gate_ran = false;
12314 state.resource_block = Some(crate::run::ResourceBlock {
12315 cause: crate::run::ResourceBlockCause::Other,
12316 node: if stage == "review-e2e" {
12317 "review"
12318 } else {
12319 "gate"
12320 }
12321 .into(),
12322 kind: kind.into(),
12323 detail: "resource wait expired".into(),
12324 at: jiff::Timestamp::now(),
12325 });
12326 state.save_under(home.path()).unwrap();
12327 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
12328 assert!(!exhausted_review_budget(&state));
12329 let mut task = held_task_with(&state.id);
12330 task.release();
12331 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
12332 assert_eq!(
12333 choose_starter(None, false, unfinished.as_deref()),
12334 Starter::Resume(state.id.clone())
12335 );
12336 let question = action_question(&state.id, resume_action(&state.id));
12337 assert_eq!(
12338 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
12339 state.clone()
12340 )),
12341 ActionDecision::Resume(state.id.clone())
12342 );
12343 assert_eq!(
12344 crate::resume::validate(home.path(), &state.id).unwrap().id,
12345 state.id
12346 );
12347 let queue = Queue::at(home.path().join("queue"));
12348 let mut controlled = Task::new(
12349 "resume verification".into(),
12350 state.instruction.clone(),
12351 state.repo.clone(),
12352 Source::Agent {
12353 run: "talk".into(),
12354 node: crate::queue::CHAT_NODE.into(),
12355 },
12356 );
12357 controlled.runs.push(state.id.clone());
12358 controlled.hold_machine(Some("verification resource unavailable".into()));
12359 queue.put(&mut controlled).unwrap();
12360 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
12361 assert_eq!(resumed.status, TaskStatus::Queued);
12362 assert_eq!(
12363 resumed.resume_override.unwrap().pinned_run,
12364 Some(state.id.clone())
12365 );
12366 assert_eq!(resumed.attempts, 0);
12367
12368 state.config.graph.review_rounds = 1;
12370 let mut unresolved = review_round(1);
12371 unresolved.blocking = 1;
12372 state.reviews = vec![unresolved];
12373 state.resource_block = None;
12374 state.save_under(home.path()).unwrap();
12375 assert!(exhausted_review_budget(&state));
12376 assert!(crate::resume::validate(home.path(), &state.id).is_err());
12377 }
12378 }
12379
12380 #[tokio::test]
12381 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
12382 if !crate::standalone_test(
12383 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
12384 ) {
12385 return;
12386 }
12387 let home = tempfile::tempdir().unwrap();
12388 let mut owner = stopped_public_run(RunStatus::Blocked);
12389 owner.save_under(home.path()).unwrap();
12390 let history = vec![owner.id.clone()];
12391 let repo = owner.repo.clone();
12392 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
12393 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
12394 let initial =
12395 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
12396 assert!(
12397 initial.is_pending(),
12398 "a busy record must not hold or resume the task"
12399 );
12400 owner.retired_to = Some("20260101-000001-bbbb".into());
12401 owner.save_under(home.path()).unwrap();
12402 drop(lock);
12403 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
12404 .await
12405 .expect("ownership lock released")
12406 .expect("read settled owner");
12407 assert!(
12408 result.is_none(),
12409 "the retired owner must not be resumed from a stale read"
12410 );
12411 }
12412
12413 #[tokio::test]
12414 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
12415 if !crate::standalone_test(
12416 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
12417 ) {
12418 return;
12419 }
12420 let home = tempfile::tempdir().unwrap();
12421 let mut owner = stopped_public_run(RunStatus::Blocked);
12422 let repo = tempfile::tempdir().unwrap();
12423 crate::git::git(repo.path(), &["init"]).await.unwrap();
12424 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
12425 owner.id = "20260101-000000-aaaa".into();
12426 owner.save_under(home.path()).unwrap();
12427 let mut successor = run_state(RunStatus::Blocked);
12428 successor.id = "20260101-000001-bbbb".into();
12429 successor.save_under(home.path()).unwrap();
12430 let runs = vec![owner.id.clone(), successor.id.clone()];
12431 assert_eq!(
12432 public_resume_owner(home.path(), &runs, &owner.repo)
12433 .await
12434 .unwrap()
12435 .unwrap()
12436 .id,
12437 owner.id
12438 );
12439 let subdir = owner.repo.join("nested");
12440 std::fs::create_dir(&subdir).unwrap();
12441 assert_eq!(
12442 public_resume_owner(home.path(), &runs, &subdir)
12443 .await
12444 .unwrap()
12445 .unwrap()
12446 .id,
12447 owner.id
12448 );
12449 #[cfg(unix)]
12450 {
12451 let alias = home.path().join("repo-alias");
12452 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
12453 assert_eq!(
12454 public_resume_owner(home.path(), &runs, &alias.join("nested"))
12455 .await
12456 .unwrap()
12457 .unwrap()
12458 .id,
12459 owner.id
12460 );
12461 }
12462 let different = tempfile::tempdir().unwrap();
12463 crate::git::git(different.path(), &["init"]).await.unwrap();
12464 assert!(
12465 public_resume_owner(home.path(), &runs, different.path())
12466 .await
12467 .is_err()
12468 );
12469 std::fs::remove_file(
12470 home.path()
12471 .join("runs")
12472 .join(&successor.id)
12473 .join("run.json"),
12474 )
12475 .unwrap();
12476 assert!(
12477 public_resume_owner(home.path(), &runs, &owner.repo)
12478 .await
12479 .is_err()
12480 );
12481 }
12482
12483 #[test]
12484 fn an_ordinary_release_keeps_a_resumable_run_available() {
12485 let mut released = task();
12486 released.start("stalled-run".to_owned());
12487 released.release();
12488 let unfinished = (!released.fresh_start)
12489 .then(|| Some("stalled-run".to_owned()))
12490 .flatten();
12491 assert_eq!(
12492 choose_starter(None, false, unfinished.as_deref()),
12493 Starter::Resume("stalled-run".to_owned()),
12494 "manual release must preserve the normal resume path"
12495 );
12496 }
12497
12498 #[test]
12499 fn resume_refusals_explain_admission_without_changing_task_or_run() {
12500 use crate::handover::{ResumeRefusal, resume_refusal_under};
12501 let home = tempfile::tempdir().unwrap();
12502 let mut exhausted = run_state(RunStatus::Blocked);
12503 exhausted.driver_exited = true;
12504 exhausted.config.graph.review_rounds = 3;
12505 exhausted.reviews = (1..=3).map(review_round).collect();
12506 for refusal in [
12507 ResumeRefusal::ReviewBudget { used: 3, limit: 3 },
12508 ResumeRefusal::Status(RunStatus::Ready),
12509 ResumeRefusal::Released("20261022-000000-aaaa".into()),
12510 ResumeRefusal::Retired("retirement-successor".into()),
12511 ResumeRefusal::OwnershipTransferred("20261022-000000-bbbb".into()),
12512 ] {
12513 let mut state = exhausted.clone();
12514 let expected = match refusal {
12515 ResumeRefusal::Status(_) => {
12516 state.status = RunStatus::Ready;
12517 "status `ready`"
12518 }
12519 ResumeRefusal::Released(ref target) => {
12520 state.released_to = Some(target.clone());
12521 "execution was released"
12522 }
12523 ResumeRefusal::Retired(ref target) => {
12524 state.retired_to = Some(target.clone());
12525 "execution was retired"
12526 }
12527 ResumeRefusal::OwnershipTransferred(ref target) => {
12528 state.pr_taken_over_by = Some(target.clone());
12529 "PR ownership was transferred"
12530 }
12531 _ => "review budget exhausted (3/3 rounds used)",
12532 };
12533 assert_eq!(
12534 resume_refusal_under(&state, Some(home.path())),
12535 Some(refusal.clone())
12536 );
12537 assert_eq!(
12538 refusal.successor(),
12539 state
12540 .released_to
12541 .as_deref()
12542 .or(state.retired_to.as_deref())
12543 .or(state.pr_taken_over_by.as_deref())
12544 );
12545 state.save_under(home.path()).unwrap();
12546 let queue = Queue::at(home.path().join("queue"));
12547 let mut task = Task::new(
12548 "retained implementation".into(),
12549 state.instruction.clone(),
12550 state.repo.clone(),
12551 Source::Agent {
12552 run: "talk".into(),
12553 node: crate::queue::CHAT_NODE.into(),
12554 },
12555 );
12556 task.runs.push(state.id.clone());
12557 task.hold_machine(Some("stopped".into()));
12558 queue.put(&mut task).unwrap();
12559 let before_task = serde_json::to_value(queue.get(&task.id).unwrap()).unwrap();
12560 let before_run = serde_json::to_value(&state).unwrap();
12561 let error = crate::chat_control::resume(&queue, home.path(), &state.id)
12562 .unwrap_err()
12563 .to_string();
12564 assert!(error.contains(expected), "{error}");
12565 assert!(!error.contains("only a stalled or blocked"), "{error}");
12566 if matches!(refusal, ResumeRefusal::Status(_)) {
12567 assert!(
12568 error.contains("is final; there is nothing left to resume"),
12569 "{error}"
12570 );
12571 }
12572 if matches!(refusal, ResumeRefusal::ReviewBudget { .. }) {
12573 assert!(error.contains("magi review <saved-branch> --task <task-id> --merge none"));
12574 assert!(error.contains("review-only successor with a new review budget"));
12575 assert!(error.contains("does not resume this run"));
12576 }
12577 for target in [
12578 &state.released_to,
12579 &state.retired_to,
12580 &state.pr_taken_over_by,
12581 ]
12582 .into_iter()
12583 .flatten()
12584 {
12585 let short = if state.retired_to.as_ref() == Some(target) {
12586 target.as_str()
12587 } else {
12588 crate::run::short_of(target)
12589 };
12590 assert!(
12591 error.contains(&format!("run {short}; inspect `magi show {short}`")),
12592 "{error}"
12593 );
12594 if short != target {
12595 assert!(!error.contains(target), "full successor id leaked: {error}");
12596 }
12597 }
12598 assert_eq!(
12599 serde_json::to_value(queue.get(&task.id).unwrap()).unwrap(),
12600 before_task
12601 );
12602 assert_eq!(
12603 serde_json::to_value(RunState::load_under(&state.id, home.path()).unwrap())
12604 .unwrap(),
12605 before_run
12606 );
12607 let question = action_question(&state.id, resume_action(&state.id));
12608 for phrases in [&PHRASES_EN, &PHRASES_JA] {
12609 let decision = decide_action(home.path(), &task, &question, phrases, |id| {
12610 RunState::load_under(id, home.path())
12611 });
12612 let ActionDecision::Refuse(detail) = decision else {
12613 panic!(
12614 "refusal expected: decision={decision:?}, refusal={refusal:?}, run={}, home={}",
12615 state.id,
12616 home.path().display()
12617 );
12618 };
12619 assert!(detail.contains(expected), "{detail}");
12620 }
12621 }
12622 exhausted.status = RunStatus::Ready;
12624 exhausted.released_to = Some("released".into());
12625 exhausted.retired_to = Some("retired".into());
12626 exhausted.pr_taken_over_by = Some("owner".into());
12627 let reason = resume_refusal_under(&exhausted, Some(home.path())).unwrap();
12628 assert_eq!(reason, ResumeRefusal::Status(RunStatus::Ready));
12629 let detail = reason.diagnostic(&exhausted);
12630 for target in ["released", "retired", "owner"] {
12631 assert!(detail.contains(&format!("magi show {target}")));
12632 }
12633 }
12634
12635 #[test]
12636 fn resume_action_ignores_question_checkpoints_in_another_home() {
12637 crate::run::pin_test_home();
12638 let home = tempfile::tempdir().unwrap();
12639 let mut state = run_state(RunStatus::Blocked);
12640 state.id = format!(
12643 "resume-context-{}",
12644 home.path().file_name().unwrap().to_str().unwrap()
12645 );
12646 state.driver_exited = true;
12647 state.config.graph.review_rounds = 1;
12648 state.reviews = vec![review_round(1)];
12649 state.save_under(home.path()).unwrap();
12650 let task = held_task_with(&state.id);
12651 let answer = action_question(&state.id, resume_action(&state.id));
12652 let mut checkpoint = approval_question(&state.id);
12653 let foreign = Questions::open();
12654 foreign.put(&mut checkpoint).unwrap();
12655 let refusal = crate::handover::resume_refusal_under(&state, Some(home.path()));
12656 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
12657 RunState::load_under(id, home.path())
12658 });
12659 let matching: Vec<_> = foreign
12660 .list()
12661 .into_iter()
12662 .filter(|q| q.run == state.id && q.status.open())
12663 .collect();
12664 std::fs::remove_file(foreign.path_of(&checkpoint.id)).unwrap();
12665 assert!(
12666 matches!(decision, ActionDecision::Refuse(_)),
12667 "decision={decision:?}, refusal={refusal:?}, run={}, home={}, foreign_home={}, matching={matching:?}",
12668 state.id,
12669 home.path().display(),
12670 crate::run::test_home().display()
12671 );
12672 assert_eq!(
12673 refusal,
12674 Some(crate::handover::ResumeRefusal::ReviewBudget { used: 1, limit: 1 })
12675 );
12676 let local = Questions::at(home.path().join("questions"));
12677 for status in [
12678 ask::QuestionStatus::Open,
12679 ask::QuestionStatus::Answered,
12680 ask::QuestionStatus::Abandoned,
12681 ] {
12682 let mut question = checkpoint.clone();
12683 match status {
12684 ask::QuestionStatus::Open => {}
12685 ask::QuestionStatus::Answered => {
12686 question
12687 .answer(ask::Answer::Choice(question.choices[0].clone()))
12688 .unwrap();
12689 }
12690 ask::QuestionStatus::Abandoned => question.abandon("fixture closed"),
12691 }
12692 local.put(&mut question).unwrap();
12693 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
12694 RunState::load_under(id, home.path())
12695 });
12696 if status.open() {
12697 assert_eq!(decision, ActionDecision::Resume(state.id.clone()));
12698 } else {
12699 assert!(
12700 matches!(decision, ActionDecision::Refuse(ref detail) if detail.contains("review budget exhausted")),
12701 "status={status:?}, decision={decision:?}"
12702 );
12703 }
12704 }
12705 }
12706
12707 #[test]
12708 fn resume_action_checks_the_driver_lease_in_the_supplied_home() {
12709 let home = tempfile::tempdir().unwrap();
12710 let mut state = run_state(RunStatus::Stalled);
12711 state.id = format!(
12712 "resume-driver-{}",
12713 home.path().file_name().unwrap().to_str().unwrap()
12714 );
12715 state.driver_claimed = Some(false);
12716 state.save_under(home.path()).unwrap();
12717 let task = held_task_with(&state.id);
12718 let question = action_question(&state.id, resume_action(&state.id));
12719 let decide = || {
12720 decide_action(home.path(), &task, &question, &PHRASES_EN, |id| {
12721 RunState::load_under(id, home.path())
12722 })
12723 };
12724 let lease = crate::driver::maintenance(home.path(), &state.id).unwrap();
12725 assert_eq!(state.liveness_under(home.path(), false), Liveness::Live);
12726 assert_eq!(
12727 decide(),
12728 ActionDecision::Refuse("run already has a live driver".into())
12729 );
12730 drop(lease);
12731 assert_eq!(decide(), ActionDecision::Resume(state.id.clone()));
12732 }
12733
12734 #[test]
12735 fn resume_checkpoint_admission_uses_the_supplied_question_store() {
12736 let home = tempfile::tempdir().unwrap();
12737 let mut state = run_state(RunStatus::Blocked);
12738 state.driver_exited = true;
12739 state.config.graph.review_rounds = 1;
12740 state.reviews = vec![review_round(1)];
12741 assert!(crate::handover::resume_refusal_under(&state, Some(home.path())).is_some());
12742 for checkpoint in ["publication", "approval", "armed", "landing", "question"] {
12743 let mut saved = state.clone();
12744 match checkpoint {
12745 "publication" => saved.pr_publication_pending = true,
12746 "approval" => {
12747 saved.land_approval = Some(crate::run::LandApproval {
12748 question: "approval".into(),
12749 head: "head".into(),
12750 })
12751 }
12752 "armed" => saved.land_armed_head = Some("head".into()),
12753 "landing" => saved.status = RunStatus::Landing,
12754 _ => {
12755 let mut question = ask::Question::new(
12756 saved.id.clone(),
12757 "review".into(),
12758 "review-1".into(),
12759 "continue?".into(),
12760 String::new(),
12761 vec!["continue".into()],
12762 );
12763 ask::Questions::at(home.path().join("questions"))
12764 .put(&mut question)
12765 .unwrap();
12766 }
12767 }
12768 saved.save_under(home.path()).unwrap();
12769 assert!(
12770 crate::handover::resume_refusal_under(&saved, Some(home.path())).is_none(),
12771 "{checkpoint}"
12772 );
12773 assert!(
12774 crate::resume::validate(home.path(), &saved.id).is_ok(),
12775 "{checkpoint}"
12776 );
12777 }
12778 }
12779
12780 #[test]
12781 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
12782 let mut state = run_state(RunStatus::Blocked);
12783 state.config.graph.review_rounds = 3;
12784 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
12785 assert!(exhausted_review_budget(&state));
12786
12787 state.reviews.pop();
12789 assert!(!exhausted_review_budget(&state));
12790
12791 let mut stalled = run_state(RunStatus::Stalled);
12794 stalled.config.graph.review_rounds = 1;
12795 stalled.reviews = vec![review_round(1)];
12796 assert!(!exhausted_review_budget(&stalled));
12797 }
12798
12799 fn review_round(round: usize) -> crate::run::ReviewRound {
12800 crate::run::ReviewRound {
12801 round,
12802 head: "deadbeef".to_owned(),
12803 verified_head: None,
12804 verified_at: None,
12805 reviews: Vec::new(),
12806 e2e: Vec::new(),
12807 verify_retried: false,
12808 e2e_deferred: false,
12809 e2e_defer_reason: None,
12810 fix: None,
12811 blocking: 0,
12812 answered: 1,
12813 expected: 1,
12814 clean: false,
12815 progressed: true,
12816 vote_split: false,
12817 reconsideration: Vec::new(),
12818 verdict: None,
12819 }
12820 }
12821
12822 #[test]
12823 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
12824 let mut run = RunState::new(
12825 PathBuf::from("/repo"),
12826 "main".to_owned(),
12827 "abc1234def".to_owned(),
12828 "add retries".to_owned(),
12829 Config::default(),
12830 );
12831 run.status = RunStatus::Judging;
12832 run.parked = true;
12833 let mut task = Task::new(
12834 "add retries".to_owned(),
12835 "add retries".to_owned(),
12836 PathBuf::from("/repo"),
12837 crate::queue::Source::Human,
12838 );
12839 task.status = TaskStatus::Failed;
12840 task.runs = vec![run.id.clone()];
12841 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
12842 assert!(with(&task, &run), "parked after judging is the case");
12843 let mut parked_task = task.clone();
12844 parked_task.status = TaskStatus::Parked;
12845 assert!(with(&parked_task, &run), "the Parked status is the case");
12846
12847 let mut not_parked = run.clone();
12848 not_parked.parked = false;
12849 not_parked.status = RunStatus::Stalled;
12850 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
12851
12852 let mut fresh = task.clone();
12853 fresh.fresh_start = true;
12854 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
12855
12856 let mut review = task.clone();
12857 review.review_branch = Some("magi/x/A".to_owned());
12858 assert!(!with(&review, &run), "review is ranked before resume");
12859
12860 let mut held = task.clone();
12861 held.status = TaskStatus::Held;
12862 assert!(!with(&held, &run), "a hold stays visible to the conductor");
12863
12864 let mut released = run.clone();
12865 released.released_to = Some("20260901-000000-new1".to_owned());
12866 assert!(!with(&task, &released), "nothing left to resume into");
12867
12868 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
12869 "unreadable"
12870 )));
12871 }
12872
12873 #[test]
12874 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
12875 let mut released = RunState::new(
12876 PathBuf::from("/repo"),
12877 "main".to_owned(),
12878 "abc1234def".to_owned(),
12879 "add retries".to_owned(),
12880 Config::default(),
12881 );
12882 released.status = RunStatus::Blocked;
12883 assert_eq!(
12884 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12885 Some(released.id.clone())
12886 );
12887 released.released_to = Some("20260901-000000-new1".to_owned());
12888 assert_eq!(
12889 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12890 None,
12891 "there is nothing left to resume it into"
12892 );
12893 }
12894
12895 #[test]
12896 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
12897 let mut exhausted = RunState::new(
12906 PathBuf::from("/repo"),
12907 "main".to_owned(),
12908 "abc1234def".to_owned(),
12909 "add retries".to_owned(),
12910 Config::default(),
12911 );
12912 exhausted.status = RunStatus::Blocked;
12913 exhausted.config.graph.review_rounds = 1;
12914 exhausted.reviews = vec![review_round(1)];
12915
12916 assert_eq!(
12917 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
12918 None,
12919 "an exhausted `Blocked` run must not be offered as resumable"
12920 );
12921
12922 let mut has_budget_left = RunState::new(
12925 PathBuf::from("/repo"),
12926 "main".to_owned(),
12927 "abc1234def".to_owned(),
12928 "add retries".to_owned(),
12929 Config::default(),
12930 );
12931 has_budget_left.status = RunStatus::Blocked;
12932 has_budget_left.config.graph.review_rounds = 3;
12933 has_budget_left.reviews = vec![review_round(1)];
12934
12935 assert_eq!(
12936 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
12937 Ok(has_budget_left.clone())
12938 }),
12939 Some(has_budget_left.id.clone())
12940 );
12941 }
12942
12943 #[test]
12944 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
12945 let mut run = RunState::new(
12946 PathBuf::from("/repo"),
12947 "main".to_owned(),
12948 "abc1234def".to_owned(),
12949 "add retries".to_owned(),
12950 Config::default(),
12951 );
12952 run.status = RunStatus::Blocked;
12953 run.config.graph.review_rounds = 1;
12954 run.reviews = vec![review_round(1)];
12955 run.pr_publication_pending = true;
12956 assert_eq!(
12957 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
12958 Some(run.id.clone()),
12959 "a spent review budget must not turn a failed publication into a re-competition"
12960 );
12961 }
12962
12963 #[test]
12964 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
12965 let mut task = Task::new(
12966 "recover work".into(),
12967 "recover work".into(),
12968 PathBuf::from("/repo"),
12969 crate::queue::Source::Human,
12970 );
12971 let mut history = std::collections::BTreeMap::new();
12972 for index in 0..8 {
12973 let mut run = run_state(RunStatus::Blocked);
12974 run.id = format!("20260101-000000-{index:04}");
12975 run.base_commit = format!("upstream-{index}");
12976 if index > 0 {
12978 run.candidates.clear();
12979 }
12980 task.runs.extend([run.id.clone(), run.id.clone()]);
12981 history.insert(run.id.clone(), run);
12982 task.requeue();
12983 }
12984 let check = |task: &Task, starter: &Starter, limit| {
12985 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
12986 };
12987 assert!(
12988 check(&task, &Starter::Start, 8)
12989 .unwrap()
12990 .contains("8 unretired")
12991 );
12992 assert!(check(&task, &Starter::Start, 9).is_none());
12993 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
12994 task.release_operator("owner", None);
12995 assert_eq!(task.attempts, 0);
12996 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
12997 assert!(check(&restored, &Starter::Start, 8).is_some());
12998 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
12999 assert!(check(&restored, &Starter::Start, 0).is_none());
13000 }
13001
13002 #[test]
13003 fn retained_attempt_bound_includes_failed_startups_without_execution_history() {
13004 let tmp = tempfile::tempdir().unwrap();
13005 let mut task = Task::new(
13006 "recover selected PR".into(),
13007 "preserve implementation".into(),
13008 tmp.path().join("repo"),
13009 crate::queue::Source::Human,
13010 );
13011 let mut failed = run_state(RunStatus::Failed);
13012 failed.id = "20260101-000000-fail".into();
13013 failed.candidates.clear();
13014 failed.prep_complete = Some(false);
13015 failed.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13016 failed.startup = Some(crate::run::Startup {
13017 kind: "pr-recovery".into(),
13018 branch: Some("selected-public-head".into()),
13019 stage: "worktree-setup".into(),
13020 complete: false,
13021 failure: Some("setup modified a tracked file and failed".into()),
13022 });
13023 let worktree = failed.worktree_root().join("under-review");
13026 std::fs::create_dir_all(&worktree).unwrap();
13027 let retained_edit = worktree.join("implementation");
13028 std::fs::write(&retained_edit, "retained setup edit").unwrap();
13029 task.record_startup_failure(crate::run::StartupFailure {
13030 run: failed.id.clone(),
13031 stage: "worktree-setup".into(),
13032 });
13033 let task: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
13034 assert!(task.runs.is_empty() && task.passes.is_empty());
13035 assert_eq!(task.attempts, 0);
13036 let starter = Starter::Review("selected-public-head".into());
13037 assert!(
13038 retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone()))
13039 .unwrap()
13040 .contains("1 unretired")
13041 );
13042 assert!(retained_attempt_hold(&task, &starter, 2, |_| Ok(failed.clone())).is_none());
13043 assert!(
13044 retained_attempt_hold(&task, &starter, 2, |_| anyhow::bail!("unreadable"))
13045 .unwrap()
13046 .contains("Cannot verify")
13047 );
13048 let mut mixed = task.clone();
13050 mixed.runs.push(failed.id.clone());
13051 mixed
13052 .startup_failures
13053 .push(task.startup_failures[0].clone());
13054 assert!(retained_attempt_hold(&mixed, &starter, 2, |_| Ok(failed.clone())).is_none());
13055 failed.retired_to = Some("verified-recovery-owner".into());
13056 assert!(retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone())).is_none());
13057 assert!(
13058 retained_attempt_hold(&task, &Starter::Resume(failed.id.clone()), 1, |_| {
13059 panic!("resuming the existing owner must not allocate a new attempt")
13060 })
13061 .is_none()
13062 );
13063 assert_eq!(
13064 std::fs::read_to_string(retained_edit).unwrap(),
13065 "retained setup edit"
13066 );
13067 assert!(task.runs.is_empty() && task.passes.is_empty());
13068 assert_eq!(task.attempts, 0);
13069 }
13070
13071 #[test]
13072 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
13073 let mut task = Task::new(
13074 "recover work".into(),
13075 "recover work".into(),
13076 PathBuf::from("/repo"),
13077 crate::queue::Source::Human,
13078 );
13079 let mut run = run_state(RunStatus::Superseded);
13080 run.candidates.clear();
13081 task.runs.push(run.id.clone());
13082 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
13083 run.retired_to = Some("authority".into());
13084 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
13085 run.retired_to = None;
13086 run.released_to = Some("authority".into());
13087 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
13088 assert!(
13089 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
13090 .unwrap()
13091 .contains("Cannot verify")
13092 );
13093 assert!(
13095 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
13096 panic!("resume must not load historical attempts")
13097 })
13098 .is_none()
13099 );
13100 }
13101
13102 #[test]
13103 fn retired_resume_requires_a_valid_same_repository_authority() {
13104 let mut source = run_state(RunStatus::Blocked);
13105 let mut empty = run_state(RunStatus::Blocked);
13106 empty.retired_to = Some(source.id.clone());
13107 let runs = vec![source.id.clone(), empty.id.clone()];
13108 let resolve = |source: &RunState, empty: &RunState| {
13109 unfinished_run_with(&runs, "fixture", |id| {
13110 if id == source.id {
13111 Ok(source.clone())
13112 } else if id == empty.id {
13113 Ok(empty.clone())
13114 } else {
13115 bail!("unreadable authority")
13116 }
13117 })
13118 };
13119 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
13120 assert_eq!(
13121 choose_starter(None, false, resolve(&source, &empty).as_deref()),
13122 Starter::Resume(source.id.clone())
13123 );
13124 source.retired_to = Some(empty.id.clone());
13125 assert!(
13126 resolve(&source, &empty).is_none(),
13127 "cycles do not revive obsolete runs"
13128 );
13129 source.retired_to = None;
13130 source.repo = PathBuf::from("/another-repository");
13131 assert!(resolve(&source, &empty).is_none());
13132 source.repo = empty.repo.clone();
13133 source.released_to = Some("later".into());
13134 assert!(resolve(&source, &empty).is_none());
13135 empty.retired_to = Some("unlisted-run".into());
13136 assert!(resolve(&source, &empty).is_none());
13137 }
13138
13139 #[test]
13140 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
13141 let mut older_stalled = RunState::new(
13149 PathBuf::from("/repo"),
13150 "main".to_owned(),
13151 "abc1234def".to_owned(),
13152 "add retries".to_owned(),
13153 Config::default(),
13154 );
13155 older_stalled.status = RunStatus::Stalled;
13156
13157 let mut newest_exhausted = RunState::new(
13158 PathBuf::from("/repo"),
13159 "main".to_owned(),
13160 "abc1234def".to_owned(),
13161 "add retries".to_owned(),
13162 Config::default(),
13163 );
13164 newest_exhausted.status = RunStatus::Blocked;
13165 newest_exhausted.config.graph.review_rounds = 1;
13166 newest_exhausted.reviews = vec![review_round(1)];
13167
13168 assert_eq!(
13169 unfinished_run_with(
13170 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
13171 "t",
13172 |_| Ok(newest_exhausted.clone())
13173 ),
13174 None,
13175 "the newest run is exhausted, so nothing here is worth resuming - \
13176 least of all the older, already-superseded run"
13177 );
13178 }
13179
13180 #[test]
13181 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
13182 assert_eq!(
13183 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
13184 Err(anyhow::anyhow!("fixture is absent"))
13185 }),
13186 None
13187 );
13188 }
13189
13190 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
13191 let mut q = ask::Question::new(
13192 run.to_owned(),
13193 "implement".to_owned(),
13194 "impl-A".to_owned(),
13195 "continue?".to_owned(),
13196 String::new(),
13197 vec!["resume で続行する".to_owned(), "other".to_owned()],
13198 );
13199 q.actions.insert("resume で続行する".to_owned(), action);
13200 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
13201 .unwrap();
13202 q
13203 }
13204
13205 fn held_task_with(run: &str) -> Task {
13206 let mut t = task();
13207 t.runs = vec![run.to_owned()];
13208 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
13209 t
13210 }
13211
13212 fn resume_action(run: &str) -> ask::ChoiceAction {
13213 ask::ChoiceAction::Resume { run: run.into() }
13214 }
13215
13216 #[test]
13217 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
13218 let home = crate::run::test_home();
13219 for parked in [false, true] {
13220 let mut t = task();
13221 t.start("A".into());
13222 t.start("B".into());
13223 assert_eq!(t.runs, vec!["A", "B"]);
13224 t.start("A".into());
13225 let mut resumed = run_state(RunStatus::Blocked);
13226 resumed.id = "A".into();
13227 resumed.parked = parked;
13228 resumed.driver_exited = true;
13229 t.hold_machine(Some("resumed run stopped".into()));
13230 let q = action_question("A", resume_action("A"));
13231 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
13232 assert_eq!(
13233 decide_action(&home, &t, &q, &PHRASES_EN, |id| {
13234 assert_eq!(id, "A");
13235 Ok(resumed.clone())
13236 }),
13237 ActionDecision::Resume("A".into())
13238 );
13239 assert_eq!(
13240 action_standing(&t, &action_question("B", resume_action("B"))),
13241 ActionStanding::Stale
13242 );
13243 t.release_operator("cli", None);
13244 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
13245 assert_eq!(id, "A");
13246 Ok(resumed.clone())
13247 });
13248 assert_eq!(
13249 choose_starter(None, false, unfinished.as_deref()),
13250 Starter::Resume("A".into())
13251 );
13252 assert_eq!(t.runs, vec!["B", "A"]);
13253 }
13254 }
13255
13256 #[test]
13257 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
13258 let home = crate::run::test_home();
13259 let mut state = crate::land::tests::repair_state(true, true);
13260 state.status = RunStatus::Landing;
13261 state.parked = true;
13262 state.driver_exited = true;
13263 let mut q = ask::Question::new(
13264 state.id.clone(),
13265 land::APPROVAL_NODE.into(),
13266 "land".into(),
13267 "Merge?".into(),
13268 String::new(),
13269 vec![land::APPROVE.into(), land::HOLD.into()],
13270 );
13271 q.id = state.id.clone();
13272 state.land_approval = Some(crate::run::LandApproval {
13273 question: q.id.clone(),
13274 head: "current-head".into(),
13275 });
13276 assert!(land::upgrade_repair_choice(&mut q, &state));
13277 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
13278 let mut task = held_task_with(&state.id);
13279 task.status = TaskStatus::Parked;
13280 assert_eq!(
13281 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
13282 ActionDecision::Resume(state.id.clone())
13283 );
13284 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
13285 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
13286 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
13287 assert_eq!(
13288 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
13289 ActionDecision::Resume(state.id.clone()),
13290 "a repair answer newer than the manual hold resumes the existing owner"
13291 );
13292 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
13293 for answered_at in [q.answered_at, None] {
13294 let mut older = q.clone();
13295 older.answered_at = answered_at;
13296 let refused = decide_action(&home, &task, &older, &PHRASES_EN, |_| {
13297 panic!("a later manual hold must prevent dispatch")
13298 });
13299 let ActionDecision::Refuse(reason) = refused else {
13300 panic!("expected an actionable refusal, got {refused:?}");
13301 };
13302 assert!(reason.contains("later operator decision"));
13303 assert!(reason.contains(&format!("magi task release {}", task.id)));
13304 }
13305 task.operator_decided_at = None;
13306 let mut superseded = state.clone();
13307 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
13308 assert_eq!(
13309 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(superseded)),
13310 ActionDecision::Stale
13311 );
13312 for case in 1..4 {
13313 let mut changed = state.clone();
13314 match case {
13315 1 => changed.status = RunStatus::Blocked,
13316 2 => changed.released_to = Some("new-owner".into()),
13317 _ => changed.land_approval.as_mut().unwrap().head.clear(),
13318 }
13319 assert!(matches!(
13320 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(changed)),
13321 ActionDecision::Refuse(_)
13322 ));
13323 }
13324 assert!(matches!(
13325 decide_action(&home, &task, &q, &PHRASES_EN, |_| anyhow::bail!(
13326 "missing owner"
13327 )),
13328 ActionDecision::Refuse(_)
13329 ));
13330 task.runs.push("replacement".into());
13331 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
13332 assert_eq!(
13333 decide_action(&home, &task, &q, &PHRASES_EN, |_| panic!(
13334 "old owner must not load"
13335 )),
13336 ActionDecision::Stale
13337 );
13338 }
13339
13340 #[test]
13341 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
13342 let mut state = crate::land::tests::repair_state(true, true);
13343 state.status = RunStatus::Landing;
13344 state.parked = true;
13345 state.driver_exited = true;
13346 let mut q = ask::Question::new(
13347 state.id.clone(),
13348 land::APPROVAL_NODE.into(),
13349 "land".into(),
13350 "Merge?".into(),
13351 String::new(),
13352 vec![land::APPROVE.into(), land::HOLD.into()],
13353 );
13354 q.id = state.id.clone();
13355 state.land_approval = Some(crate::run::LandApproval {
13356 question: q.id.clone(),
13357 head: "current-head".into(),
13358 });
13359 state.save().unwrap();
13360 land::upgrade_repair_choice(&mut q, &state);
13361 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
13362 Questions::open().put(&mut q).unwrap();
13363 let tmp = tempfile::tempdir().unwrap();
13364 let queue = Queue::at(tmp.path().join("queue"));
13365 let mut task = held_task_with(&state.id);
13366 task.status = TaskStatus::Parked;
13367 queue.put(&mut task).unwrap();
13368 let empty = Arc::new(tokio::sync::Semaphore::new(0));
13369 let free = Arc::new(tokio::sync::Semaphore::new(1));
13370 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
13371 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
13372 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
13373 assert!(claim.permit.is_some());
13374 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
13375 drop(claim);
13376 Questions::open()
13377 .update(&q.id, |q| {
13378 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
13379 Ok(())
13380 })
13381 .unwrap();
13382 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
13383 assert!(claim.permit.is_none());
13384 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
13385 }
13386
13387 #[test]
13388 fn decide_action_resumes_only_the_latest_resumable_run() {
13389 let home = crate::run::test_home();
13390 let t = held_task_with("r1");
13391 let q = action_question("r1", resume_action("r1"));
13392 let load = |s: RunState| move |_: &str| Ok(s);
13393 assert_eq!(
13394 decide_action(
13395 &home,
13396 &t,
13397 &q,
13398 &PHRASES_EN,
13399 load(run_state(RunStatus::Blocked))
13400 ),
13401 ActionDecision::Resume("r1".into())
13402 );
13403 let q_other = action_question("r1", resume_action("r0"));
13405 assert!(matches!(
13406 decide_action(
13407 &home,
13408 &t,
13409 &q_other,
13410 &PHRASES_EN,
13411 load(run_state(RunStatus::Blocked))
13412 ),
13413 ActionDecision::Refuse(_)
13414 ));
13415 assert!(matches!(
13417 decide_action(
13418 &home,
13419 &t,
13420 &q,
13421 &PHRASES_EN,
13422 load(run_state(RunStatus::Ready))
13423 ),
13424 ActionDecision::Refuse(_)
13425 ));
13426 let mut released = run_state(RunStatus::Blocked);
13428 released.released_to = Some("elsewhere".into());
13429 assert!(matches!(
13430 decide_action(&home, &t, &q, &PHRASES_EN, load(released)),
13431 ActionDecision::Refuse(_)
13432 ));
13433 assert!(matches!(
13435 decide_action(&home, &t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
13436 ActionDecision::Refuse(_)
13437 ));
13438 }
13439
13440 #[test]
13441 fn decide_action_ignores_a_question_about_an_earlier_run() {
13442 let home = crate::run::test_home();
13443 let mut t = held_task_with("r1");
13444 t.runs.push("r2".to_owned());
13445 let never = |_: &str| -> Result<RunState> { bail!("not read") };
13446 assert_eq!(
13447 decide_action(
13448 &home,
13449 &t,
13450 &action_question("r1", ask::ChoiceAction::Done),
13451 &PHRASES_EN,
13452 never
13453 ),
13454 ActionDecision::Stale
13455 );
13456 }
13457
13458 #[test]
13459 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
13460 let home = crate::run::test_home();
13461 let mut t = held_task_with("r1");
13462 let never = |_: &str| -> Result<RunState> { bail!("not read") };
13463 assert_eq!(
13464 decide_action(
13465 &home,
13466 &t,
13467 &action_question("r1", ask::ChoiceAction::Requeue),
13468 &PHRASES_EN,
13469 never
13470 ),
13471 ActionDecision::Requeue
13472 );
13473 let done_q = action_question("r1", ask::ChoiceAction::Done);
13474 assert_eq!(
13475 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
13476 ActionDecision::Done
13477 );
13478 t.mark_action_applied(&done_q.id);
13479 assert_eq!(
13480 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
13481 ActionDecision::Skip
13482 );
13483
13484 let mut plain = action_question("r1", ask::ChoiceAction::Done);
13486 plain.actions.clear();
13487 assert_eq!(
13488 decide_action(&home, &held_task_with("r1"), &plain, &PHRASES_EN, never),
13489 ActionDecision::Skip
13490 );
13491 let mut running = held_task_with("r1");
13493 running.status = TaskStatus::Running;
13494 assert_eq!(
13495 decide_action(
13496 &home,
13497 &running,
13498 &action_question("r1", ask::ChoiceAction::Done),
13499 &PHRASES_EN,
13500 never
13501 ),
13502 ActionDecision::Skip
13503 );
13504 }
13505
13506 #[test]
13507 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
13508 crate::run::pin_test_home();
13509 for legacy in [false, true] {
13510 let dir = tempfile::tempdir().unwrap();
13511 let queue = Queue::at(dir.path().join("queue"));
13512 let questions = Questions::at(dir.path().join("questions"));
13513 let run = "20260101-000000-hold";
13514 let mut task = held_task_with(run);
13515 task.status = TaskStatus::Parked;
13516 let mut q = ask::Question::new(
13517 run.into(),
13518 land::APPROVAL_NODE.into(),
13519 "land".into(),
13520 "Repair?".into(),
13521 String::new(),
13522 vec![land::REPAIR.into()],
13523 );
13524 q.actions.insert(land::REPAIR.into(), resume_action(run));
13525 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
13526 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
13528 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
13529 questions.put(&mut q).unwrap();
13530 task.hold_manual(Some("Keep this PR paused".into()));
13531 if legacy {
13532 task.hold_source = None;
13533 }
13534 queue.put(&mut task).unwrap();
13535
13536 apply_task_choice_actions(&queue, &questions, dir.path());
13537 let after = queue.get(&task.id).unwrap();
13538 assert!(after.operator_held());
13539 assert_eq!(after.hold_source, task.hold_source);
13540 assert_eq!(after.operator_decided_at, task.operator_decided_at);
13541 assert_eq!(after.runs, task.runs);
13542 assert_eq!(after.instruction, task.instruction);
13543 assert!(after.resume_override.is_none());
13544 assert!(after.action_applied(&q.id));
13545 let reason = after.hold_reason.as_deref().unwrap();
13546 assert!(reason.starts_with("Keep this PR paused"));
13547 assert!(reason.contains("later operator decision"));
13548 assert!(reason.contains(&format!("magi task release {}", task.id)));
13549
13550 for recovery in [
13551 crate::conduct::Recovery::Review,
13552 crate::conduct::Recovery::Requeue,
13553 ] {
13554 crate::conduct::apply(
13555 &queue,
13556 &questions,
13557 &crate::conduct::Verdict {
13558 decisions: vec![crate::conduct::Decision {
13559 id: task.id.clone(),
13560 recovery: Some(recovery),
13561 ..Default::default()
13562 }],
13563 },
13564 )
13565 .unwrap();
13566 }
13567 apply_task_choice_actions(&queue, &questions, dir.path());
13570 let still_held = queue.get(&task.id).unwrap();
13571 assert!(still_held.operator_held());
13572 assert_eq!(still_held.hold_reason, after.hold_reason);
13573 assert_eq!(still_held.runs, task.runs);
13574 assert!(still_held.review_branch.is_none());
13575 assert!(!still_held.fresh_start);
13576 assert!(queue.next_runnable().is_none());
13577 }
13578 }
13579
13580 #[test]
13581 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
13582 for held in [false, true] {
13583 for retry in [false, true] {
13584 let dir = tempfile::tempdir().unwrap();
13585 let queue = Queue::at(dir.path().join("queue"));
13586 let questions = Questions::at(dir.path().join("questions"));
13587 let home = dir.path().join("home");
13588 let mut t = task();
13589 if held {
13590 t.hold_machine(Some("driver state unknown".into()));
13591 }
13592 queue.put(&mut t).unwrap();
13593 let verdict = serde_json::from_value(serde_json::json!({
13594 "decisions": [{
13595 "id": t.id,
13596 "question": "Driver state unknown; retry the checks or hold?",
13597 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
13598 "actions": {
13599 "daemon: recheck and retry": {"do": "requeue"},
13600 "daemon: hold the task": {"do": "hold"}
13601 }
13602 }]
13603 }))
13604 .unwrap();
13605 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
13606 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13607 let mut q = questions.list().pop().unwrap();
13608 let choice = if retry {
13609 "daemon: recheck and retry"
13610 } else {
13611 "daemon: hold the task"
13612 };
13613 assert_eq!(q.actions.len(), 2);
13614 q.answer(ask::Answer::Choice(choice.into())).unwrap();
13615 questions.put(&mut q).unwrap();
13616
13617 questions.beat(&q.id, ask::WaiterKind::Deputy);
13619 for _ in 0..2 {
13620 resolve_blockers(&queue, &questions);
13621 apply_task_choice_actions(&queue, &questions, &home);
13622 let waiting = queue.get(&t.id).unwrap();
13623 assert_eq!(waiting.status, TaskStatus::Blocked);
13624 assert_eq!(waiting.blocked_by, [q.id.clone()]);
13625 assert!(waiting.answers.is_empty());
13626 assert!(!waiting.action_applied(&q.id));
13627 assert!(runnable(&queue).is_empty());
13628 let free = Arc::new(tokio::sync::Semaphore::new(1));
13629 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
13630 }
13631 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13632
13633 resolve_blockers(&queue, &questions);
13635 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13636 apply_task_choice_actions(&queue, &questions, &home);
13637 let after = queue.get(&t.id).unwrap();
13638 assert_eq!(
13639 after.status,
13640 if retry {
13641 TaskStatus::Queued
13642 } else {
13643 TaskStatus::Held
13644 }
13645 );
13646 assert!(after.action_applied(&q.id));
13647 assert_eq!(after.operator_held(), !retry);
13648 assert_eq!(after.answers.len(), 1);
13649
13650 let mut changed = after;
13652 if retry {
13653 changed.hold_manual(Some("later hold".into()));
13654 } else {
13655 changed.release_operator("test", None);
13656 }
13657 let expected = changed.status;
13658 queue.put(&mut changed).unwrap();
13659 apply_task_choice_actions(&queue, &questions, &home);
13660 assert_eq!(queue.get(&t.id).unwrap().status, expected);
13661 }
13662 }
13663 }
13664
13665 #[test]
13666 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
13667 let dir = tempfile::tempdir().unwrap();
13668 let queue = Queue::at(dir.path().join("queue"));
13669 let questions = Questions::at(dir.path().join("questions"));
13670 let home = dir.path().join("home");
13671 let mut state = run_state(RunStatus::Blocked);
13672 state.id = "20260101-000000-act1".to_owned();
13673 state.save_under(&home).unwrap();
13674
13675 let mut t = held_task_with(&state.id);
13676 queue.put(&mut t).unwrap();
13677 let mut q = action_question(&state.id, resume_action(&state.id));
13678 questions.put(&mut q).unwrap();
13679
13680 apply_task_choice_actions(&queue, &questions, &home);
13681 let after = queue.get(&t.id).unwrap();
13682 assert_eq!(after.status, TaskStatus::Queued);
13683 assert!(!after.fresh_start);
13684 assert!(after.action_applied(&q.id));
13685 let pin = after.resume_override.clone().unwrap();
13686 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
13687 assert!(pin.forced);
13688
13689 let mut again = queue.get(&t.id).unwrap();
13691 again.hold_machine(Some("later".into()));
13692 queue.put(&mut again).unwrap();
13693 apply_task_choice_actions(&queue, &questions, &home);
13694 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13695 }
13696
13697 #[test]
13698 fn a_delivered_answer_is_still_acted_on_exactly_once() {
13699 let dir = tempfile::tempdir().unwrap();
13700 let queue = Queue::at(dir.path().join("queue"));
13701 let questions = Questions::at(dir.path().join("questions"));
13702 let home = dir.path().join("home");
13703 let mut state = run_state(RunStatus::Blocked);
13704 state.id = "20260101-000000-act2".to_owned();
13705 state.save_under(&home).unwrap();
13706
13707 let mut t = held_task_with(&state.id);
13708 queue.put(&mut t).unwrap();
13709 let mut q = action_question(&state.id, resume_action(&state.id));
13710 q.answer_delivered = true;
13711 questions.put(&mut q).unwrap();
13712
13713 apply_task_choice_actions(&queue, &questions, &home);
13716 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13717 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
13718
13719 let mut again = queue.get(&t.id).unwrap();
13721 again.hold_machine(Some("later".into()));
13722 queue.put(&mut again).unwrap();
13723 apply_task_choice_actions(&queue, &questions, &home);
13724 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13725 }
13726
13727 #[test]
13728 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
13729 let dir = tempfile::tempdir().unwrap();
13730 let queue = Queue::at(dir.path().join("queue"));
13731 let questions = Questions::at(dir.path().join("questions"));
13732 let home = dir.path().join("home");
13733 let mut state = run_state(RunStatus::Blocked);
13734 state.id = "20260101-000000-act3".to_owned();
13735 state.save_under(&home).unwrap();
13736 let mut t = held_task_with(&state.id);
13737 queue.put(&mut t).unwrap();
13738 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
13739 questions.put(&mut q).unwrap();
13740
13741 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
13742 apply_task_choice_actions(&queue, &questions, &home);
13743 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13744
13745 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13746 apply_task_choice_actions(&queue, &questions, &home);
13747 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13748 }
13749
13750 #[test]
13751 fn action_standing_tells_the_reasons_apart() {
13752 let mut t = held_task_with("r1");
13753 let q = action_question("r1", ask::ChoiceAction::Requeue);
13754 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
13755 let mut plain = q.clone();
13756 plain.actions.clear();
13757 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
13758
13759 t.mark_action_applied(&q.id);
13761 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
13762
13763 let mut t = held_task_with("r1");
13765 t.start("r2".to_owned());
13766 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
13767 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
13769 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13770 t.status = TaskStatus::Blocked;
13771 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13772
13773 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
13775 c.node = crate::conduct::NODE.to_owned();
13776 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
13777 assert!(!ActionStanding::Busy.daemon_owns());
13778 }
13779
13780 #[test]
13781 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
13782 let dir = tempfile::tempdir().unwrap();
13783 let queue = Queue::at(dir.path().join("queue"));
13784 let questions = Questions::at(dir.path().join("questions"));
13785 let home = dir.path().join("home");
13786 let mut state = run_state(RunStatus::Blocked);
13787 state.id = "20260101-000000-act4".to_owned();
13788 state.save_under(&home).unwrap();
13789 let mut t = held_task_with(&state.id);
13790 t.status = TaskStatus::Running;
13791 queue.put(&mut t).unwrap();
13792 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
13793 q.cwd = Some(dir.path().display().to_string());
13794 questions.put(&mut q).unwrap();
13795
13796 let running = queue.get(&t.id).unwrap();
13799 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
13800 assert!(matches!(
13801 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
13802 crate::waiter::Action::Deliver(_)
13803 ));
13804 apply_task_choice_actions(&queue, &questions, &home);
13806 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
13807 let mut back = queue.get(&t.id).unwrap();
13808 back.hold_machine(Some("later".into()));
13809 queue.put(&mut back).unwrap();
13810 apply_task_choice_actions(&queue, &questions, &home);
13811 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
13812 let mut again = queue.get(&t.id).unwrap();
13813 again.hold_machine(Some("again".into()));
13814 queue.put(&mut again).unwrap();
13815 apply_task_choice_actions(&queue, &questions, &home);
13816 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13817 }
13818}