1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{BuildCache, Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub dispatch_wait: Option<DispatchWait>,
139 #[serde(default)]
141 pub question_recovery: Option<crate::question_delivery::Recovery>,
142 #[serde(default)]
144 pub worker_protocol: Option<u32>,
145}
146
147impl Status {
148 #[must_use]
150 pub fn new() -> Self {
151 let now = Timestamp::now();
152 Self {
153 schema: SCHEMA,
154 pid: std::process::id(),
155 started_at: now,
156 updated_at: now,
157 idle: true,
158 current: Vec::new(),
159 completed: 0,
160 polls: 0,
161 dispatch_wait: None,
162 question_recovery: None,
163 worker_protocol: Some(1),
164 }
165 }
166}
167
168impl Default for Status {
169 fn default() -> Self {
170 Self::new()
171 }
172}
173
174#[derive(Debug, Clone, Serialize, Deserialize)]
176#[serde(default)]
177pub struct Opts {
178 pub repo: PathBuf,
180 pub config: Option<PathBuf>,
182 pub poll: Duration,
184 pub max_attempts: usize,
186 pub once: bool,
188 pub merge: Option<String>,
190 pub worktrees_root: Option<PathBuf>,
199}
200
201impl Default for Opts {
202 fn default() -> Self {
203 Self {
204 repo: PathBuf::from("."),
205 config: None,
206 poll: POLL,
207 max_attempts: 2,
208 once: false,
209 merge: None,
210 worktrees_root: None,
211 }
212 }
213}
214
215impl Opts {
216 fn absolute(&self) -> Result<Self> {
217 let mut opts = self.clone();
218 opts.repo = std::path::absolute(&opts.repo)?;
219 opts.config = opts
220 .config
221 .as_deref()
222 .map(std::path::absolute)
223 .transpose()?;
224 opts.worktrees_root = opts
225 .worktrees_root
226 .as_deref()
227 .map(std::path::absolute)
228 .transpose()?;
229 Ok(opts)
230 }
231}
232
233fn max_concurrent(n: usize) -> usize {
238 n.max(1)
239}
240
241#[must_use]
243pub fn status_path() -> PathBuf {
244 crate::run::home().join("daemon.json")
245}
246
247pub fn write_status(status: &Status) -> Result<()> {
249 write_status_to(&status_path(), status)
250}
251
252pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
257 if let Some(parent) = path.parent() {
258 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
259 }
260 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
261 let tmp = path.with_extension("json.tmp");
262 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
263 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
264 Ok(())
265}
266
267pub fn clear_status() {
270 clear_status_at(&status_path());
271}
272
273fn clear_status_at(path: &Path) {
277 let _ = std::fs::remove_file(path);
278}
279
280#[derive(Debug, Clone, Default)]
292pub struct Stop {
293 stopped: Arc<AtomicBool>,
297 busy: Arc<std::sync::atomic::AtomicUsize>,
299 wake: Arc<Notify>,
303 pause: crate::graph::Pause,
306}
307
308impl Stop {
309 #[must_use]
311 pub fn new() -> Self {
312 Self::default()
313 }
314
315 pub fn stop(&self) {
318 self.stopped.store(true, Ordering::SeqCst);
319 self.wake.notify_one();
323 }
324
325 #[must_use]
327 pub fn stopped(&self) -> bool {
328 self.stopped.load(Ordering::SeqCst)
329 }
330
331 #[must_use]
334 pub fn finishing(&self) -> bool {
335 self.stopped() && self.busy_now()
336 }
337
338 pub fn park(&self, cause: crate::run::ParkCause) {
344 self.pause.park_for(cause);
345 self.stop();
346 }
347
348 #[must_use]
350 pub fn parking(&self) -> bool {
351 self.pause.parked()
352 }
353
354 #[must_use]
356 pub fn pause(&self) -> crate::graph::Pause {
357 self.pause.clone()
358 }
359
360 #[must_use]
362 pub fn busy_now(&self) -> bool {
363 self.busy.load(Ordering::SeqCst) > 0
364 }
365
366 async fn idle(&self, poll: Duration) {
368 tokio::select! {
369 () = tokio::time::sleep(poll) => {}
370 () = self.wake.notified() => {}
371 }
372 }
373}
374
375#[derive(Debug, Clone, Default, Deserialize)]
382#[serde(default)]
383pub struct Reading {
384 pub schema: u32,
386 pub pid: Option<u32>,
388 pub started_at: Option<Timestamp>,
390 pub updated_at: Option<Timestamp>,
392 pub idle: bool,
394 #[serde(deserialize_with = "de_current")]
406 pub current: Vec<Current>,
407 pub completed: u64,
409 pub polls: u64,
411 pub dispatch_wait: Option<DispatchWait>,
413 #[serde(default)]
415 pub question_recovery: Option<crate::question_delivery::Recovery>,
416 pub worker_protocol: Option<u32>,
418}
419
420fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
423where
424 D: serde::Deserializer<'de>,
425{
426 #[derive(Deserialize)]
427 #[serde(untagged)]
428 enum Shape {
429 Many(Vec<Current>),
430 One(Current),
431 }
432 Ok(
433 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
434 Shape::Many(v) => v,
435 Shape::One(c) => vec![c],
436 }),
437 )
438}
439
440impl Reading {
441 #[must_use]
444 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
445 self.updated_at
446 .map(|at| (now.as_second() - at.as_second()).max(0))
447 }
448
449 #[must_use]
453 pub fn running(&self, now: Timestamp) -> bool {
454 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
455 }
456}
457
458#[must_use]
465pub fn read_status(home: &Path) -> Option<Reading> {
466 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
467 serde_json::from_str(&body).ok()
468}
469
470#[must_use]
479pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
480 let mut current = read_status(home)
481 .filter(|reading| reading.running(now))
482 .map(|reading| reading.current)
483 .unwrap_or_default();
484 let workers = crate::worker::observe_active(home).and_then(|jobs| {
485 jobs.into_iter()
486 .filter(|job| job.kind == "task")
487 .map(|job| {
488 let request: TaskWorker = serde_json::from_value(job.payload)?;
489 let run = Queue::at(request.queue)
490 .get(&request.task)
491 .ok()
492 .and_then(|task| task.execution_runs().last().cloned())
493 .unwrap_or_default();
494 Ok(Current {
495 worker: Some(job.id),
496 task: request.task,
497 run,
498 })
499 })
500 .collect::<Result<Vec<_>>>()
501 });
502 match workers {
503 Ok(workers) => {
504 for worker in workers {
505 if !current.iter().any(|claim| {
506 claim.task == worker.task
507 && claim.run == worker.run
508 && claim.worker == worker.worker
509 }) {
510 current.push(worker);
511 }
512 }
513 }
514 Err(error) => {
515 tracing::error!("cannot determine active workers: {error:#}");
518 current.extend(
519 Queue::at(home.join("queue"))
520 .list()
521 .into_iter()
522 .map(|task| Current {
523 worker: None,
524 task: task.id.clone(),
525 run: task.execution_runs().last().cloned().unwrap_or_default(),
526 }),
527 );
528 current.extend(
529 crate::run::list_ids_in(&home.join("runs"))
530 .into_iter()
531 .map(|run| Current {
532 worker: None,
533 task: String::new(),
534 run,
535 }),
536 );
537 }
538 }
539 current
540}
541
542#[must_use]
544pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
545 current_work(home, now).iter().any(|c| c.run == run)
546}
547
548pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
553 let same_home = std::env::var_os("MAGI_WORKER_HOME")
556 .is_some_and(|worker_home| Path::new(&worker_home) == home);
557 let caller = if same_home {
558 crate::worker::current_job(home)?
559 } else {
560 None
561 }
562 .filter(|job| {
563 job.kind == "task"
564 && job.status == crate::worker::Status::Running
565 && job.pid == Some(std::process::id())
566 });
567 Ok(other_claims_run(
568 ¤t_work(home, now),
569 run,
570 caller.as_ref().map(|job| job.id.as_str()),
571 ))
572}
573
574fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
575 claims
576 .iter()
577 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
578}
579
580#[must_use]
590pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
591 current_work(home, now)
592 .iter()
593 .any(|c| crate::run::short_of(&c.run) == short)
594}
595
596#[must_use]
598pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
599 current_work(home, now).iter().any(|c| c.task == task)
600}
601
602pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
642 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
643}
644
645fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
649where
650 F: Fn(u32) -> bool,
651{
652 let this_process = std::process::id();
653 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
654 .into_iter()
655 .flatten()
656 .flatten()
657 .map(|e| e.path())
658 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
659 .filter(|p| {
660 match std::fs::read_to_string(p)
661 .ok()
662 .and_then(|body| body.trim().parse::<u32>().ok())
663 {
664 Some(pid) if pid == this_process => false,
668 Some(pid) => !pid_alive(pid),
669 None => p
670 .metadata()
671 .and_then(|m| m.modified())
672 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
673 .is_ok_and(|age| age >= older_than),
674 }
675 })
676 .filter(|p| std::fs::remove_file(p).is_ok())
677 .filter_map(|p| {
678 p.file_stem()
679 .and_then(|s| s.to_str())
680 .map(std::borrow::ToOwned::to_owned)
681 })
682 .collect();
683 swept.sort_unstable();
684 swept
685}
686
687fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
692 task.status == TaskStatus::Running
693 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
694 && !is_working_on_task(home, &task.id, now)
695}
696
697fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
700 queue
701 .list()
702 .into_iter()
703 .filter(|t| is_stalled(t, home, now))
704 .collect()
705}
706
707fn queued_tasks(queue: &Queue) -> Vec<Task> {
713 queue
714 .list()
715 .into_iter()
716 .filter(|t| t.status == TaskStatus::Queued)
717 .collect()
718}
719
720fn finished_tasks(queue: &Queue) -> Vec<Task> {
723 queue
724 .list()
725 .into_iter()
726 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
727 .collect()
728}
729
730fn resolve_blockers(queue: &Queue, questions: &Questions) {
747 for listed in queue.list() {
748 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
749 continue;
750 }
751 let Ok(_claim) = queue.claim(&listed.id) else {
752 continue;
753 };
754 let Ok(mut task) = queue.get(&listed.id) else {
755 continue;
756 };
757 if task.status != TaskStatus::Blocked {
758 continue;
759 }
760 let deleted = queue.apply_deleted_blockers(&mut task, questions);
764 if !deleted.is_empty() {
765 record(queue, &mut task);
766 for id in &deleted {
767 queue.note_dependency_deleted(&task, id);
768 }
769 if task.status != TaskStatus::Blocked {
770 continue;
771 }
772 }
773 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
774 if !missing.is_empty() {
775 let language = language_of(&task, Path::new("."));
776 let mut cause = missing.clone();
777 cause.sort();
778 cause.dedup();
779 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
780 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
781 &task.blocked_by,
782 &missing,
783 &language,
784 )));
785 record(queue, &mut task);
786 continue;
787 }
788 if let Some(q) = task.blocked_by.iter().find_map(|id| {
791 let crate::queue::Dependency::Question(full) =
792 crate::queue::dependency(queue, questions, id)
793 else {
794 return None;
795 };
796 questions.get_for_effects(&full).ok().filter(|q| {
797 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
798 })
799 }) {
800 let language = language_of(&task, Path::new("."));
801 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
802 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
803 record(queue, &mut task);
804 continue;
805 }
806 let mut changed = false;
807 for id in task.blocked_by.clone() {
808 let resolved = crate::queue::dependency(queue, questions, &id);
809 if let crate::queue::Dependency::Task(full) = &resolved {
810 if queue
811 .get(full)
812 .is_ok_and(|dep| dep.status == TaskStatus::Done)
813 {
814 task.unblock(&id);
815 changed = true;
816 }
817 continue;
818 }
819 if let crate::queue::Dependency::Question(full) = resolved
820 && let Ok(q) = questions.get_for_effects(&full)
821 && q.status == ask::QuestionStatus::Answered
822 {
823 if q.node == crate::conduct::NODE
827 && q.chosen_action().is_some()
828 && !task.action_applied(&q.id)
829 {
830 continue;
831 }
832 let answer = match &q.answer {
833 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
834 None => String::new(),
835 };
836 task.record_answer(q.summary.clone(), answer);
837 task.unblock(&id);
838 changed = true;
839 }
840 }
841 if changed {
842 record(queue, &mut task);
843 }
844 }
845}
846
847fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
849 if crate::lang::is_japanese(language) {
850 format!(
851 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
852 q.short(),
853 q.summary
854 )
855 } else {
856 format!(
857 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
858 q.short(),
859 q.summary
860 )
861 }
862}
863
864#[derive(Debug, Clone, Copy, PartialEq, Eq)]
867pub(crate) enum ActionStanding {
868 NoAction,
870 Pending,
872 Applied,
875 Stale,
877 Busy,
879}
880
881impl ActionStanding {
882 pub(crate) fn daemon_owns(self) -> bool {
884 matches!(self, Self::Pending | Self::Applied | Self::Stale)
885 }
886}
887
888pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
892 if task.stale_question(q) {
893 ActionStanding::Stale
894 } else if q.chosen_action().is_none() {
895 ActionStanding::NoAction
896 } else if task.action_applied(&q.id) {
897 ActionStanding::Applied
898 } else if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
899 ActionStanding::Stale
900 } else if matches!(
901 task.status,
902 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
903 ) {
904 ActionStanding::Busy
905 } else {
906 ActionStanding::Pending
907 }
908}
909
910#[derive(Debug, Clone, PartialEq, Eq)]
912enum ActionDecision {
913 Skip,
915 Resume(String),
917 Requeue,
919 Hold,
921 Done,
923 Stale,
926 Refuse(String),
929}
930
931fn decide_action<F>(
940 home: &Path,
941 task: &Task,
942 q: &ask::Question,
943 p: &Phrases,
944 load: F,
945) -> ActionDecision
946where
947 F: FnOnce(&str) -> Result<RunState>,
948{
949 if task.stale_question(q) {
950 return ActionDecision::Stale;
951 }
952 let Some(action) = q.chosen_action() else {
953 return ActionDecision::Skip;
954 };
955 if task.action_applied(&q.id)
956 || matches!(
957 task.status,
958 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
959 )
960 {
961 return ActionDecision::Skip;
962 }
963 if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
966 return ActionDecision::Stale;
967 }
968 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
969 let stopping = q.node == land::APPROVAL_NODE
970 && q.resolution()
971 .as_deref()
972 .is_some_and(crate::merge_decision::stops);
973 let decision_at = q.answered_at.unwrap_or(q.asked_at);
977 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
978 return if repair || stopping {
979 ActionDecision::Refuse(format!(
980 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
981 if repair { "repair" } else { "stop" },
982 q.id,
983 q.run,
984 task.id
985 ))
986 } else {
987 ActionDecision::Stale
988 };
989 }
990 match action {
991 a if a.maintenance() => ActionDecision::Skip,
992 ask::ChoiceAction::RecoverBase { .. }
993 | ask::ChoiceAction::ResumeRun { .. }
994 | ask::ChoiceAction::ReleaseTask { .. }
995 | ask::ChoiceAction::RerunChecks { .. }
996 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
997 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
998 ask::ChoiceAction::Hold => ActionDecision::Hold,
999 ask::ChoiceAction::Done => ActionDecision::Done,
1000 ask::ChoiceAction::Resume { run } => {
1001 if task.execution_runs().last() != Some(run) {
1002 return ActionDecision::Refuse((p.resume_not_latest)(
1003 q.short(),
1004 ask::short_id(run),
1005 ));
1006 }
1007 match load(run) {
1008 Ok(s) if stopping => {
1009 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
1010 || !s
1011 .land_approval
1012 .as_ref()
1013 .is_some_and(|binding| binding.question == q.id)
1014 {
1015 ActionDecision::Stale
1016 } else if s.liveness_under(home, false) == Liveness::Live {
1017 ActionDecision::Skip
1018 } else if s.status == RunStatus::Landing
1019 && !s.released()
1020 && s.retired_to.is_none()
1021 {
1022 ActionDecision::Resume(run.clone())
1023 } else {
1024 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1025 }
1026 }
1027 Ok(s)
1028 if q.node == land::APPROVAL_NODE
1029 && q.resolution().as_deref() == Some(land::REPAIR) =>
1030 {
1031 if s.merge_repairs.iter().any(|r| r.question == q.id)
1032 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1033 {
1034 ActionDecision::Stale
1037 } else if s.liveness_under(home, false) == Liveness::Live {
1038 ActionDecision::Skip
1039 } else if s.status != RunStatus::Landing
1040 || s.released()
1041 || s.reviews.is_empty()
1042 || !implementers_match(task, &s)
1043 || !s
1044 .land_approval
1045 .as_ref()
1046 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1047 {
1048 ActionDecision::Refuse(format!(
1049 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1050 q.id, run, run
1051 ))
1052 } else {
1053 ActionDecision::Resume(run.clone())
1054 }
1055 }
1056 Ok(s) if s.liveness_under(home, false) == Liveness::Live => {
1057 ActionDecision::Refuse("run already has a live driver".into())
1058 }
1059 Ok(s) => match crate::handover::resume_refusal_under(&s, Some(home)) {
1060 None => ActionDecision::Resume(run.clone()),
1061 Some(reason) => ActionDecision::Refuse((p.resume_cannot_progress)(
1062 q.short(),
1063 ask::short_id(run),
1064 &reason.diagnostic(&s),
1065 )),
1066 },
1067 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1068 q.short(),
1069 ask::short_id(run),
1070 &format!("{e:#}"),
1071 )),
1072 }
1073 }
1074 }
1075}
1076
1077struct Phrases {
1091 graph_stopped: fn(&str, &str) -> String,
1093 quorum_lost: &'static str,
1094 quota_took_out: &'static str,
1096 run_ended: &'static str,
1098 waiting_for_answer: &'static str,
1100 recovered_running: &'static str,
1101 no_run_to_recover: &'static str,
1102 could_not_start: &'static str,
1103 resume_not_latest: fn(&str, &str) -> String,
1104 resume_cannot_progress: fn(&str, &str, &str) -> String,
1105 resume_unreadable: fn(&str, &str, &str) -> String,
1106 handover_refused: fn(&crate::handover::Refused) -> String,
1109 handover_hint: &'static str,
1111 reviewers_never_answered: fn(usize, usize) -> String,
1114}
1115
1116const PHRASES_EN: Phrases = Phrases {
1117 graph_stopped: |status, detail| {
1118 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1119 },
1120 quorum_lost: "the judging panel lost its quorum",
1121 quota_took_out: "; quota took out ",
1122 run_ended: "run ended ",
1123 waiting_for_answer: " - waiting for operator answer to question ",
1124 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1125 no_run_to_recover: "task was `running` with no live daemon and no readable \
1126 run to recover; held for a human to check what happened",
1127 could_not_start: "could not start the run: ",
1128 resume_not_latest: |q, run| {
1129 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1130 },
1131 resume_cannot_progress: |q, run, reason| {
1132 format!("question {q} asked to resume run {run}, which cannot make progress: {reason}")
1133 },
1134 resume_unreadable: |q, run, e| {
1135 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1136 },
1137 handover_refused: |r| r.to_string(),
1138 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1139 reviewers_never_answered: |missing, rounds| {
1140 format!(
1141 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1142 refusing to call it clean"
1143 )
1144 },
1145};
1146
1147const PHRASES_JA: Phrases = Phrases {
1148 graph_stopped: |status, detail| {
1149 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1150 },
1151 quorum_lost: "審査パネルが定足数を失いました",
1152 quota_took_out: "。クォータで脱落: ",
1153 run_ended: "run 終了: ",
1154 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1155 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1156 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1157 何が起きたか人が確認するため保留にしました",
1158 could_not_start: "run を開始できませんでした: ",
1159 resume_not_latest: |q, run| {
1160 format!(
1161 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1162 )
1163 },
1164 resume_cannot_progress: |q, run, reason| {
1165 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません: {reason}")
1166 },
1167 resume_unreadable: |q, run, e| {
1168 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1169 },
1170 handover_refused: |r| {
1171 use crate::handover::Refused;
1172 match r {
1173 Refused::Foreign { branch, path, why } => format!(
1174 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1175 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1176 から、やり直してください(詳細: {why})"
1177 ),
1178 Refused::Unsafe { branch, path, why } => format!(
1179 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1180 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1181 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1182 ),
1183 Refused::ReleaseFailed { branch, path, run } => format!(
1184 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1185 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1186 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1187 ),
1188 }
1189 },
1190 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1191 reviewers_never_answered: |missing, rounds| {
1192 format!(
1193 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1194 クリーンとは認めません"
1195 )
1196 },
1197};
1198
1199fn phrases(language: &str) -> &'static Phrases {
1200 if crate::lang::is_japanese(language) {
1201 &PHRASES_JA
1202 } else {
1203 &PHRASES_EN
1204 }
1205}
1206
1207fn language_of(task: &Task, fallback: &Path) -> String {
1211 crate::lang::of_repo(&repo_for(task, fallback))
1212}
1213
1214pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1218 if q.node == crate::conduct::NODE {
1219 return tasks.iter().find(|t| t.id == q.run);
1220 }
1221 tasks.iter().find(|t| t.runs.contains(&q.run))
1222}
1223
1224fn repair_choice_offers(questions: &Questions) {
1227 let ownership = crate::task_chat::InquiryOwnership::default();
1228 for q in questions
1229 .list_for_effects_in(&ownership)
1230 .into_iter()
1231 .filter(|q| {
1232 q.status.open()
1233 && q.node == crate::conduct::NODE
1234 && q.choices
1235 .iter()
1236 .any(|c| crate::choice_effect::describe(q, c).is_none())
1237 })
1238 {
1239 let _ = questions.update_in(&q.id, &ownership, |current| {
1240 if current.status.open() {
1241 crate::choice_effect::executable_conductor(current);
1242 if let Some(deputy) = &mut current.deputy {
1243 deputy.brief = crate::deputy::brief(
1244 ¤t.run,
1245 ¤t.detail,
1246 ¤t.choices,
1247 ¤t.actions,
1248 );
1249 }
1250 }
1251 Ok(())
1252 });
1253 }
1254}
1255
1256pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1259 apply_maintenance_choices(queue, questions, home).await;
1260 apply_task_choice_actions(queue, questions, home);
1261}
1262
1263async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1266 let ownership = crate::task_chat::InquiryOwnership::default();
1267 for q in questions.list_for_effects_in(&ownership) {
1268 let Ok(_execution) = crate::task_inquiry::Outbox::at(home).execution_claim(&q.id) else {
1270 continue;
1271 };
1272 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1273 continue;
1274 };
1275 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1276 continue;
1277 };
1278 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1281 continue;
1282 }
1283 if queue.list().iter().any(|task| {
1284 (task.id == q.run
1285 || task.runs.contains(&q.run)
1286 || q.routed_task.as_deref() == Some(&task.id))
1287 && task.stale_question(&q)
1288 }) {
1289 continue;
1290 }
1291 if let ask::ChoiceAction::RecoverBase { run, stop, episode } = action {
1292 let tasks = queue.list();
1293 let owner = crate::task_chat::task_of(&q, &tasks);
1294 if let Some(owner) = owner
1295 && q.routed_task
1296 .as_deref()
1297 .is_none_or(|routed| routed == owner)
1298 {
1299 let result = crate::base_recovery_control::recover(
1300 queue,
1301 home,
1302 crate::base_recovery_control::Request {
1303 run: run.clone(),
1304 head: stop.head.clone(),
1305 actor: "decisions".into(),
1306 request: q.id.clone(),
1307 decision: Some((owner, q.answered_at.unwrap_or(q.asked_at))),
1308 expected_stop: Some((stop.clone(), *episode)),
1309 },
1310 )
1311 .await;
1312 if let Err(error) = result {
1313 tracing::warn!(question = %q.id, %error, "base recovery decision refused");
1314 notices::raise_in(home, Notice::warn(
1315 &format!("base-recovery:{}", q.id),
1316 format!("Base recovery decision {} was refused: {error:#}. Inspect the existing run and request; no replacement run was started.", q.id),
1317 ).link(Link::Run { id: run.clone() }));
1318 }
1319 }
1320 continue;
1321 }
1322 let dir = home.join("choice-actions");
1323 if std::fs::create_dir_all(&dir).is_err() {
1324 continue;
1325 }
1326 let Ok(lock) = std::fs::OpenOptions::new()
1327 .create(true)
1328 .truncate(false)
1329 .write(true)
1330 .open(dir.join(format!("{}.lock", q.id)))
1331 else {
1332 continue;
1333 };
1334 if crate::pr_head::try_lock_file(&lock).is_err() {
1335 continue;
1336 }
1337 let fresh = !home
1338 .join("choice-actions")
1339 .join(format!("{}.json", q.id))
1340 .exists();
1341 let result = async {
1342 let receipt = dir.join(format!("{}.json", q.id));
1343 if receipt.exists() {
1344 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1345 anyhow::ensure!(
1346 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1347 "previous maintenance attempt did not confirm completion; inspect its receipt"
1348 );
1349 return Ok::<_, anyhow::Error>(());
1350 }
1351 let record = |status: &str| -> anyhow::Result<()> {
1354 let tmp = receipt.with_extension("tmp");
1355 std::fs::write(
1356 &tmp,
1357 serde_json::to_vec(&serde_json::json!({
1358 "action": action, "status": status,
1359 }))?,
1360 )?;
1361 std::fs::rename(tmp, &receipt)?;
1362 Ok(())
1363 };
1364 let applied = async {
1365 match action {
1366 ask::ChoiceAction::ResumeRun { run } => {
1367 crate::resume::validate(home, run)?;
1368 record("started")?;
1369 let decided = q.answered_at.unwrap_or(q.asked_at);
1370 if !crate::resume::pin_queued(queue, run, Some(decided), Some(&q.id))? {
1371 crate::resume::validate_unqueued(home, run)?;
1372 crate::resume::start(home, run.clone()).await?;
1373 }
1374 }
1375 ask::ChoiceAction::ReleaseTask {
1376 task,
1377 held_at,
1378 reason,
1379 } => {
1380 anyhow::ensure!(
1381 queue.resolve_id(task)? == *task,
1382 "task action requires an exact task ID"
1383 );
1384 let _claim = queue.claim(task)?;
1385 let mut task = queue.get(task)?;
1386 if task.action_applied(&q.id) {
1387 return Ok(());
1388 }
1389 if q.node == crate::conduct::NODE
1394 && q.run == task.id
1395 && task.status == TaskStatus::Blocked
1396 && task.blocked_from == Some(TaskStatus::Held)
1397 && task.blocked_by.as_slice() == [q.id.as_str()]
1398 {
1399 task.unblock(&q.id);
1400 }
1401 anyhow::ensure!(
1402 task.status == TaskStatus::Held
1403 && task.held_at == Some(*held_at)
1404 && task.hold_reason.as_deref() == reason.as_deref()
1405 && task
1406 .operator_decided_at
1407 .is_none_or(|at| at <= q.answered_at.unwrap_or(q.asked_at)),
1408 "task no longer has the hold this question names"
1409 );
1410 task.release_operator("choice", Some(&q.id));
1411 task.mark_action_applied(&q.id);
1412 queue.put(&mut task)?;
1413 }
1414 ask::ChoiceAction::AbandonRun { run } => {
1415 record("started")?;
1416 crate::driver::abandon_proven_dead(home, run)?;
1417 }
1418 ask::ChoiceAction::RerunChecks {
1419 repo,
1420 pr,
1421 head,
1422 runs,
1423 } => {
1424 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1425 .await?;
1426 }
1427 _ => unreachable!(),
1428 }
1429 Ok::<_, anyhow::Error>(())
1430 }
1431 .await;
1432 match applied {
1433 Ok(()) => record("applied"),
1434 Err(e) => {
1435 record(&format!("refused or failed: {e:#}"))?;
1436 Err(e)
1437 }
1438 }
1439 }
1440 .await;
1441 if home
1446 .join("choice-actions")
1447 .join(format!("{}.json", q.id))
1448 .exists()
1449 && let Some(listed) = task_of_question(&queue.list(), &q)
1450 && !listed.action_applied(&q.id)
1451 && let Ok(_claim) = queue.claim(&listed.id)
1452 && let Ok(mut origin) = queue.get(&listed.id)
1453 {
1454 if !origin.operator_held()
1455 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1456 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1457 {
1458 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1459 origin.hold_machine(Some(format!(
1460 "Maintenance choice {} settled; inspect its target before releasing this task",
1461 q.id
1462 )));
1463 } else {
1464 origin.unblock(&q.id);
1465 }
1466 }
1467 origin.mark_action_applied(&q.id);
1468 let _ = queue.put(&mut origin);
1469 }
1470 if let Err(e) = result
1471 && fresh
1472 {
1473 notices::raise_in(
1474 home,
1475 Notice::warn(
1476 &format!("action:{}", q.id),
1477 format!("Choice action {} was not completed: {e:#}", q.id),
1478 )
1479 .link(if q.run_names_task() {
1480 Link::Task { id: q.run.clone() }
1481 } else {
1482 Link::Run { id: q.run.clone() }
1483 }),
1484 );
1485 }
1486 }
1487}
1488
1489pub(crate) async fn rerun_checks_once(
1496 home: &Path,
1497 request: &str,
1498 repo: &Path,
1499 pr: &str,
1500 head: &str,
1501 runs: &[String],
1502 started: impl FnOnce() -> anyhow::Result<()>,
1503) -> anyhow::Result<()> {
1504 use crate::release_watch::{GhForge, ReleaseForge};
1505 anyhow::ensure!(
1506 !runs.is_empty()
1507 && runs.len() <= 20
1508 && runs
1509 .iter()
1510 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1511 && !pr.starts_with('-'),
1512 "invalid rerun targets"
1513 );
1514 let snap = GhForge.snapshot(repo, pr).await?;
1515 anyhow::ensure!(
1516 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1517 "pull request head or lifecycle changed"
1518 );
1519 anyhow::ensure!(
1520 runs.iter().all(|r| snap
1521 .checks
1522 .iter()
1523 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1524 "named workflow is no longer failed on this pull request"
1525 );
1526 started()?;
1527 let dir = home.join("choice-actions");
1528 std::fs::create_dir_all(&dir)?;
1529 let rerun_lock = std::fs::OpenOptions::new()
1530 .create(true)
1531 .truncate(false)
1532 .write(true)
1533 .open(dir.join("reruns.lock"))?;
1534 crate::pr_head::try_lock_file(&rerun_lock)?;
1535 let ledger = dir.join("reruns.json");
1536 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1537 serde_json::from_slice(&std::fs::read(&ledger)?)?
1538 } else {
1539 Default::default()
1540 };
1541 for run in runs {
1542 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1543 if !recorded.insert(key) {
1544 continue;
1545 }
1546 let tmp = ledger.with_extension("tmp");
1547 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1548 std::fs::rename(tmp, &ledger)?;
1549 GhForge.rerun(repo, run).await?;
1550 }
1551 Ok(())
1552}
1553
1554pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1563 let tasks = queue.list();
1564 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1565 for q in questions.list_for_effects_in(&ownership) {
1566 let Ok(_execution) = crate::task_inquiry::Outbox::at(home).execution_claim(&q.id) else {
1568 continue;
1569 };
1570 let Ok(q) = questions.get_for_effects_in(&q.id, &ownership) else {
1571 continue;
1572 };
1573 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1575 continue;
1576 }
1577 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1578 continue;
1579 }
1580 let Some(listed) = task_of_question(&tasks, &q) else {
1581 if q.node == land::APPROVAL_NODE
1582 && q.resolution().as_deref().is_some_and(|choice| {
1583 choice == land::REPAIR || crate::merge_decision::stops(choice)
1584 })
1585 {
1586 notices::raise_in(
1587 home,
1588 Notice::warn(
1589 &format!("repair-owner:{}", q.id),
1590 format!(
1591 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1592 q.run, q.run
1593 ),
1594 ).link(Link::Run { id: q.run.clone() }),
1595 );
1596 }
1597 continue;
1598 };
1599 if listed.action_applied(&q.id) {
1600 continue;
1601 }
1602 let Ok(_claim) = queue.claim(&listed.id) else {
1603 continue;
1604 };
1605 let Ok(mut task) = queue.get(&listed.id) else {
1606 continue;
1607 };
1608 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1611 let blockers: Vec<_> = task
1612 .blocked_by
1613 .iter()
1614 .filter(|id| {
1615 crate::queue::dependency(queue, questions, id)
1616 == crate::queue::Dependency::Question(q.id.clone())
1617 })
1618 .cloned()
1619 .collect();
1620 if !blockers.is_empty() {
1621 for id in blockers {
1622 task.unblock(&id);
1623 }
1624 let answer = match &q.answer {
1625 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1626 None => String::new(),
1627 };
1628 task.record_answer(q.summary.clone(), answer);
1629 }
1630 }
1631 let language = language_of(&task, Path::new("."));
1632 let decision = decide_action(home, &task, &q, phrases(&language), |id| {
1633 RunState::load_under(id, home)
1634 });
1635 let ran = matches!(
1636 decision,
1637 ActionDecision::Resume(_)
1638 | ActionDecision::Requeue
1639 | ActionDecision::Hold
1640 | ActionDecision::Done
1641 );
1642 if ran && asker_may_still_read(questions, &q, home) {
1643 continue;
1652 }
1653 let done = matches!(decision, ActionDecision::Done);
1654 let completed = matches!(
1657 decision,
1658 ActionDecision::Requeue | ActionDecision::Hold | ActionDecision::Done
1659 );
1660 match decision {
1661 ActionDecision::Skip => continue,
1662 ActionDecision::Resume(run) => {
1663 task.release_operator("choice", Some(&q.id));
1664 task.resume_override = Some(crate::queue::OperatorResume {
1665 question_id: q.id.clone(),
1666 at: Timestamp::now(),
1667 conductor_rehold: None,
1668 forced: true,
1669 pinned_run: Some(run),
1670 });
1671 }
1672 ActionDecision::Stale => {}
1673 ActionDecision::Requeue => {
1674 task.release_operator("choice-requeue", Some(&q.id));
1675 task.requeue();
1676 }
1677 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1678 ActionDecision::Done => task.succeed(),
1679 ActionDecision::Refuse(why) => {
1680 let _ =
1681 crate::task_inquiry::Outbox::at(home).update_decision(&q.id, |d| d.fail(&why));
1682 if task.operator_held() {
1683 task.hold_reason = Some(match task.hold_reason.take() {
1686 Some(reason) => format!("{reason}\n\n{why}"),
1687 None => why,
1688 });
1689 } else {
1690 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1691 task.hold_machine(Some(why));
1692 }
1693 notices::raise(
1694 Notice::warn(
1695 &format!("action:{}", q.id),
1696 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1697 )
1698 .link(Link::Task {
1699 id: task.id.clone(),
1700 }),
1701 );
1702 }
1703 }
1704 task.mark_action_applied(&q.id);
1705 if completed {
1706 task.decision_effects.insert(
1707 q.id.clone(),
1708 format!("decision: {}", q.resolution().expect("answered action")),
1709 );
1710 }
1711 record(queue, &mut task);
1714 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1715 supersede_prior_runs(&task, home);
1716 }
1717 let _ = questions.update_in(&q.id, &ownership, |r| {
1720 r.answer_delivered = true;
1721 Ok(())
1722 });
1723 }
1724}
1725
1726fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1730 let now = Timestamp::now();
1731 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1732 return true;
1733 }
1734 q.node != crate::conduct::NODE
1735 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1736 s.seats_active()
1737 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1738 })
1739}
1740
1741fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1752 let tasks = queue.list();
1753 let ownership = crate::task_chat::InquiryOwnership::from_tasks(&tasks);
1754 let by_id: std::collections::BTreeMap<&str, &Task> =
1755 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1756 let referenced: Vec<&str> = tasks
1757 .iter()
1758 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1759 .collect();
1760
1761 for mut question in questions.list_for_effects_in(&ownership) {
1762 if !question.status.open()
1763 || question.node != crate::conduct::NODE
1764 || questions.held_reply_in(&question.id, &ownership)
1765 {
1766 continue;
1767 }
1768 if referenced
1773 .iter()
1774 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1775 {
1776 continue;
1777 }
1778 let Some(task) = by_id.get(question.run.as_str()) else {
1779 continue;
1780 };
1781 question.abandon(format!(
1782 "task {} no longer waits for this answer",
1783 task.short()
1784 ));
1785 if let Err(e) = questions.put_in(&mut question, &ownership) {
1786 tracing::warn!(
1787 "could not retire question {} for task {}: {e:#}",
1788 question.short(),
1789 task.short()
1790 );
1791 }
1792 }
1793}
1794
1795#[derive(Debug, Clone, Copy)]
1802pub struct Verdict {
1803 pub status: RunStatus,
1805 pub left_pr: bool,
1807 pub quota_hit: bool,
1809 pub parked: bool,
1811 pub no_viable_candidates: bool,
1818}
1819
1820pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1879 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1880}
1881
1882fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1884 if verdict.parked {
1890 task.park(detail);
1891 return;
1892 }
1893 match verdict.status {
1894 RunStatus::Merged | RunStatus::Ready
1895 if task.pr_recovery.as_ref().is_some_and(|s| {
1896 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1897 }) =>
1898 {
1899 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1900 }
1901 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1902 RunStatus::AlreadyInBase => task.already_landed(detail),
1903 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1904 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1905 task.stall(detail)
1906 }
1907 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1908 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1909 RunStatus::Blocked => task.fail(detail, max_attempts),
1910 RunStatus::VerifiedNoop => task.handed_off(detail),
1911 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1912 }
1913}
1914
1915pub fn supersede_prior_runs(task: &Task, home: &Path) {
1964 let now = Timestamp::now();
1965 let last_run_succeeded = task
1972 .runs
1973 .last()
1974 .and_then(|id| RunState::load_under(id, home).ok())
1975 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1976 for id in task.superseded_attempts(last_run_succeeded) {
1977 let mut state = match RunState::load_under(id, home) {
1978 Ok(s) => s,
1979 Err(e) => {
1980 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1981 continue;
1982 }
1983 };
1984 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1985 continue;
1986 }
1987 let daemon_claims = is_working_on(home, id, now);
1988 if state.liveness(daemon_claims) == Liveness::Live {
1989 continue;
1990 }
1991 state.status = RunStatus::Superseded;
1992 if let Err(e) = state.save_under(home) {
1993 tracing::warn!("could not mark run {id} superseded: {e:#}");
1994 }
1995 }
1996}
1997
1998fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
2020 for task in queue.list() {
2021 if task.status != TaskStatus::Done || task.runs.len() < 2 {
2022 continue;
2023 }
2024 supersede_prior_runs(&task, home);
2025 }
2026}
2027
2028fn settle_and_diagnose(
2035 task: &mut Task,
2036 verdict: Verdict,
2037 detail: &str,
2038 max_attempts: usize,
2039 state: &RunState,
2040) {
2041 let p = phrases(&state.config.graph.language);
2042 if let Some(d) = &state.merge_decision {
2045 use crate::merge_decision as md;
2046 if task
2047 .execution_runs()
2048 .last()
2049 .is_some_and(|id| id != &state.id)
2050 {
2051 return;
2052 }
2053 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
2054 task.succeed();
2055 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
2056 if state.liveness(false) != Liveness::Dead {
2057 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
2058 return;
2059 }
2060 let replacement = md::Replacement {
2061 question: d.question.clone(),
2062 run: state.id.clone(),
2063 pr: d.target.pr.clone(),
2064 agent: agent.to_owned(),
2065 context: format!(
2066 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
2067 d.target.pr,
2068 state.id,
2069 serde_json::to_string(&state.reviews).unwrap_or_default()
2070 ),
2071 };
2072 task.requeue();
2073 task.pr_recovery = None;
2074 task.review_of = None;
2075 task.replacement = Some(replacement);
2076 } else {
2077 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
2078 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
2079 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
2080 }
2081 task.mark_action_applied(&d.question);
2082 if d.choice.eq_ignore_ascii_case(land::HOLD) {
2083 task.decision_effects
2084 .insert(d.question.clone(), format!("decision: {}", d.choice));
2085 }
2086 if let Some(pass) = state.passes.last() {
2087 task.record_pass_at(
2088 &state.id,
2089 pass.index,
2090 Some(pass.started_at),
2091 task.attempts,
2092 None,
2093 );
2094 }
2095 return;
2096 }
2097 if let Some(replacement) = &task.replacement
2098 && !state.parked
2099 && state.resource_block.is_none()
2100 && (state.viable().is_empty()
2101 || (state.status == RunStatus::Blocked && state.tally.is_none()))
2102 {
2103 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
2104 return;
2105 }
2106 if !verdict.parked && verdict.quota_hit && state.status == RunStatus::Stalled {
2107 let mut agents: Vec<_> = state
2108 .quota
2109 .iter()
2110 .filter(|q| {
2111 q.cause == crate::agent::QuotaCause::Authentication
2112 && state
2113 .passes
2114 .last()
2115 .is_none_or(|pass| q.at >= pass.started_at)
2116 })
2117 .map(|q| q.agent.as_str())
2118 .collect();
2119 agents.sort_unstable();
2120 agents.dedup();
2121 if !agents.is_empty() {
2122 task.stall(detail);
2123 task.notice_cause("agent-authentication", None, None);
2124 task.hold_machine(Some(format!("[authentication] Agents {} cannot authenticate; re-authenticate, then resume the held task. No attempt was spent.", agents.join(", "))));
2125 return;
2126 }
2127 }
2128 let attempts_before = task.attempts;
2129 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2136 let disk_checked_at = task.disk_checked_at;
2137 task.notice_cause(
2138 &format!("resource:{}:{}", block.kind, block.node),
2139 None,
2140 None,
2141 );
2142 task.last_error = Some(detail.to_owned());
2143 task.diagnostic = None;
2144 task.attempts = task.attempts.saturating_sub(1);
2145 if block.kind == "agent authentication" {
2146 task.hold_machine(Some(format!(
2147 "[authentication] {}; no attempt was spent",
2148 block.detail
2149 )));
2150 } else if block.kind == "agent usage limit" {
2151 if let Some(agent) = &state.config.graph.selected_implementer {
2153 task.hold_machine(Some(format!("[agent usage limit] Selected implementer `{agent}` is unavailable. No other implementer was dispatched; resume after usage is available. No attempt was spent.")));
2154 } else {
2155 task.status = TaskStatus::Failed;
2156 task.last_error = Some(block.detail.clone());
2157 }
2158 } else if block.kind == "disk pressure" {
2159 task.hold_machine(Some(format!(
2160 "{}{}",
2161 crate::disk::RUNNING_PRESSURE,
2162 block.detail
2163 )));
2164 } else {
2165 task.hold_machine(Some(format!(
2166 "{}{}; no attempt was spent; resume to retry the affected operation",
2167 crate::cache::RESOURCE_HOLD,
2168 block.description()
2169 )));
2170 }
2171 if matches!(block.kind.as_str(), "disk pressure" | "disk admission")
2172 && matches!(block.cause, crate::run::ResourceBlockCause::Other)
2173 {
2174 task.mark_disk_capacity();
2175 task.disk_checked_at = disk_checked_at;
2176 }
2177 if let Some(pass) = state.passes.last() {
2178 task.record_pass_at(
2179 &state.id,
2180 pass.index,
2181 Some(pass.started_at),
2182 attempts_before,
2183 None,
2184 );
2185 }
2186 return;
2187 }
2188 if let Some(hold) = state.land_unrelated.as_ref().filter(|hold| {
2189 state.parked
2190 && state.status == RunStatus::Landing
2191 && state
2192 .pr_pushed_head
2193 .as_ref()
2194 .is_none_or(|h| h == &hold.head)
2195 && task.execution_runs().last() == Some(&state.id)
2196 }) {
2197 let pass = state.passes.last().map_or(0, |p| p.index);
2200 if !task
2201 .passes
2202 .iter()
2203 .any(|p| p.run == state.id && p.pass_index == pass)
2204 {
2205 task.attempts = task.attempts.saturating_sub(1);
2206 }
2207 task.hold_machine(Some(format!(
2208 "[ci-unrelated] CI failure reproduced on base; owner question {}",
2209 hold.question
2210 )));
2211 task.last_error = Some(detail.to_owned());
2212 task.diagnostic = None;
2213 task.record_pass_at(
2214 &state.id,
2215 pass,
2216 state.passes.last().map(|p| p.started_at),
2217 attempts_before,
2218 Some(crate::run::ParkCause::Question),
2219 );
2220 return;
2221 }
2222 let unresolved = task.hold_notice_cause.clone();
2223 if state.base_sync_blocked() {
2224 task.hold_machine(Some(detail.to_owned()));
2226 task.last_error = Some(detail.to_owned());
2227 } else if !verdict.parked
2228 && task
2229 .pr_recovery
2230 .as_ref()
2231 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2232 && matches!(
2233 verdict.status,
2234 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2235 )
2236 {
2237 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2238 task.succeed();
2239 } else {
2240 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2241 }
2242 } else if !verdict.parked
2243 && task.pr_recovery.is_some()
2244 && matches!(
2245 verdict.status,
2246 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2247 )
2248 && !verdict.quota_hit
2249 {
2250 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2251 } else {
2252 settle_in(task, verdict, detail, max_attempts, p);
2253 }
2254 if !verdict.parked
2255 && !verdict.quota_hit
2256 && matches!(
2257 verdict.status,
2258 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2259 )
2260 {
2261 task.recovery_failed(state);
2262 }
2263 if state.prep_failure.is_some() {
2264 task.hold_notice_cause = unresolved;
2267 task.startup_notice_cause("startup-candidates");
2268 }
2269 if let Some(pass) = state.passes.last() {
2272 let parked = match &pass.end {
2273 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2274 _ => None,
2275 };
2276 task.record_pass_at(
2277 &state.id,
2278 pass.index,
2279 Some(pass.started_at),
2280 attempts_before,
2281 parked,
2282 );
2283 }
2284 if task.status == TaskStatus::Held {
2285 if state.base_sync_blocked() {
2286 task.base_sync_notice(state);
2287 } else if state.prep_failure.is_none() {
2288 task.notice_cause(
2289 label(verdict.status),
2290 Some(&state.id),
2291 state.passes.last().map(|p| p.index),
2292 );
2293 }
2294 task.diagnostic = diagnostic(state);
2295 note_open_question(task, &state.id, p);
2296 }
2297}
2298
2299fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2314 let Some(home) = crate::run::try_home() else {
2315 return;
2316 };
2317 let open = Questions::at(home.join("questions")).open_for(run);
2318 let Some(q) = open.first() else {
2319 return;
2320 };
2321 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2322 decisions.sort_unstable();
2323 task.notice_cause(
2324 &format!("questions:{}", decisions.join(",")),
2325 Some(run),
2326 None,
2327 );
2328 let base = task.hold_reason.clone().unwrap_or_default();
2329 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2330}
2331
2332fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2343 match last_run {
2344 Some(state) => {
2345 let verdict = Verdict {
2346 status: state.status,
2347 left_pr: state.pr.is_some(),
2348 quota_hit: state.quota.iter().any(|loss| {
2349 state
2350 .passes
2351 .last()
2352 .is_none_or(|pass| loss.at >= pass.started_at)
2353 }),
2354 parked: state.parked,
2355 no_viable_candidates: state.viable().is_empty(),
2356 };
2357 let detail = format!(
2358 "{}{}",
2359 phrases(&state.config.graph.language).recovered_running,
2360 describe(&state)
2361 );
2362 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2363 }
2364 None => {
2365 let why = phrases(language).no_run_to_recover;
2368 task.last_error = Some(why.to_owned());
2369 task.notice_cause("missing-run", None, None);
2372 task.hold_machine(Some(why.to_owned()));
2373 }
2374 }
2375}
2376
2377fn reclaim_orphaned_running(
2399 queue: &Queue,
2400 max_attempts: usize,
2401 live_workers: &std::collections::BTreeSet<String>,
2402) -> Vec<String> {
2403 let mut reclaimed = Vec::new();
2404 for listed in queue.list() {
2405 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2406 continue;
2407 }
2408 let Ok(_claim) = queue.claim(&listed.id) else {
2409 continue;
2410 };
2411 let Ok(mut task) = queue.get(&listed.id) else {
2415 continue;
2416 };
2417 if task.status != TaskStatus::Running {
2418 continue;
2419 }
2420 let last_run = task
2421 .execution_runs()
2422 .last()
2423 .and_then(|id| RunState::load(id).ok());
2424 if let Some(state) = &last_run
2434 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2435 {
2436 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2437 }
2438 let language = if last_run.is_none() {
2439 language_of(&task, Path::new("."))
2440 } else {
2441 String::new()
2442 };
2443 reclaim(&mut task, last_run, max_attempts, &language);
2444 if task.status == TaskStatus::Done {
2445 supersede_prior_runs(&task, &crate::run::home());
2446 }
2447 record(queue, &mut task);
2448 reclaimed.push(task.id.clone());
2449 }
2450 reclaimed
2451}
2452
2453fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2485 reclaim_abandoned_runs_with(
2486 home,
2487 now,
2488 crate::proc::pid_status,
2489 crate::proc::process_started_at,
2490 )
2491}
2492
2493fn reclaim_abandoned_runs_with<F, G>(
2499 home: &Path,
2500 now: Timestamp,
2501 query: F,
2502 identity: G,
2503) -> Vec<String>
2504where
2505 F: Fn(u32) -> Option<bool>,
2506 G: Fn(u32) -> Option<String>,
2507{
2508 let mut abandoned = Vec::new();
2509 for entry in std::fs::read_dir(home.join("runs"))
2510 .into_iter()
2511 .flatten()
2512 .flatten()
2513 {
2514 let id = entry.file_name().to_string_lossy().into_owned();
2515 if !crate::run::is_run_id(&id) {
2516 continue;
2517 }
2518 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2521 continue;
2522 };
2523 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2531 continue;
2532 };
2533 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2534 continue;
2535 };
2536 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2537 continue;
2538 }
2539 let daemon_claims = is_working_on(home, &id, now);
2551 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2552 != crate::run::Liveness::Dead
2553 {
2554 continue;
2555 }
2556 state.abandon("daemon");
2557 if let Err(e) = state.save_under(home) {
2558 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2559 continue;
2560 }
2561 if let Some(notice) = notices::run_ended(&state) {
2564 notices::raise_in(home, notice);
2565 }
2566 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2574 tracing::warn!("abandon questions for {id}: {e:#}");
2575 }
2576 abandoned.push(id);
2577 }
2578 abandoned
2579}
2580
2581#[derive(Debug, Clone, Serialize, Deserialize)]
2582struct TaskWorker {
2583 opts: Opts,
2584 queue: PathBuf,
2585 task: String,
2586 #[serde(default)]
2587 lane: PermitKind,
2588 #[serde(default)]
2589 standalone: bool,
2590}
2591
2592fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2593 Ok(crate::worker::active(home)?
2594 .into_iter()
2595 .filter(|job| job.kind == "task")
2596 .collect())
2597}
2598
2599fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2600 task_jobs(home)?
2601 .into_iter()
2602 .filter_map(
2603 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2604 Ok(request) if !request.standalone => Some(Ok(job)),
2605 Ok(_) => None,
2606 Err(error) => Some(Err(error.into())),
2607 },
2608 )
2609 .collect()
2610}
2611
2612#[must_use]
2614pub fn managed(home: &Path) -> bool {
2615 match crate::worker::active(home) {
2616 Err(error) => {
2617 tracing::error!("cannot determine scheduler ownership: {error:#}");
2618 true
2619 }
2620 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2621 Ok(_) => {
2622 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2623 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2624 }
2625 }
2626}
2627
2628pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2630 anyhow::ensure!(
2631 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2632 "the scheduler is stopping; wait for retirement before starting it again"
2633 );
2634 crate::worker::spawn(
2635 home,
2636 "scheduler",
2637 "scheduler",
2638 serde_json::to_value(opts.absolute()?)?,
2639 )
2640 .await
2641}
2642
2643fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2644 home.join("worker-parks").join(format!("{}.json", job.id))
2645}
2646
2647fn request_task_park(
2648 home: &Path,
2649 job: &crate::worker::Job,
2650 cause: crate::run::ParkCause,
2651) -> Result<()> {
2652 let path = park_request_path(home, job);
2653 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2654 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2655 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2656}
2657
2658pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2660 if control == crate::worker::Control::Park {
2661 for job in owned_task_jobs(home)? {
2662 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2663 }
2664 }
2665 crate::worker::request(home, "scheduler", control)
2666}
2667
2668pub async fn restart_detached(home: &Path) -> Result<()> {
2670 let active = crate::worker::active(home)?;
2671 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2672 if prior.is_none() && !managed(home) {
2673 return Ok(());
2674 }
2675 let opts: Opts = match prior {
2676 Some(job) => serde_json::from_value(job.payload)?,
2677 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2678 };
2679 request_control(home, crate::worker::Control::Stop)?;
2680 loop {
2681 let reserved = crate::worker::active(home)?
2682 .iter()
2683 .any(|job| job.kind == "scheduler");
2684 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2685 break;
2686 }
2687 tokio::time::sleep(Duration::from_millis(100)).await;
2688 }
2689 spawn_detached(home, &opts).await?;
2690 Ok(())
2691}
2692
2693pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2695 let opts: Opts = serde_json::from_value(payload)?;
2696 let _ = home;
2697 serve_until(opts, Stop::new()).await
2698}
2699
2700pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2702 let request: TaskWorker = serde_json::from_value(payload)?;
2703 let queue = Queue::at(request.queue);
2704 let key = format!("task:{}", request.task);
2705 let job = task_jobs(home)?
2706 .into_iter()
2707 .find(|job| job.key == key)
2708 .context("task worker reservation is missing")?;
2709 let park_path = park_request_path(home, &job);
2710 let _claim = loop {
2713 match queue.claim(&request.task) {
2714 Ok(claim) => break claim,
2715 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2718 return Err(e.context("claim the task for its worker"));
2719 }
2720 Err(_) => {
2721 sweep_stale_claims(&queue, STALE_CLAIM);
2722 tokio::time::sleep(Duration::from_millis(50)).await;
2723 }
2724 }
2725 };
2726 let mut task = queue.get(&request.task)?;
2727 if !task.status.runnable()
2728 || (!request.standalone
2729 && request.lane
2730 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2731 {
2732 return Ok(());
2733 }
2734 let stop = Stop::new();
2735 let watcher = {
2736 let home = home.to_path_buf();
2737 let stop = stop.clone();
2738 let park_path = park_path.clone();
2739 tokio::spawn(async move {
2740 loop {
2741 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2742 let cause = std::fs::read(&park_path)
2743 .ok()
2744 .and_then(|body| serde_json::from_slice(&body).ok())
2745 .unwrap_or(crate::run::ParkCause::Operator);
2746 stop.park(cause);
2747 return;
2748 }
2749 tokio::time::sleep(Duration::from_millis(200)).await;
2750 }
2751 })
2752 };
2753 let quota = if request.standalone {
2754 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2755 } else {
2756 let status = Arc::new(Mutex::new(Status::new()));
2757 Box::pin(attempt(
2758 &request.opts,
2759 &queue,
2760 &status,
2761 &stop,
2762 crate::graph::Pause::new(),
2763 &mut task,
2764 ))
2765 .await
2766 };
2767 watcher.abort();
2768 let _ = std::fs::remove_file(park_path);
2769 persist_cooldown(home, "a).await?;
2770 Ok(())
2771}
2772
2773pub async fn spawn_claimed(
2776 home: &Path,
2777 opts: &Opts,
2778 queue: &Queue,
2779 task: &Task,
2780 claim: crate::queue::Claim,
2781) -> Result<crate::worker::Job> {
2782 let request = TaskWorker {
2783 opts: opts.absolute()?,
2784 queue: std::path::absolute(queue.root())?,
2785 task: task.id.clone(),
2786 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2787 standalone: true,
2788 };
2789 let job = crate::worker::spawn(
2790 home,
2791 "task",
2792 &format!("task:{}", task.id),
2793 serde_json::to_value(request)?,
2794 )
2795 .await?;
2796 drop(claim);
2797 Ok(job)
2798}
2799
2800pub async fn run_detached_claimed(
2803 opts: &Opts,
2804 queue: &Queue,
2805 task: &mut Task,
2806 claim: crate::queue::Claim,
2807) -> Result<()> {
2808 let home = crate::run::home();
2809 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2810 loop {
2811 crate::worker::active(&home)?;
2814 let current = crate::worker::get(&home, &job.id)?;
2815 if matches!(
2816 current.status,
2817 crate::worker::Status::Finished | crate::worker::Status::Failed
2818 ) {
2819 *task = queue.get(&task.id)?;
2820 anyhow::ensure!(
2821 current.status != crate::worker::Status::Failed,
2822 "task worker failed: {}",
2823 current.result.as_deref().unwrap_or("unknown failure")
2824 );
2825 return Ok(());
2826 }
2827 tokio::time::sleep(Duration::from_millis(200)).await;
2828 }
2829}
2830
2831async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2832 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2833 let _lease = loop {
2834 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2835 break lease;
2836 }
2837 tokio::time::sleep(Duration::from_millis(10)).await;
2838 };
2839 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2840 let path = home.join("task-quota-cooldown.json");
2841 let temporary = path.with_extension("json.tmp");
2842 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2843 std::fs::rename(temporary, path)?;
2844 }
2845 }
2846 Ok(())
2847}
2848
2849fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2850 match std::fs::read(home.join("task-quota-cooldown.json")) {
2851 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2852 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2853 Err(error) => Err(error.into()),
2854 }
2855}
2856
2857pub async fn serve(opts: Opts) -> Result<()> {
2863 tracing::info!(
2864 "automatic update and restart requires magi web; standalone serve does not restart"
2865 );
2866 serve_until(opts, Stop::new()).await
2867}
2868
2869pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2880 let opts = opts.absolute()?;
2881 let home = crate::run::home();
2882 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2883 .context("another scheduler already owns this home")?;
2884 if let Some(reading) = read_status(&home)
2885 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2886 {
2887 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2888 }
2889 if !crate::worker::active(&home)?
2890 .iter()
2891 .any(|job| job.kind == "scheduler")
2892 {
2893 crate::worker::clear_control(&home, "scheduler")?;
2894 }
2895 std::fs::write(
2896 home.join("scheduler-options.json"),
2897 serde_json::to_vec(&opts)?,
2898 )?;
2899 let control = {
2900 let home = home.clone();
2901 let stop = stop.clone();
2902 tokio::spawn(async move {
2903 loop {
2904 match crate::worker::control(&home, "scheduler").ok().flatten() {
2905 Some(crate::worker::Control::Park) => {
2906 let _ = request_control(&home, crate::worker::Control::Park);
2907 stop.park(crate::run::ParkCause::Operator);
2908 break;
2909 }
2910 Some(crate::worker::Control::Stop) => {
2911 stop.stop();
2912 break;
2913 }
2914 None => {}
2915 }
2916 tokio::time::sleep(Duration::from_millis(100)).await;
2917 }
2918 })
2919 };
2920 let signal = {
2921 let stop = stop.clone();
2922 tokio::spawn(async move {
2923 if tokio::signal::ctrl_c().await.is_ok() {
2924 stop.stop();
2925 tracing::info!("scheduler shutdown requested; independent workers continue");
2926 }
2927 })
2928 };
2929
2930 let worktrees_root = opts
2931 .worktrees_root
2932 .clone()
2933 .unwrap_or_else(crate::run::default_worktree_root);
2934 let outcome = drive(
2935 &opts,
2936 &Queue::open(),
2937 &status_path(),
2938 &crate::run::home(),
2939 &worktrees_root,
2940 &stop,
2941 )
2942 .await;
2943
2944 signal.abort();
2945 control.abort();
2946 outcome
2947}
2948
2949async fn drive(
2962 opts: &Opts,
2963 queue: &Queue,
2964 status_file: &Path,
2965 home: &Path,
2966 worktrees_root: &Path,
2967 stop: &Stop,
2968) -> Result<()> {
2969 let recovery_cfg = prepare(&opts.repo, opts).ok();
2977 let daemon_cfg = recovery_cfg
2978 .as_ref()
2979 .map(|c| c.daemon.clone())
2980 .unwrap_or_default();
2981 let reconciled = crate::driver::reconcile(home);
2982 if !reconciled.is_empty() {
2983 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2984 }
2985 let mut initial_status = Status::new();
2986 let owned = owned_task_jobs(home)?;
2987 initial_status.current = current_work(home, Timestamp::now())
2988 .into_iter()
2989 .filter(|current| {
2990 owned
2991 .iter()
2992 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2993 })
2994 .collect();
2995 initial_status.idle = initial_status.current.is_empty();
2996 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2997 recovery_cfg.as_ref(),
2998 daemon_cfg.max_deputies,
2999 ));
3000 let status = Arc::new(Mutex::new(initial_status));
3001 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
3002 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
3003
3004 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
3010
3011 tracing::info!(
3012 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
3013 queue.root().display(),
3014 opts.poll.as_secs(),
3015 opts.max_attempts,
3016 concurrency,
3017 if daemon_cfg.pause_for_interrupts {
3018 ", interrupts enabled"
3019 } else {
3020 ""
3021 }
3022 );
3023
3024 let outcome = tokio::select! {
3025 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
3026 max_concurrent: concurrency,
3027 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
3028 recovery_cfg,
3029 }) => outcome,
3030 () = async {
3031 while !stop.stopped() {
3032 tokio::time::sleep(Duration::from_millis(25)).await;
3033 }
3034 } => {
3035 if stop.parking() {
3036 request_control(home, crate::worker::Control::Park)?;
3037 }
3038 Ok(())
3039 }
3040 };
3041
3042 stop.busy.store(0, Ordering::SeqCst);
3043 beat.abort();
3044 clear_status_at(status_file);
3045 outcome
3046}
3047
3048pub(crate) fn release_watch_settings_from_config(
3051 home: &Path,
3052 repo: &Path,
3053 cfg: &Config,
3054) -> (Vec<PathBuf>, u64) {
3055 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
3056 .into_iter()
3057 .map(|r| r.path)
3058 .collect();
3059 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
3062 .list()
3063 .into_iter()
3064 .map(|t| t.repo)
3065 .collect();
3066 let runs = home.join("runs");
3067 for id in crate::run::list_ids_in(&runs) {
3068 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
3069 .ok()
3070 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
3071 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
3072 known.extend(repo);
3073 }
3074 for k in known {
3075 let k = k.canonicalize().unwrap_or(k);
3076 if k.join(".git").exists() && !paths.contains(&k) {
3077 paths.push(k);
3078 }
3079 }
3080 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
3081 if !paths.contains(&own) {
3082 paths.push(own);
3083 }
3084 (paths, cfg.daemon.release_stall_minutes)
3085}
3086
3087const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
3089
3090async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
3096 loop {
3097 tokio::time::sleep(HEARTBEAT).await;
3098 let snapshot = {
3099 let mut guard = lock(&status);
3100 guard.updated_at = Timestamp::now();
3101 guard.clone()
3102 };
3103 if let Err(e) = write_status_to(&path, &snapshot) {
3104 tracing::warn!("could not refresh the daemon status file: {e:#}");
3107 }
3108 }
3109}
3110
3111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3114enum LandResume {
3115 NotLanding,
3118 StillWaiting,
3123 Ready,
3127}
3128
3129fn land_resume_state(task: &Task) -> LandResume {
3133 let Some(run_id) = task.execution_runs().last() else {
3134 return LandResume::NotLanding;
3135 };
3136 let Ok(state) = RunState::load(run_id) else {
3137 return LandResume::NotLanding;
3138 };
3139 if !state.parked {
3140 return LandResume::NotLanding;
3141 }
3142 if state.status == RunStatus::Landing
3143 && state.land_approval.as_ref().is_some_and(|binding| {
3144 ask::Questions::open()
3145 .get_for_effects(&binding.question)
3146 .is_ok_and(|q| {
3147 q.run == state.id
3148 && q.resolution()
3149 .as_deref()
3150 .is_some_and(crate::merge_decision::stops)
3151 })
3152 })
3153 {
3154 return LandResume::Ready;
3155 }
3156 if !implementers_match(task, &state) {
3157 return LandResume::NotLanding;
3158 }
3159 let node = if state.status == RunStatus::Landing {
3163 land::APPROVAL_NODE
3164 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
3165 crate::github_text::ASK_NODE
3166 } else {
3167 return LandResume::NotLanding;
3168 };
3169 let store = ask::Questions::open();
3170 let waiting = store
3171 .list_for_effects()
3172 .into_iter()
3173 .filter(|q| &q.run == run_id && q.node == node)
3174 .max_by(|a, b| a.id.cmp(&b.id));
3175 let Some(mut q) = waiting else {
3176 return LandResume::Ready;
3177 };
3178 if !q.status.open() {
3179 if q.resolution().as_deref() == Some(land::REPAIR) {
3182 return if state.liveness(false) == Liveness::Live {
3183 LandResume::StillWaiting
3184 } else {
3185 LandResume::NotLanding
3186 };
3187 }
3188 return LandResume::Ready;
3189 }
3190 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
3197 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
3198 if !store.held_reply(&q.id) && elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3199 q.abandon(format!(
3200 "no answer within {}s of asking",
3201 timeout.as_secs().max(1)
3202 ));
3203 if store.put(&mut q).is_ok() {
3206 return LandResume::Ready;
3207 }
3208 }
3209 LandResume::StillWaiting
3210}
3211
3212const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3220
3221const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3233
3234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3247enum Interrupt {
3248 Idle,
3250 Parking {
3262 parked: Vec<String>,
3263 interrupt_task: String,
3264 },
3265 Running {
3273 parked: Vec<String>,
3274 interrupt_task: String,
3275 },
3276 Resuming { parked: Vec<String> },
3283}
3284
3285fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3306 match state {
3307 Interrupt::Idle => {
3308 if in_flight.len() != 1 {
3320 return Interrupt::Idle;
3321 }
3322 match runnable.iter().find(|t| t.interrupt) {
3323 Some(t) => Interrupt::Parking {
3324 parked: in_flight.to_vec(),
3325 interrupt_task: t.id.clone(),
3326 },
3327 None => Interrupt::Idle,
3328 }
3329 }
3330 Interrupt::Parking {
3331 parked,
3332 interrupt_task,
3333 } => {
3334 if in_flight.iter().any(|id| parked.contains(id)) {
3335 Interrupt::Parking {
3337 parked,
3338 interrupt_task,
3339 }
3340 } else if in_flight.contains(&interrupt_task) {
3341 Interrupt::Running {
3342 parked,
3343 interrupt_task,
3344 }
3345 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3346 Interrupt::Parking {
3350 parked,
3351 interrupt_task,
3352 }
3353 } else {
3354 Interrupt::Resuming { parked }
3359 }
3360 }
3361 Interrupt::Running {
3362 parked,
3363 interrupt_task,
3364 } => {
3365 if in_flight.contains(&interrupt_task) {
3366 Interrupt::Running {
3367 parked,
3368 interrupt_task,
3369 }
3370 } else {
3371 Interrupt::Resuming { parked }
3377 }
3378 }
3379 Interrupt::Resuming { parked } => {
3380 if in_flight.iter().any(|id| parked.contains(id)) {
3381 Interrupt::Idle
3387 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3388 Interrupt::Resuming { parked }
3389 } else {
3390 Interrupt::Idle
3393 }
3394 }
3395 }
3396}
3397
3398fn advance_interrupt_tick(
3404 enabled: bool,
3405 state: Interrupt,
3406 in_flight: &[String],
3407 runnable: &[Task],
3408) -> Interrupt {
3409 if !enabled {
3410 return Interrupt::Idle;
3411 }
3412 advance_interrupt(state, in_flight, runnable)
3413}
3414
3415fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3420 match state {
3421 Interrupt::Idle => candidates,
3422 Interrupt::Parking {
3423 parked,
3424 interrupt_task,
3425 } => {
3426 if in_flight.iter().any(|id| parked.contains(id)) {
3427 Vec::new()
3428 } else {
3429 candidates
3430 .into_iter()
3431 .filter(|t| &t.id == interrupt_task)
3432 .collect()
3433 }
3434 }
3435 Interrupt::Running { .. } => Vec::new(),
3436 Interrupt::Resuming { parked } => candidates
3444 .into_iter()
3445 .find(|t| parked.contains(&t.id))
3446 .into_iter()
3447 .collect(),
3448 }
3449}
3450
3451struct DispatchLimits {
3455 max_concurrent: usize,
3458 pause_for_interrupts: bool,
3460 recovery_cfg: Option<Config>,
3461}
3462
3463#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3472enum PermitKind {
3473 None,
3478 Urgent,
3485 #[default]
3488 Ordinary,
3489}
3490
3491fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3494 if priority {
3495 PermitKind::None
3496 } else if urgent {
3497 PermitKind::Urgent
3498 } else {
3499 PermitKind::Ordinary
3500 }
3501}
3502
3503fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3504 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3505 (0usize, 0usize),
3506 |(ordinary, urgent), request| match request.lane {
3507 PermitKind::Ordinary => (ordinary + 1, urgent),
3508 PermitKind::Urgent => (ordinary, urgent + 1),
3509 PermitKind::None => (ordinary, urgent),
3510 },
3511 );
3512 (
3513 limit.saturating_sub(ordinary),
3514 1usize.saturating_sub(urgent),
3515 )
3516}
3517
3518#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
3520enum SkipReason {
3521 ClaimUnavailable,
3522 RereadFailed,
3523 NoLongerRunnable,
3524 WaitingOnLanding,
3525 CoolingDown,
3526 OrdinaryCapacity,
3527 UrgentCapacity,
3528 InterruptGate,
3529}
3530
3531impl SkipReason {
3532 fn key(self) -> &'static str {
3533 match self {
3534 Self::ClaimUnavailable => "claim_unavailable",
3535 Self::RereadFailed => "reread_failed",
3536 Self::NoLongerRunnable => "no_longer_runnable",
3537 Self::WaitingOnLanding => "waiting_on_landing",
3538 Self::CoolingDown => "cooling_down",
3539 Self::OrdinaryCapacity => "ordinary_capacity",
3540 Self::UrgentCapacity => "urgent_capacity",
3541 Self::InterruptGate => "interrupt_gate",
3542 }
3543 }
3544
3545 fn describe(self) -> &'static str {
3546 match self {
3547 Self::ClaimUnavailable => "could not claim it (held elsewhere or claim failed)",
3548 Self::RereadFailed => "could not re-read it",
3549 Self::NoLongerRunnable => "no longer runnable",
3550 Self::WaitingOnLanding => "waiting on landing",
3551 Self::CoolingDown => "quota cooldown",
3552 Self::OrdinaryCapacity => "no free ordinary slot",
3553 Self::UrgentCapacity => "urgent slot busy",
3554 Self::InterruptGate => "withheld by the interrupt sequence",
3555 }
3556 }
3557}
3558
3559#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3561pub struct DispatchReasonCount {
3562 pub reason: String,
3564 pub count: usize,
3566}
3567
3568#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3570pub struct DispatchWait {
3571 pub runnable: usize,
3573 pub reasons: Vec<DispatchReasonCount>,
3575}
3576
3577type DispatchSkips = std::collections::BTreeSet<(String, SkipReason)>;
3578
3579fn aggregate_dispatch_wait(runnable: usize, skips: &DispatchSkips) -> Option<DispatchWait> {
3580 if runnable == 0 {
3581 return None;
3582 }
3583 let mut counts = std::collections::BTreeMap::new();
3584 for (_, reason) in skips {
3585 *counts.entry(*reason).or_insert(0usize) += 1;
3586 }
3587 let mut counts: Vec<_> = counts.into_iter().collect();
3588 counts.sort_by(|(a, n), (b, m)| m.cmp(n).then(a.cmp(b)));
3589 Some(DispatchWait {
3590 runnable,
3591 reasons: counts
3592 .into_iter()
3593 .map(|(reason, count)| DispatchReasonCount {
3594 reason: reason.key().to_owned(),
3595 count,
3596 })
3597 .collect(),
3598 })
3599}
3600
3601fn interrupt_skips(before: &[String], after: &[Task]) -> DispatchSkips {
3602 before
3603 .iter()
3604 .filter(|id| !after.iter().any(|task| &task.id == *id))
3605 .map(|id| (id.clone(), SkipReason::InterruptGate))
3606 .collect()
3607}
3608
3609fn changed_dispatch_skips(previous: &mut DispatchSkips, current: DispatchSkips) -> bool {
3610 if *previous == current {
3611 return false;
3612 }
3613 *previous = current;
3614 true
3615}
3616
3617struct DispatchClaim {
3618 task: Task,
3619 claim: crate::queue::Claim,
3620 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3621}
3622
3623fn claim_dispatch(
3626 queue: &Queue,
3627 candidate: &Task,
3628 cooling_down: bool,
3629 ordinary: &Arc<tokio::sync::Semaphore>,
3630 urgent: &Arc<tokio::sync::Semaphore>,
3631) -> std::result::Result<DispatchClaim, SkipReason> {
3632 let claim = queue
3633 .claim(&candidate.id)
3634 .map_err(|_| SkipReason::ClaimUnavailable)?;
3635 let task = match queue.get(&candidate.id) {
3636 Ok(t) if t.status.runnable() => t,
3637 Ok(_) => return Err(SkipReason::NoLongerRunnable),
3638 Err(e) => {
3639 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3640 return Err(SkipReason::RereadFailed);
3641 }
3642 };
3643 let resume = land_resume_state(&task);
3644 if resume == LandResume::StillWaiting {
3645 return Err(SkipReason::WaitingOnLanding);
3646 }
3647 let priority = resume == LandResume::Ready;
3648 if !priority && cooling_down {
3649 return Err(SkipReason::CoolingDown);
3650 }
3651 let permit = match permit_kind(priority, task.urgent) {
3654 PermitKind::None => None,
3655 PermitKind::Urgent => Some(
3656 Arc::clone(urgent)
3657 .try_acquire_owned()
3658 .map_err(|_| SkipReason::UrgentCapacity)?,
3659 ),
3660 PermitKind::Ordinary => Some(
3661 Arc::clone(ordinary)
3662 .try_acquire_owned()
3663 .map_err(|_| SkipReason::OrdinaryCapacity)?,
3664 ),
3665 };
3666 let stamp = Timestamp::now();
3669 let mut task = task;
3670 task.dispatched_at = Some(stamp);
3671 if let Err(e) = queue.modify(&task.id, |t| {
3672 t.dispatched_at = Some(stamp);
3673 true
3674 }) {
3675 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3676 }
3677 Ok(DispatchClaim {
3678 task,
3679 claim,
3680 permit,
3681 })
3682}
3683
3684#[derive(Serialize, Deserialize)]
3703struct ServiceWorker {
3704 opts: Opts,
3705 queue: PathBuf,
3706 worktrees_root: PathBuf,
3707 operation: String,
3708}
3709
3710fn load_conductor(home: &Path) -> Result<Conductor> {
3711 match std::fs::read(home.join("conductor-state.json")) {
3712 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3713 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3714 Err(error) => Err(error.into()),
3715 }
3716}
3717
3718fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3719 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3720 .into_iter()
3721 .map(|c| c.task)
3722 .collect();
3723 let stalled = stalled_tasks(queue, home, Timestamp::now());
3724 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3725 let finished = finished_tasks(queue)
3726 .into_iter()
3727 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3728 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3729 .collect();
3730 let queued = queued_tasks(queue)
3731 .into_iter()
3732 .filter(|t| !live.contains(&t.id))
3733 .collect();
3734 (queued, stalled, finished)
3735}
3736
3737fn retention_service_receipt(
3740 home: &Path,
3741 operation: &str,
3742 phase: &str,
3743 outcome: &str,
3744 detail: &str,
3745 cfg: Option<&Config>,
3746) {
3747 if !matches!(operation, "housekeeping" | "disk-pressure") {
3748 return;
3749 }
3750 let receipt = serde_json::json!({"checked_at": Timestamp::now(),
3751 "version": env!("CARGO_PKG_VERSION"), "pid": std::process::id(),
3752 "operation": operation, "outcome": outcome, "detail": detail,
3753 "auto_fold": cfg.map(|c| c.disk.auto_fold),
3754 "reclaim_bytes": cfg.map(|c| c.disk.worktree_reclaim_bytes)});
3755 if let Err(error) = crate::fold_evidence::atomic_write(
3756 &home.join(format!("source-eviction-{operation}-{phase}.json")),
3757 &serde_json::to_vec(&receipt).expect("service receipt"),
3758 ) {
3759 tracing::warn!("retention service receipt failed: {error:#}");
3760 }
3761}
3762
3763async fn start_service(
3764 home: &Path,
3765 opts: &Opts,
3766 queue: &Queue,
3767 worktrees_root: &Path,
3768 operation: &str,
3769 cfg: Option<&Config>,
3770 busy: bool,
3771) -> Result<()> {
3772 let fetch_cfg = if operation == "fetch" {
3775 match prepare(&opts.repo, opts) {
3776 Ok(cfg) => Some(cfg),
3777 Err(error) => {
3778 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3779 return Ok(());
3780 }
3781 }
3782 } else {
3783 None
3784 };
3785 let cfg = fetch_cfg.as_ref().or(cfg);
3786 let ticks = home.join("service-ticks");
3787 std::fs::create_dir_all(&ticks)?;
3788 let receipt = ticks.join(format!("daemon-{operation}"));
3789 let last = std::fs::read(&receipt)
3790 .ok()
3791 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3792 let interval = match operation {
3793 "disk-pressure" => 30,
3794 "chat-goals" => 60,
3795 "cache" => CACHE_CHECK_INTERVAL_SECS,
3796 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3797 _ => opts.poll.as_secs().max(1),
3798 };
3799 let now = Timestamp::now();
3800 if !cache_check_due(last, now, interval) {
3801 retention_service_receipt(
3802 home,
3803 operation,
3804 "dispatch",
3805 "skipped",
3806 "cadence not due",
3807 cfg,
3808 );
3809 return Ok(());
3810 }
3811 let key = format!("service:daemon-{operation}");
3812 if crate::worker::active(home)?
3813 .iter()
3814 .any(|job| job.key == key)
3815 {
3816 retention_service_receipt(
3817 home,
3818 operation,
3819 "dispatch",
3820 "skipped",
3821 "service worker already active",
3822 cfg,
3823 );
3824 return Ok(());
3825 }
3826 let eligible = match operation {
3827 "conductor" => {
3828 let (queued, stalled, finished) = conductor_work(queue, home);
3829 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3830 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3831 }
3832 "maintenance" => {
3833 queue.list().iter().any(|task| {
3834 crate::triage::disk_wait_eligible(task)
3835 && task
3836 .disk_checked_at
3837 .is_none_or(|at| now.as_second() - at.as_second() >= 60)
3838 }) || Questions::at(home.join("questions"))
3839 .list_for_effects()
3840 .iter()
3841 .any(|q| {
3842 q.chosen_action().is_some_and(|action| action.maintenance())
3843 && !home
3844 .join("choice-actions")
3845 .join(format!("{}.json", q.id))
3846 .exists()
3847 })
3848 }
3849 "housekeeping" => !busy && runnable(queue).is_empty(),
3850 "cache" => {
3851 !busy
3852 && !runnable(queue).is_empty()
3853 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3854 }
3855 "disk-pressure" => busy,
3856 "fetch" => {
3857 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3858 }
3859 "chat-goals" => crate::talk::Talks::at(home.join("talks"))
3860 .list()
3861 .iter()
3862 .any(|talk| {
3863 talk.status.open()
3864 && talk.pending.is_empty()
3865 && talk.pending_attachments.is_empty()
3866 && talk.goals.iter().any(|goal| goal.active())
3867 }),
3868 _ => false,
3869 };
3870 if !eligible {
3871 retention_service_receipt(
3872 home,
3873 operation,
3874 "dispatch",
3875 "skipped",
3876 if operation == "housekeeping" {
3877 "active work or runnable task queue"
3878 } else {
3879 "no active work; idle housekeeping owns scan"
3880 },
3881 cfg,
3882 );
3883 return Ok(());
3884 }
3885 retention_service_receipt(
3886 home,
3887 operation,
3888 "dispatch",
3889 "launching",
3890 "eligible and cadence due",
3891 cfg,
3892 );
3893 let request = ServiceWorker {
3894 opts: opts.absolute()?,
3895 queue: std::path::absolute(queue.root())?,
3896 worktrees_root: std::path::absolute(worktrees_root)?,
3897 operation: operation.to_owned(),
3898 };
3899 if let Err(error) =
3900 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3901 {
3902 retention_service_receipt(
3903 home,
3904 operation,
3905 "dispatch",
3906 "error",
3907 &format!("worker launch: {error:#}"),
3908 cfg,
3909 );
3910 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3911 }
3912 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3913 Ok(())
3914}
3915
3916pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3919 let request: ServiceWorker = serde_json::from_value(payload)?;
3920 let opts = &request.opts;
3921 let queue = Queue::at(request.queue);
3922 let questions = Questions::at(home.join("questions"));
3923 match request.operation.as_str() {
3924 "chat-goals" => {
3925 let talks = crate::talk::Talks::at(home.join("talks"));
3926 crate::chat_goal_runtime::sweep(&talks, false).await;
3927 }
3928 "maintenance" => {
3929 apply_maintenance_choices(&queue, &questions, home).await;
3930 triage::monitor_disk_waits(
3931 &queue,
3932 &questions,
3933 opts.config.as_deref(),
3934 Timestamp::now(),
3935 );
3936 }
3937 "conductor" => {
3938 let path = home.join("conductor-state.json");
3939 let mut conductor = load_conductor(home)?;
3940 let (queued, stalled, finished) = conductor_work(&queue, home);
3941 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3942 && conductor.worth_a_look(&queue, &stalled, &finished)
3943 {
3944 let cfg = prepare(&opts.repo, opts)?;
3945 conductor
3946 .maybe_run(
3947 &cfg,
3948 &opts.repo,
3949 &queue,
3950 &questions,
3951 home,
3952 &queued,
3953 &stalled,
3954 &finished,
3955 opts.max_attempts,
3956 )
3957 .await;
3958 let temporary = path.with_extension("json.tmp");
3959 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3960 std::fs::rename(temporary, path)?;
3961 }
3962 }
3963 "fetch" => {
3964 let cfg = prepare(&opts.repo, opts)?;
3965 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3966 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3967 tracing::debug!("fetch origins: {outcome:?}");
3968 }
3969 }
3970 "cache" => {
3971 if task_jobs(home)?.is_empty() {
3972 let mut checked = None;
3973 maybe_prune_cache_between_runs(
3974 &opts.repo,
3975 opts,
3976 home,
3977 &Stop::new(),
3978 &mut checked,
3979 Timestamp::now(),
3980 )
3981 .await;
3982 }
3983 }
3984 "disk-pressure" => {
3985 if let Ok(cfg) = prepare(&opts.repo, opts) {
3986 retention_service_receipt(
3987 home,
3988 "disk-pressure",
3989 "worker",
3990 "running",
3991 "retention sweep",
3992 Some(&cfg),
3993 );
3994 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3995 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3996 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3997 }
3998 if let Err(e) = crate::build_target::prune_registered(home, &cfg.disk) {
3999 tracing::warn!("disk pressure isolated build cleanup failed: {e:#}");
4000 }
4001 for current in current_work(home, Timestamp::now()) {
4002 if let Ok(state) = RunState::load_under(¤t.run, home)
4003 && let Some(reason) = disk_gate(&state.repo, &state.config)
4004 {
4005 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
4006 job.key.strip_prefix("task:") == Some(current.task.as_str())
4007 }) {
4008 request_task_park(
4009 home,
4010 &job,
4011 crate::run::ParkCause::Interrupt {
4012 reason: reason.clone(),
4013 },
4014 )?;
4015 }
4016 notices::raise_in(
4017 home,
4018 Notice::warn(
4019 "disk:pressure",
4020 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
4021 ),
4022 );
4023 }
4024 }
4025 } else {
4026 retention_service_receipt(
4027 home,
4028 "disk-pressure",
4029 "worker",
4030 "error",
4031 "configuration could not be loaded",
4032 None,
4033 );
4034 }
4035 }
4036 "housekeeping" => {
4037 if task_jobs(home)?.is_empty() {
4038 match prepare(&opts.repo, opts) {
4039 Ok(cfg) => {
4040 retention_service_receipt(
4041 home,
4042 "housekeeping",
4043 "worker",
4044 "running",
4045 "retention sweep",
4046 Some(&cfg),
4047 );
4048 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
4049 }
4050 Err(error) => retention_service_receipt(
4051 home,
4052 "housekeeping",
4053 "worker",
4054 "error",
4055 &format!("configuration: {error:#}"),
4056 None,
4057 ),
4058 }
4059 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
4060 resweep_superseded_attempts(&queue, home);
4061 triage_held(&queue, home, opts).await;
4062 } else {
4063 retention_service_receipt(
4064 home,
4065 "housekeeping",
4066 "worker",
4067 "skipped",
4068 "task worker became active",
4069 None,
4070 );
4071 }
4072 }
4073 operation => bail!("unknown daemon service {operation}"),
4074 }
4075 Ok(())
4076}
4077
4078fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
4082 let key = format!("task:{}", request.task);
4083 let payload = serde_json::to_value(request)?;
4084 if crate::worker::active(home)?
4085 .iter()
4086 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
4087 {
4088 tracing::warn!(
4089 "task {} acquired a competing worker reservation: {error:#}",
4090 crate::run::short_of(&request.task)
4091 );
4092 return Ok(());
4093 }
4094 tracing::error!("task worker startup stopped dispatch: {error:#}");
4095 notices::raise_in(
4096 home,
4097 Notice::error(
4098 "loop:worker-startup",
4099 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
4102 ).link(Link::Task { id: request.task.clone() }),
4103 );
4104 Err(error).with_context(|| {
4105 format!(
4106 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
4107 crate::run::short_of(&request.task)
4108 )
4109 })
4110}
4111
4112async fn poll(
4113 opts: &Opts,
4114 queue: &Queue,
4115 status: &Arc<Mutex<Status>>,
4116 home: &Path,
4117 worktrees_root: &Path,
4118 stop: &Stop,
4119 limits: DispatchLimits,
4120) -> Result<()> {
4121 let mut attempted = Vec::new();
4122 let mut settling_once_services = false;
4123 let interrupt_path = home.join("scheduler-interrupt.json");
4124 let mut interrupt = match std::fs::read(&interrupt_path) {
4125 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
4126 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
4127 Err(error) => return Err(error.into()),
4128 };
4129 let mut previous = std::collections::BTreeSet::<String>::new();
4130 let mut previous_skips = DispatchSkips::new();
4131 while !stop.stopped() {
4132 let jobs: Vec<_> = crate::worker::active(home)?
4133 .into_iter()
4134 .filter(|job| job.kind == "task")
4135 .collect();
4136 let requests: Vec<_> = jobs
4137 .iter()
4138 .map(|job| {
4139 serde_json::from_value::<TaskWorker>(job.payload.clone())
4140 .with_context(|| format!("read live task worker {}", job.id))
4141 })
4142 .collect::<Result<_>>()?;
4143 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
4144 let owned: std::collections::BTreeSet<_> = requests
4145 .iter()
4146 .filter(|r| !r.standalone)
4147 .map(|r| r.task.clone())
4148 .collect();
4149 let current: Vec<_> = requests
4150 .iter()
4151 .zip(&jobs)
4152 .filter(|(request, _)| !request.standalone)
4153 .map(|(request, job)| Current {
4154 worker: Some(job.id.clone()),
4155 task: request.task.clone(),
4156 run: queue
4157 .get(&request.task)
4158 .ok()
4159 .and_then(|task| task.execution_runs().last().cloned())
4160 .unwrap_or_default(),
4161 })
4162 .collect();
4163 {
4164 let mut snapshot = lock(status);
4165 snapshot.polls += 1;
4166 snapshot.completed += previous.difference(&owned).count();
4167 snapshot.current = current;
4168 snapshot.idle = owned.is_empty();
4169 snapshot.dispatch_wait = None;
4170 }
4171 previous = owned.clone();
4172 stop.busy.store(owned.len(), Ordering::SeqCst);
4173
4174 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
4177 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
4178 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
4179 let mut permits = Vec::new();
4180 sweep_stale_claims(queue, STALE_CLAIM);
4181 reclaim_orphaned_running(queue, opts.max_attempts, &live);
4182 reclaim_abandoned_runs(home, Timestamp::now());
4183 let questions = Questions::at(home.join("questions"));
4184
4185 crate::task_chat::sweep(
4188 queue,
4189 &crate::talk::Talks::at(home.join("talks")),
4190 &questions,
4191 home,
4192 );
4193 if let Err(error) = crate::task_inquiry::Outbox::at(home)
4194 .sweep(queue, &crate::talk::Talks::at(home.join("talks")))
4195 {
4196 tracing::warn!("task inquiry delivery remains pending: {error:#}");
4197 }
4198 repair_choice_offers(&questions);
4199 resolve_blockers(queue, &questions);
4200 apply_task_choice_actions(queue, &questions, home);
4201 triage::apply_answers(queue, &questions);
4202 reconcile_task_questions(queue, &questions);
4203 let talks = crate::talk::Talks::at(home.join("talks"));
4204 crate::chat_notify::sweep(queue, &talks, home);
4205 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
4206 crate::chat_watch::sweep(home, &talks);
4207 if let Err(error) = crate::chat_control::reconcile_decisions(queue, home) {
4208 tracing::warn!("decision receipt reconciliation remains pending: {error:#}");
4209 }
4210 let mut turns = tokio::task::JoinSet::new();
4211 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
4212 while turns.join_next().await.is_some() {}
4214
4215 crate::consult::start_services(
4216 home,
4217 &opts.repo,
4218 limits.recovery_cfg.as_ref(),
4219 opts.config.as_deref(),
4220 )
4221 .await?;
4222 for operation in [
4223 "conductor",
4224 "maintenance",
4225 "housekeeping",
4226 "disk-pressure",
4227 "cache",
4228 "fetch",
4229 "chat-goals",
4230 ] {
4231 if settling_once_services {
4235 break;
4236 }
4237 if operation == "chat-goals" && stop.parking() {
4238 continue;
4239 }
4240 start_service(
4241 home,
4242 opts,
4243 queue,
4244 worktrees_root,
4245 operation,
4246 limits.recovery_cfg.as_ref(),
4247 !live.is_empty(),
4248 )
4249 .await?;
4250 }
4251 let latest = crate::worker::active(home)?;
4252 if latest.iter().any(|job| job.key == "service:daemon-cache")
4253 || latest
4254 .iter()
4255 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
4256 {
4257 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
4258 continue;
4259 }
4260 let candidates: Vec<Task> = runnable(queue)
4261 .into_iter()
4262 .filter(|task| !live.contains(&task.id))
4263 .filter(|task| !opts.once || !attempted.contains(&task.id))
4264 .collect();
4265 let in_flight: Vec<_> = owned.iter().cloned().collect();
4266 let prior_interrupt = std::mem::discriminant(&interrupt);
4267 interrupt = advance_interrupt_tick(
4268 limits.pause_for_interrupts,
4269 interrupt,
4270 &in_flight,
4271 &candidates,
4272 );
4273 if prior_interrupt != std::mem::discriminant(&interrupt) {
4276 let temporary = interrupt_path.with_extension("json.tmp");
4277 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
4278 std::fs::rename(temporary, &interrupt_path)?;
4279 }
4280 if let Interrupt::Parking {
4281 parked,
4282 interrupt_task,
4283 } = &interrupt
4284 {
4285 for job in jobs.iter().filter(|job| {
4286 parked
4287 .iter()
4288 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
4289 }) {
4290 if crate::worker::control(home, &job.key)?.is_none() {
4291 request_task_park(
4292 home,
4293 job,
4294 crate::run::ParkCause::Interrupt {
4295 reason: format!(
4296 "task {} asked to run first",
4297 crate::run::short_of(interrupt_task)
4298 ),
4299 },
4300 )?;
4301 }
4302 }
4303 }
4304 let candidate_ids: Vec<_> = candidates.iter().map(|task| task.id.clone()).collect();
4305 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
4306 let mut skips = interrupt_skips(&candidate_ids, &candidates);
4307 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
4308 let mut started = false;
4309 let mut resync = false;
4310 for candidate in candidates {
4311 if stop.stopped() {
4312 break;
4313 }
4314 let DispatchClaim {
4315 task,
4316 claim,
4317 permit,
4318 } = match claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem) {
4319 Ok(claim) => claim,
4320 Err(reason) => {
4321 skips.insert((candidate.id, reason));
4322 continue;
4323 }
4324 };
4325 if crate::worker::active(home)?
4326 .iter()
4327 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
4328 {
4329 drop(claim);
4330 resync = true;
4331 break;
4332 }
4333 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
4334 let request = TaskWorker {
4335 opts: opts.absolute()?,
4336 queue: std::path::absolute(queue.root())?,
4337 task: task.id.clone(),
4338 lane,
4339 standalone: false,
4340 };
4341 if let Err(error) = crate::worker::spawn(
4344 home,
4345 "task",
4346 &format!("task:{}", task.id),
4347 serde_json::to_value(&request)?,
4348 )
4349 .await
4350 {
4351 task_worker_spawn_error(home, &request, error)?;
4352 drop(claim);
4353 resync = true;
4354 break;
4355 }
4356 drop(claim);
4357 permits.extend(permit);
4358 attempted.push(task.id);
4359 started = true;
4360 }
4361 if !resync && !stop.stopped() {
4362 if !started {
4363 lock(status).dispatch_wait = aggregate_dispatch_wait(candidate_ids.len(), &skips);
4364 }
4365 if changed_dispatch_skips(&mut previous_skips, skips) {
4366 if previous_skips.is_empty() {
4367 tracing::info!("dispatch skips cleared");
4368 }
4369 for (task, reason) in &previous_skips {
4370 tracing::info!(
4371 task,
4372 reason = reason.key(),
4373 "task {} skipped: {}",
4374 crate::run::short_of(task),
4375 reason.describe()
4376 );
4377 }
4378 }
4379 }
4380 if started || resync {
4381 continue;
4382 }
4383 if owned.is_empty() && opts.once {
4384 settling_once_services = true;
4385 if crate::worker::active(home)?.iter().any(|job| {
4389 job.kind == "daemon-service"
4390 && matches!(
4391 job.key.as_str(),
4392 "service:daemon-housekeeping"
4393 | "service:daemon-conductor"
4394 | "service:daemon-maintenance"
4395 )
4396 }) {
4397 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
4398 continue;
4399 }
4400 break;
4401 }
4402 stop.idle(if live.is_empty() {
4403 opts.poll
4404 } else {
4405 RECHECK_WHILE_BUSY.min(opts.poll)
4406 })
4407 .await;
4408 }
4409 stop.busy.store(0, Ordering::SeqCst);
4412 lock(status).dispatch_wait = None;
4413 Ok(())
4414}
4415
4416async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
4420 let busy = |error: &anyhow::Error| {
4421 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
4422 error.kind() == std::io::ErrorKind::WouldBlock
4423 || cfg!(windows) && error.raw_os_error() == Some(33)
4424 })
4425 };
4426 match crate::pr_head::owner_lock(home, id) {
4427 Ok(lock) => return Ok(lock),
4428 Err(error) if busy(&error) => {}
4429 Err(error) => return Err(error),
4430 }
4431 let wait = RunState::load_under(id, home)?
4432 .config
4433 .graph
4434 .timeout_resource_wait;
4435 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
4436}
4437
4438async fn public_resume_owner(
4442 home: &Path,
4443 runs: &[String],
4444 repo: &Path,
4445) -> Result<Option<RunState>> {
4446 let mut owner = None;
4447 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
4448 let _lock = resume_owner_lock(home, id).await?;
4449 let state = RunState::load_under(id, home)?;
4450 if state.released()
4451 || state.retired_to.is_some()
4452 || state.pr_taken_over_by.is_some()
4453 || !crate::handover::landing_checkpoint(&state)
4454 {
4455 continue;
4456 }
4457 anyhow::ensure!(
4458 crate::git::toplevel(&state.repo).await?.canonicalize()?
4459 == crate::git::toplevel(repo).await?.canonicalize()?,
4460 "public owner repository differs; inspect run lineage"
4461 );
4462 anyhow::ensure!(
4463 owner.is_none(),
4464 "multiple retained public owners; inspect run lineage"
4465 );
4466 if let Some(reason) = crate::handover::resume_refusal_under(&state, Some(home)) {
4467 anyhow::bail!(reason.diagnostic(&state));
4468 }
4469 let claimed = is_working_on_other(home, id, Timestamp::now())?;
4470 anyhow::ensure!(
4471 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
4472 "public owner's driver is live or unknown; wait before retrying"
4473 );
4474 owner = Some(state);
4475 }
4476 Ok(owner)
4477}
4478
4479async fn attempt(
4485 opts: &Opts,
4486 queue: &Queue,
4487 status: &Arc<Mutex<Status>>,
4488 stop: &Stop,
4489 interrupt_pause: crate::graph::Pause,
4490 task: &mut Task,
4491) -> Vec<QuotaLoss> {
4492 let repo = repo_for(task, &opts.repo);
4493 tracing::info!(
4494 "task {} — {} (repo {})",
4495 task.short(),
4496 task.title,
4497 repo.display()
4498 );
4499
4500 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
4501 Ok(owner) => owner,
4502 Err(e) => {
4503 task.hold_machine(Some(format!(
4504 "Cannot verify existing public owner: {e:#}; no attempt started."
4505 )));
4506 record(queue, task);
4507 return Vec::new();
4508 }
4509 };
4510 let resume_runs = public_owner
4511 .as_ref()
4512 .map(|s| vec![s.id.clone()])
4513 .unwrap_or_else(|| task.execution_runs().to_vec());
4514
4515 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4519 && previous
4520 .merge_decision
4521 .as_ref()
4522 .is_some_and(|decision| !decision.closed)
4523 && previous.status != RunStatus::Landing
4524 && !crate::merge_decision::resumable_hold(&previous)
4525 {
4526 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4527 record(queue, task);
4528 return Vec::new();
4529 }
4530 let stop_owner = resume_runs
4531 .last()
4532 .and_then(|id| RunState::load(id).ok())
4533 .filter(|state| {
4534 crate::merge_decision::resumable_hold(state)
4535 || state.status == RunStatus::Landing
4536 && state.land_approval.as_ref().is_some_and(|binding| {
4537 Questions::open()
4538 .get_for_effects(&binding.question)
4539 .is_ok_and(|q| {
4540 q.run == state.id
4541 && q.resolution()
4542 .as_deref()
4543 .is_some_and(crate::merge_decision::stops)
4544 })
4545 })
4546 });
4547 if let Some(owner) = stop_owner {
4548 let mut runner = match Runner::resume(&owner.id) {
4549 Ok(runner) => runner,
4550 Err(e) => {
4551 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4552 record(queue, task);
4553 return Vec::new();
4554 }
4555 };
4556 runner.on_pause(stop.pause());
4557 runner.watch_interrupt(interrupt_pause);
4558 let quota_before = runner.state.quota.clone();
4559 let before_start = task.clone();
4560 task.start(owner.id.clone());
4561 if let Err(error) = queue.put(task) {
4562 *task = before_start;
4563 let error = runner.fail_startup("task-start", error);
4564 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4565 task.record_startup_failure(failure.clone());
4566 }
4567 task.hold_manual(Some(format!(
4568 "Could not persist task start; no agents dispatched: {error:#}"
4569 )));
4570 record(queue, task);
4571 return Vec::new();
4572 }
4573 lock(status).current.push(Current {
4574 worker: None,
4575 task: task.id.clone(),
4576 run: owner.id,
4577 });
4578 runner.state.carry_dispatched_resumes(task);
4579 runner.state.next_pass_task_entry = true;
4580 let result = runner.execute().await;
4581 return finish_attempt(
4582 opts.max_attempts,
4583 queue,
4584 task,
4585 &runner.state,
4586 "a_before,
4587 result,
4588 );
4589 }
4590
4591 if let Some(owner) = &public_owner
4592 && (task.fresh_start || !may_resume_run(owner, task))
4593 {
4594 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4595 record(queue, task);
4596 return Vec::new();
4597 }
4598
4599 if let Some(state) = task
4600 .execution_runs()
4601 .last()
4602 .and_then(|id| RunState::load(id).ok())
4603 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
4604 {
4605 task.hold_machine(Some(reason));
4606 record(queue, task);
4607 return Vec::new();
4608 }
4609
4610 if let Some(held) = crate::base_sync_recovery::held_run(queue, task)
4611 && !task.fresh_start
4612 && task.pr_recovery.is_none()
4613 {
4614 let authorized = if crate::repair_budget::conflict_pending(task, &held) {
4617 true
4618 } else if task.resumption.is_some() && compatible_run_instruction(&held, task) {
4619 let home = queue.root().parent().expect("queue has a home");
4620 match Runner::authorize_base_recovery_from_resumption(home, &held.id, &task.id).await {
4621 Ok(()) => true,
4622 Err(error) => {
4623 tracing::debug!(run = %held.id, %error, "base recovery resumption refused");
4624 false
4625 }
4626 }
4627 } else {
4628 false
4629 };
4630 if !authorized {
4631 task.hold_machine(Some(format!("Base synchronization remains stopped. {} A release without a new owner approval grants no additional synchronization budget and starts no replacement competition.", held.base_sync_next_action())));
4632 task.base_sync_notice(&held);
4634 task.resume_override = None;
4635 record(queue, task);
4636 return Vec::new();
4637 }
4638 }
4639 let unfinished = (!task.fresh_start)
4640 .then(|| unfinished_run(&resume_runs, task.short()))
4641 .flatten()
4642 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4643 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4644 if let Some(id) = &task.disk_resume_run
4645 && unfinished.as_ref() != Some(id)
4646 {
4647 task.hold_manual(Some("Automatic disk recovery cannot resume the retained execution safely. Inspect its state and recovery approvals; no replacement competition was started.".into()));
4648 record(queue, task);
4649 return Vec::new();
4650 }
4651
4652 if !task.fresh_start
4653 && task.pr_recovery.is_none()
4654 && task.review_branch.is_none()
4655 && task.resume_override.is_none()
4656 && previous
4657 .as_ref()
4658 .is_some_and(|run| run.retired_to.is_some())
4659 && unfinished.is_none()
4660 {
4661 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4662 record(queue, task);
4663 return Vec::new();
4664 }
4665
4666 let saved_recovery = previous.as_ref().filter(|run| {
4667 task.replacement.is_none()
4668 && task.pr_recovery.as_ref().is_some_and(|selected| {
4669 selected
4670 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4671 .is_ok()
4672 && selected_starter(selected, Some(run), unfinished.as_deref())
4673 == Starter::Resume(run.id.clone())
4674 })
4675 });
4676
4677 let explicit = match &task.overrides {
4681 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4682 Some(_) => opts.config.as_deref(),
4683 None => opts.config.as_deref(),
4684 };
4685 let (mut config, layers) = if let Some(run) = saved_recovery {
4686 let mut saved = run.clone();
4689 if let Err(e) = saved.configure_recovery_resume() {
4690 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4691 record(queue, task);
4692 return Vec::new();
4693 }
4694 (saved.config, Vec::new())
4695 } else {
4696 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4697 Ok(loaded) => loaded,
4698 Err(e) => {
4699 let reason = format!("[config] {e:#}");
4700 record_config_failure(task, &e);
4701 task.last_error = Some(reason.clone());
4702 task.hold_machine(Some(reason));
4703 record(queue, task);
4704 return Vec::new();
4705 }
4706 };
4707 if let Err(e) = apply_options(&mut config, opts, task) {
4708 task.hold_manual(Some(format!(
4709 "[config] {e:#}; correct the merge override before releasing the task"
4710 )));
4711 record(queue, task);
4712 return Vec::new();
4713 }
4714 if let Some(selected) = &task.pr_recovery
4715 && let Err(e) = selected.configure(&mut config)
4716 {
4717 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4718 record(queue, task);
4719 return Vec::new();
4720 }
4721 (config, layers)
4722 };
4723 let config_evidence = crate::pr_recovery::ConfigEvidence {
4724 layers,
4725 overrides: task.overrides.clone(),
4726 daemon_merge: task
4727 .overrides
4728 .as_ref()
4729 .is_none_or(|o| o.inherit_daemon_options)
4730 .then(|| opts.merge.clone())
4731 .flatten(),
4732 };
4733 apply_task_implementers(&mut config, task);
4734 if let Some(replacement) = &task.replacement {
4735 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4736 old.liveness(false) == Liveness::Dead
4737 && old.merge_decision.as_ref().is_some_and(|d| {
4738 d.closed
4739 && d.question == replacement.question
4740 && d.replacement_run.as_ref().is_none_or(|run| {
4741 task.execution_runs().last() == Some(run)
4742 && RunState::load(run).is_ok_and(|current| {
4743 current.replaces.as_ref().is_some_and(|pin| {
4744 pin.question == replacement.question
4745 && pin.agent == replacement.agent
4746 })
4747 })
4748 })
4749 })
4750 });
4751 if !safe {
4752 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4753 record(queue, task);
4754 return Vec::new();
4755 }
4756 match config.agent(&replacement.agent) {
4757 Ok(agent) if crate::agent::installed(agent) => {}
4758 _ => {
4759 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4760 record(queue, task);
4761 return Vec::new();
4762 }
4763 }
4764 config.graph.implementers = 1;
4765 config.graph.land_approval = true;
4766 config.merge.mode = MergeMode::Pr;
4767 config.graph.selected_implementer = Some(replacement.agent.clone());
4768 }
4769 let p = phrases(&config.graph.language);
4770 let start_failed = p.could_not_start;
4771 if let Ok(base) =
4776 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4777 .await
4778 && let Ok(head) =
4779 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4780 {
4781 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4782 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4783 let source = crate::git::rev_parse(&repo, branch)
4784 .await
4785 .unwrap_or_default();
4786 inputs.push_str(&format!(";review:{branch}:{source}"));
4787 }
4788 if let Some(reason) = task
4789 .recovery_exhausted(
4790 &head,
4791 &inputs,
4792 task.attempt_limit
4793 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4794 )
4795 .filter(|_| {
4796 !task.repair_grants.last().is_some_and(|receipt| {
4797 RunState::load(&receipt.request.run)
4798 .is_ok_and(|run| crate::repair_budget::pending(task, &run))
4799 })
4800 })
4801 {
4802 task.hold_unrequested_retry(Some(reason));
4803 record(queue, task);
4804 return Vec::new();
4805 }
4806 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4807 }
4808
4809 let consume_review = !previous
4830 .as_ref()
4831 .is_some_and(|state| crate::repair_budget::pending(task, state));
4832 let review_branch = consume_review
4833 .then(|| review_branch_for(task, previous.as_ref()))
4834 .flatten();
4835 let branch_exists = match review_branch {
4836 Some(branch) => crate::git::branch_exists(&repo, branch)
4837 .await
4838 .unwrap_or(false),
4839 None => false,
4840 };
4841 let proposed = review_retry_starter(
4845 task,
4846 previous.as_ref(),
4847 review_branch,
4848 branch_exists,
4849 unfinished.as_deref(),
4850 );
4851 let retry_selected = task.replacement.is_some()
4852 && task.resumption.is_some()
4853 && !task.fresh_start
4854 && previous.as_ref().is_some_and(|run| {
4855 run.replaces.is_some()
4856 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4857 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4858 });
4859 let stop_pin = task
4860 .resume_override
4861 .as_ref()
4862 .filter(|pin| {
4863 Questions::open()
4864 .get_for_effects(&pin.question_id)
4865 .is_ok_and(|q| {
4866 q.node == land::APPROVAL_NODE
4867 && q.resolution()
4868 .as_deref()
4869 .is_some_and(crate::merge_decision::stops)
4870 })
4871 })
4872 .and_then(|pin| pin.pinned_run.clone());
4873 let granted_run = task.repair_grants.last().and_then(|receipt| {
4874 let state = RunState::load(&receipt.request.run).ok()?;
4875 crate::repair_budget::pending(task, &state).then(|| receipt.request.run.clone())
4876 });
4877 let starter = if let Some(run) = granted_run {
4878 Starter::Resume(run)
4879 } else if let Some(owner) = &public_owner {
4880 Starter::Resume(owner.id.clone())
4881 } else if let Some(run) = stop_pin {
4882 Starter::Resume(run)
4885 } else if retry_selected {
4886 Starter::Resume(previous.as_ref().unwrap().id.clone())
4887 } else if let Some(selected) = &task.pr_recovery {
4888 if selected
4889 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4890 .is_ok()
4891 {
4892 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4893 } else {
4894 Starter::Review(selected.branch.clone())
4895 }
4896 } else {
4897 starter_with_implementers(task, previous.as_ref(), proposed)
4898 };
4899 let disk_reason = match &starter {
4902 Starter::Resume(id) => previous.as_ref().filter(|run| &run.id == id).map_or_else(
4903 || {
4904 RunState::load(id)
4905 .ok()
4906 .and_then(|run| disk_gate(&run.repo, &run.config))
4907 },
4908 |run| disk_gate(&run.repo, &run.config),
4909 ),
4910 _ => disk_gate(&repo, &config),
4911 };
4912 if let Some(reason) = disk_reason {
4913 task.notice_cause("disk-space", None, None);
4914 task.last_error = Some(reason.clone());
4915 let disk_checked_at = task.disk_checked_at;
4916 task.hold_machine(Some(reason.clone()));
4917 if matches!(starter, Starter::Resume(_)) {
4918 task.mark_disk_capacity();
4919 } else {
4920 task.mark_startup_disk_capacity();
4921 }
4922 task.disk_checked_at = disk_checked_at;
4923 record(queue, task);
4924 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4925 notices::raise(
4926 Notice::warn(
4927 &format!("disk:{}", repo.display()),
4928 format!("Task {} is waiting for disk space. It will resume automatically when disk and output admission checks succeed.", task.id),
4929 )
4930 .link(Link::Task {
4931 id: task.id.clone(),
4932 }),
4933 );
4934 return Vec::new();
4935 }
4936 let review_branch = if consume_review {
4937 let valid = review_branch.is_some();
4938 task.review_instruction = None;
4939 task.review_branch.take().filter(|_| valid)
4940 } else {
4941 None
4942 };
4943 if let Some(pin) = &task.resume_override
4946 && let Some(run) = &pin.pinned_run
4947 && starter != Starter::Resume(run.clone())
4948 {
4949 let run = run.clone();
4950 task.notice_cause("resume-owner", Some(&run), None);
4951 task.hold_machine(Some(format!(
4952 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4953 task.id
4954 )));
4955 record(queue, task);
4956 return Vec::new();
4957 }
4958 if let Some(run) = &task.disk_resume_run
4962 && starter != Starter::Resume(run.clone())
4963 {
4964 task.hold_manual(Some("Automatic disk recovery cannot resume the retained execution safely: the task's current settings would start a different run. Inspect its state and recovery approvals; no replacement competition was started.".into()));
4965 record(queue, task);
4966 return Vec::new();
4967 }
4968 if task.replacement.is_some()
4969 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4970 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4971 {
4972 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4973 record(queue, task);
4974 return Vec::new();
4975 }
4976 if let Some(reason) = retained_attempt_hold(
4977 task,
4978 &starter,
4979 config.disk.retained_attempt_limit,
4980 RunState::load,
4981 ) {
4982 if let Starter::Review(branch) = &starter {
4985 task.review_branch = Some(branch.clone());
4986 }
4987 task.hold_unrequested_retry(Some(reason));
4988 record(queue, task);
4989 return Vec::new();
4990 }
4991 let attachments = match task_attachments(queue, task) {
4996 Ok(a) => a,
4997 Err(e) => {
4998 task.attempts += 1;
4999 task.fail_startup_target(
5000 "startup-attachments",
5001 e.name.clone(),
5002 format!("{start_failed}{e:#}"),
5003 opts.max_attempts,
5004 );
5005 record(queue, task);
5006 return Vec::new();
5007 }
5008 };
5009 let startup_pause = stop.pause();
5010 let startup_interrupt = interrupt_pause.clone();
5011 let wait = crate::repo_lock::WaitPolicy::new(
5012 Duration::from_secs(config.graph.timeout_resource_wait),
5013 move || startup_pause.parked() || startup_interrupt.parked(),
5014 );
5015 let started = wait.scope(Box::pin(async { match &starter {
5018 Starter::Review(_) if task.pr_recovery.is_some() => {
5019 let selected = task.pr_recovery.clone().expect("selected recovery");
5020 match selected.check_task_assignment(&task.id, &task.instruction, task.reassignments.len()) {
5021 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
5022 .await
5023 .map(|mut r| {
5024 r.state.attachments = attachments.clone();
5025 r
5026 }),
5027 Err(e) => Err(e),
5028 }
5029 }
5030 Starter::Review(branch) => {
5031 tracing::info!(
5032 "task {} reopens `{branch}` as a review-only pass",
5033 task.short()
5034 );
5035 let takeover = crate::handover::Takeover {
5038 earlier: task.earlier_attempts().to_vec(),
5039 home: crate::run::home(),
5040 choice: take_divergence_answer(branch, &config.merge.remote, task),
5041 task_instruction: Some(instruction_for(task)),
5042 };
5043 Runner::review_taking_over(
5044 &repo,
5045 branch,
5046 config,
5047 Some(takeover),
5048 crate::run::Origin::queue(&task.id),
5049 )
5050 .await
5051 }
5052 Starter::Resume(id) => {
5053 tracing::info!("resuming run {id} rather than competing again");
5054 Runner::resume(id).and_then(|mut r| {
5055 if retry_selected {
5056 for candidate in &mut r.state.candidates {
5059 candidate.failed = None;
5060 candidate.empty = false;
5061 candidate.commits = 0;
5062 }
5063 r.state.status = RunStatus::Implementing;
5064 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
5065 }
5066 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
5067 r.state.sources = task.sources.clone();
5068 }
5069 if r.state.prep_failure.is_some() {
5070 let approval = task
5071 .resumption
5072 .as_ref()
5073 .context("preparation retry requires an explicit task release")?;
5074 r.retry_preparation(approval.at)?;
5075 }
5076 if let Some(instruction) =
5077 prepare_instruction(&starter, Some(&r.state.instruction), task)
5078 {
5079 r.state.instruction = instruction;
5080 }
5081 r.state.attachments = attachments.clone();
5082 if task.replacement.is_some() {
5083 r.state.config.merge.mode = MergeMode::Pr;
5084 r.state.config.graph.land_approval = true;
5085 }
5086 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
5090 .overrides
5091 .as_ref()
5092 .and_then(|o| o.merge.as_deref())
5093 .and_then(|m| merge_mode(m).ok())
5094 {
5095 r.state.config.merge.mode = mode;
5096 }
5097 Ok(r)
5098 })
5099 }
5100 Starter::Start => {
5101 if let Some(branch) = &review_branch {
5102 tracing::warn!(
5103 "conductor chose review for task {} but branch `{branch}` no longer \
5104 exists; requeuing as a fresh competition instead",
5105 task.short()
5106 );
5107 }
5108 let instruction = prepare_instruction(&starter, None, task)
5109 .unwrap_or_else(|| task.instruction.clone());
5110 Runner::start_naming(
5111 &repo,
5112 instruction,
5113 &task.title,
5114 config,
5115 crate::run::Origin::queue(&task.id),
5116 )
5117 .await
5118 .map(|mut r| {
5119 r.state.attachments = attachments.clone();
5120 r.state.sources = task.sources.clone();
5121 r
5122 })
5123 }
5124 }})).await;
5125 if let Err(error) = &started
5126 && let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>()
5127 {
5128 task.record_startup_failure(failure.clone());
5129 }
5130 let mut runner = match started {
5131 Ok(r) => r,
5132 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5133 let detail = format!("{e:#}");
5134 task.last_error = Some(detail.clone());
5135 if e.is::<crate::build_target::AdmissionDiskWait>()
5136 && let Starter::Review(branch) = &starter
5137 && review_branch.is_some()
5138 {
5139 task.review_branch = Some(branch.clone());
5140 task.review_instruction = Some(task.instruction.clone());
5141 }
5142 if stop.parking() || interrupt_pause.parked() {
5143 let attempts = task.attempts;
5144 task.park(e.to_string());
5145 task.attempts = attempts;
5147 } else if e.is::<crate::build_target::AdmissionDiskWait>() {
5148 let disk_checked_at = task.disk_checked_at;
5149 task.hold_machine(Some(detail));
5150 task.mark_startup_disk_capacity();
5151 task.disk_checked_at = disk_checked_at;
5152 } else {
5153 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
5154 }
5155 record(queue, task);
5156 return Vec::new();
5157 }
5158 Err(e) if task.pr_recovery.is_some() => {
5162 task.startup_notice_cause("startup-public-head");
5163 task.attempts += 1;
5164 task.last_error = Some(format!("{start_failed}{e:#}"));
5165 task.hold_unrequested_retry(Some(format!(
5166 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
5167 )));
5168 record(queue, task);
5169 return Vec::new();
5170 }
5171 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
5172 let detail = match e.downcast_ref::<crate::handover::Refused>() {
5173 Some(r) => (p.handover_refused)(r),
5174 None => format!("{e:#}"),
5175 };
5176 task.startup_notice_cause("startup-handover");
5177 let reason = format!("{start_failed}{detail}");
5178 task.last_error = Some(reason.clone());
5179 let branch = match &starter {
5182 Starter::Review(branch) => Some(branch.clone()),
5183 _ => None,
5184 };
5185 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
5189 record(queue, task);
5190 tracing::warn!(
5191 "holding {} for a branch it cannot take over: {e:#}",
5192 task.short()
5193 );
5194 return Vec::new();
5195 }
5196 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
5201 let d = e
5202 .downcast_ref::<crate::reconcile::Diverged>()
5203 .expect("checked by the guard");
5204 let mut q = ask::Question::new(
5205 task.id.clone(),
5206 crate::reconcile::NODE.to_owned(),
5207 crate::reconcile::SEAT.to_owned(),
5208 d.summary(),
5209 d.detail(),
5210 d.choices(),
5211 );
5212 match Questions::open().put(&mut q) {
5213 Ok(()) => {
5214 task.last_error = Some(format!("{e:#}"));
5215 task.review_branch = Some(d.branch.clone());
5218 task.review_instruction = Some(task.instruction.clone());
5219 task.block(vec![q.id.clone()], Some(d.summary()));
5220 }
5221 Err(put) => {
5222 tracing::warn!("could not file the divergence question: {put:#}");
5223 task.attempts += 1;
5224 task.fail_startup(
5225 "startup-question-store",
5226 format!("{start_failed}{e:#}"),
5227 opts.max_attempts,
5228 );
5229 }
5230 }
5231 record(queue, task);
5232 return Vec::new();
5233 }
5234 Err(e) => {
5235 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
5241 && let Starter::Review(branch) = &starter
5242 {
5243 task.review_branch = Some(branch.clone());
5244 task.review_instruction = Some(task.instruction.clone());
5245 task.last_error = Some(format!("{start_failed}{e:#}"));
5246 task.status = crate::queue::TaskStatus::Failed;
5247 record(queue, task);
5248 return Vec::new();
5249 }
5250 task.attempts += 1;
5251 task.last_error = Some(format!("{start_failed}{e:#}"));
5252 task.startup_notice_cause("startup-source");
5253 task.hold_unrequested_retry(Some(format!(
5254 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
5255 )));
5256 record(queue, task);
5257 return Vec::new();
5258 }
5259 };
5260 if !matches!(starter, Starter::Resume(_))
5261 && let Some((mode, source)) = task.build_cache_override(&runner.state.config)
5262 {
5263 runner
5264 .state
5265 .event("build-cache", format!("{mode} selected by {source}"));
5266 }
5267 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
5268 runner.state.recovery_config = Some(config_evidence);
5269 }
5270 if let Some(replacement) = &task.replacement {
5271 if matches!(starter, Starter::Start) {
5272 runner.state.instruction.push_str(&format!(
5273 "\n\n# Replacement review context\n{}",
5274 replacement.context
5275 ));
5276 }
5277 runner.state.replaces = Some(replacement.clone());
5278 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
5279 if let Ok(mut old) = RunState::load(&replacement.run)
5280 && let Some(d) = &mut old.merge_decision
5281 {
5282 d.replacement_run = Some(runner.state.id.clone());
5283 old.event(
5284 "replacement",
5285 format!(
5286 "replacement run {} owns the next attempt; old PR remains closed",
5287 runner.state.id
5288 ),
5289 );
5290 if let Err(e) = old.save() {
5291 let error = runner.fail_startup("replacement-history", e);
5292 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
5293 task.record_startup_failure(failure.clone());
5294 }
5295 task.hold_manual(Some(format!(
5296 "Could not link replacement history: {error:#}; no agents dispatched"
5297 )));
5298 record(queue, task);
5299 return Vec::new();
5300 }
5301 }
5302 }
5303 runner.state.task_instruction = Some(task.instruction.clone());
5304 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
5307 if runner.state.origin_chat.is_none() {
5308 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
5309 }
5310 runner.on_pause(stop.pause());
5312 runner.watch_interrupt(interrupt_pause);
5316
5317 let run = runner.state.id.clone();
5320 let before_start = task.clone();
5321 task.start(run.clone());
5322 if let Err(error) = queue.put(task) {
5323 *task = before_start;
5324 let error = runner.fail_startup("task-start", error);
5325 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
5326 task.record_startup_failure(failure.clone());
5327 }
5328 task.hold_manual(Some(format!(
5329 "Could not persist task start; no agents dispatched: {error:#}"
5330 )));
5331 record(queue, task);
5332 return Vec::new();
5333 }
5334 lock(status).current.push(Current {
5335 worker: None,
5336 task: task.id.clone(),
5337 run,
5338 });
5339
5340 let quota_before = runner.state.quota.clone();
5343 runner.state.carry_dispatched_resumes(task);
5344 runner.state.next_pass_task_entry = true;
5345 let result = runner.execute().await;
5346 finish_attempt(
5347 opts.max_attempts,
5348 queue,
5349 task,
5350 &runner.state,
5351 "a_before,
5352 result,
5353 )
5354}
5355
5356pub fn finish_attempt(
5361 max_attempts: usize,
5362 queue: &Queue,
5363 task: &mut Task,
5364 state: &RunState,
5365 quota_before: &[QuotaLoss],
5366 result: Result<()>,
5367) -> Vec<QuotaLoss> {
5368 let detail = match result {
5369 Ok(()) => describe(state),
5370 Err(e) => format!("{e:#}"),
5371 };
5372 let fresh = losses_this_attempt(quota_before, &state.quota);
5373 let verdict = Verdict {
5374 status: state.status,
5375 left_pr: state.pr.is_some(),
5378 quota_hit: !fresh.is_empty(),
5384 parked: state.parked,
5388 no_viable_candidates: state.viable().is_empty(),
5391 };
5392 if let Some(replacement) = &state.replaces
5393 && let Some(pr) = &state.pr
5394 && let Ok(mut old) = RunState::load(&replacement.run)
5395 && let Some(d) = &mut old.merge_decision
5396 {
5397 d.replacement_pr = Some(pr.url.clone());
5398 let _ = old.save();
5399 }
5400 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
5401 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
5402 task.hold_unrequested_retry(Some(format!(
5403 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
5404 task.short()
5405 )));
5406 }
5407 if task.status == TaskStatus::Done {
5408 supersede_prior_runs(task, &crate::run::home());
5409 }
5410 record(queue, task);
5411 tracing::info!(
5412 "task {} is {} after run {} ({})",
5413 task.short(),
5414 task.status.as_str(),
5415 state.short(),
5416 label(state.status)
5417 );
5418 fresh
5419}
5420
5421pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
5425 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
5428 && !awaiting_resume_with(task, RunState::load)
5429 && withheld_text_wait(task).is_none()
5433 && !(task.fresh_start && task.replacement.is_some())
5434 {
5435 let why = task.last_error.clone().map_or_else(
5436 || "the run did not finish".to_owned(),
5437 |e| format!("the run did not finish: {e}"),
5438 );
5439 task.hold_unrequested_retry(Some(format!(
5440 "{why}. It was started by hand, so it is not retried \
5441 automatically; `magi task release` retries it."
5442 )));
5443 record(queue, task);
5444 }
5445}
5446
5447#[must_use]
5451pub fn loop_would_resume(run: &str) -> bool {
5452 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
5453}
5454
5455#[must_use]
5465pub fn foreign_loop(
5466 reading: Option<&Reading>,
5467 now: Timestamp,
5468 own_pid: u32,
5469) -> Option<Option<u32>> {
5470 let reading = reading.filter(|r| r.running(now))?;
5471 match reading.pid {
5472 Some(pid) if pid == own_pid => None,
5473 pid => Some(pid),
5474 }
5475}
5476
5477fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
5478 reading.worker_protocol != Some(1)
5479 && (foreign_loop(Some(reading), now, own_pid).is_some()
5480 || reading
5481 .pid
5482 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
5483}
5484
5485pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
5496 run_claimed_until(opts, queue, task, &Stop::new()).await;
5497}
5498
5499async fn run_claimed_until(
5500 opts: &Opts,
5501 queue: &Queue,
5502 task: &mut Task,
5503 stop: &Stop,
5504) -> Vec<QuotaLoss> {
5505 let status = Arc::new(Mutex::new(Status::new()));
5506 let mut quota = Box::pin(attempt(
5509 opts,
5510 queue,
5511 &status,
5512 stop,
5513 crate::graph::Pause::new(),
5514 task,
5515 ))
5516 .await;
5517 let mut resumes = 0;
5526 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
5527 let Some(timeout) = withheld_text_wait(task) else {
5528 break;
5529 };
5530 eprintln!(
5531 "waiting for the owner's answer on a withheld pull request text \
5532 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
5533 );
5534 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
5537 let ready = loop {
5538 match land_resume_state(task) {
5539 LandResume::StillWaiting
5540 if !stop.parking() && std::time::Instant::now() < limit =>
5541 {
5542 tokio::time::sleep(WITHHELD_POLL).await;
5543 }
5544 LandResume::Ready => break true,
5545 _ => break false,
5546 }
5547 };
5548 if !ready || stop.parking() {
5549 break;
5550 }
5551 resumes += 1;
5552 quota.extend(
5553 Box::pin(attempt(
5554 opts,
5555 queue,
5556 &status,
5557 stop,
5558 crate::graph::Pause::new(),
5559 task,
5560 ))
5561 .await,
5562 );
5563 }
5564 hold_if_runnable(queue, task);
5565 quota
5566}
5567
5568const MAX_WITHHELD_RESUMES: u32 = 3;
5571
5572const WITHHELD_POLL: Duration = Duration::from_secs(15);
5574
5575fn withheld_text_wait(task: &Task) -> Option<u64> {
5578 let id = task.execution_runs().last()?;
5579 let s = RunState::load(id).ok()?;
5580 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
5581 .then_some(s.config.graph.answer_timeout)
5582}
5583
5584fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
5594 after
5595 .iter()
5596 .filter(|q| !before.contains(q))
5597 .cloned()
5598 .collect()
5599}
5600
5601fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
5604 let quota: Vec<_> = quota
5605 .iter()
5606 .filter(|q| q.cause == crate::agent::QuotaCause::UsageLimit)
5607 .collect();
5608 if quota.is_empty() {
5609 return None;
5610 }
5611 let with_hint = quota.iter().find(|q| q.reset.is_some());
5612 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
5613 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
5614 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
5615 Some(
5616 now.checked_add(jiff::SignedDuration::from_secs(secs))
5617 .unwrap_or(Timestamp::MAX),
5618 )
5619}
5620
5621fn implementers_match(task: &Task, state: &RunState) -> bool {
5623 task.replacement.as_ref().map_or_else(
5624 || {
5625 task.implementers()
5626 .is_none_or(|n| n == state.config.graph.implementers)
5627 },
5628 |replacement| {
5629 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
5630 && state.config.graph.implementers == 1
5631 },
5632 )
5633}
5634
5635fn starter_with_implementers(
5637 task: &Task,
5638 previous: Option<&RunState>,
5639 proposed: Starter,
5640) -> Starter {
5641 if previous.is_some_and(|s| !implementers_match(task, s)) {
5642 task.review_of
5643 .as_ref()
5644 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
5645 } else {
5646 proposed
5647 }
5648}
5649
5650fn apply_task_implementers(config: &mut Config, task: &Task) {
5652 if let Some(n) = task.implementers() {
5653 config.graph.implementers = n;
5654 }
5655}
5656
5657#[cfg(test)]
5662fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5663 let explicit = match &task.overrides {
5664 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5665 _ => opts.config.as_deref(),
5666 };
5667 let (mut config, _) = Config::discover(repo, explicit)?;
5668 apply_options(&mut config, opts, task)?;
5669 Ok(config)
5670}
5671
5672fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5673 if task
5674 .overrides
5675 .as_ref()
5676 .is_none_or(|o| o.inherit_daemon_options)
5677 && let Some(mode) = opts.merge.as_deref()
5678 {
5679 config.merge.mode = merge_mode(mode)?;
5680 }
5681 if let Some(o) = &task.overrides {
5682 o.apply(config);
5683 }
5684 if let Some((mode, _)) = task.build_cache_override(config) {
5685 config.verify.build_cache = mode;
5686 }
5687 Ok(())
5688}
5689
5690fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5692 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5693 if let Some(mode) = &opts.merge {
5694 config.merge.mode = merge_mode(mode)?;
5695 }
5696 Ok(config)
5697}
5698
5699async fn maybe_prune_cache_between_runs(
5704 repo: &Path,
5705 opts: &Opts,
5706 home: &Path,
5707 stop: &Stop,
5708 last_checked: &mut Option<Timestamp>,
5709 now: Timestamp,
5710) {
5711 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5712 return;
5713 }
5714 *last_checked = Some(now);
5715 let cfg = match prepare(repo, opts) {
5716 Ok(cfg) => cfg,
5717 Err(e) => {
5718 tracing::warn!("cache check: no config: {e:#}");
5719 return;
5720 }
5721 };
5722 match clean::prune_cache_if_over_limit(&cfg, home) {
5723 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5724 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5725 pruned.files,
5726 pruned.freed
5727 ),
5728 Ok(_) => {}
5729 Err(e) => {
5730 tracing::warn!("housekeep: prune cache: {e:#}");
5731 notices::raise_in(
5732 home,
5733 Notice::warn(
5734 "housekeep:cache",
5735 format!(
5736 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5737 e.to_string()
5738 .lines()
5739 .next()
5740 .unwrap_or("unknown cleanup error")
5741 ),
5742 ),
5743 );
5744 }
5745 }
5746}
5747
5748fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5752 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5753}
5754
5755async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5774 let cfg = match prepare(repo, opts) {
5775 Ok(cfg) => cfg,
5776 Err(e) => {
5777 tracing::warn!("housekeep: no config: {e:#}");
5778 return;
5779 }
5780 };
5781 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5790 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5791 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5796 let mut extra = Vec::new();
5797 if out.unreadable > 0 {
5798 extra.push(format!("{} unreadable", out.unreadable));
5799 }
5800 if out.orphaned_worktrees > 0 {
5801 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5802 }
5803 let detail = if extra.is_empty() {
5804 String::new()
5805 } else {
5806 format!(" ({})", extra.join(", "))
5807 };
5808 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5809 }
5810 if out.external_merges_recorded > 0 {
5811 tracing::info!(
5812 "housekeep: recorded {} run(s) as merged externally",
5813 out.external_merges_recorded
5814 );
5815 }
5816 if out.stale_pr_states_repaired > 0 {
5817 tracing::info!(
5818 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5819 out.stale_pr_states_repaired
5820 );
5821 }
5822 if out.cache_files > 0 {
5823 tracing::info!(
5824 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5825 out.cache_files,
5826 out.cache_freed
5827 );
5828 }
5829 if out.questions_abandoned > 0 {
5830 tracing::info!(
5831 "housekeep: abandoned {} question(s) left open by a finished run",
5832 out.questions_abandoned
5833 );
5834 }
5835}
5836
5837async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5846 let questions = Questions::at(home.join("questions"));
5847 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5848 if report.is_empty() {
5849 return;
5850 }
5851 if !report.quarantined.is_empty() {
5852 tracing::info!(
5853 "triage: held {} blocked task(s) whose blocked-on task or \
5854 question no longer exists: {}",
5855 report.quarantined.len(),
5856 report.quarantined.join(", ")
5857 );
5858 }
5859 if !report.resumed.is_empty() {
5860 tracing::info!(
5861 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5862 report.resumed.len(),
5863 report.resumed.join(", ")
5864 );
5865 }
5866 if !report.asked.is_empty() {
5867 tracing::info!(
5868 "triage: asked about {} held task(s): {}",
5869 report.asked.len(),
5870 report.asked.join(", ")
5871 );
5872 }
5873 if !report.answered.is_empty() {
5874 tracing::info!(
5875 "triage: applied {} operator answer(s): {}",
5876 report.answered.len(),
5877 report.answered.join(", ")
5878 );
5879 }
5880}
5881
5882pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5889 disk_gate_at(repo, config, crate::run::home)
5890}
5891
5892pub(crate) fn disk_gate_at(
5893 repo: &Path,
5894 config: &Config,
5895 home: impl FnOnce() -> PathBuf,
5896) -> Option<String> {
5897 let measure = if crate::build_target::isolated(config) {
5898 crate::disk::managed_free_bytes
5899 } else {
5900 crate::disk::free_bytes
5901 };
5902 disk_gate_with(repo, config, home, measure)
5903}
5904
5905fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5909 repo: &Path,
5910 config: &Config,
5911 home: impl FnOnce() -> PathBuf,
5912 free_bytes: F,
5913) -> Option<String> {
5914 let min = config.disk.min_free_bytes;
5915 if min == 0 {
5916 return None;
5917 }
5918 let output = match config.verify.build_cache {
5919 BuildCache::Shared => config.cache_dir(),
5920 BuildCache::Isolated => Some(home()),
5921 };
5922 let default_worktree_root;
5923 let worktree_root = match config.graph.worktree_root.as_deref() {
5924 Some(root) => root,
5925 None => {
5926 default_worktree_root = crate::run::default_worktree_root();
5927 &default_worktree_root
5928 }
5929 };
5930 for path in [Some(repo), output.as_deref(), Some(worktree_root)]
5931 .into_iter()
5932 .flatten()
5933 {
5934 let probe = if path == repo {
5935 path
5936 } else {
5937 path.ancestors().find(|p| p.exists()).unwrap_or(path)
5938 };
5939 let reason = match free_bytes(probe) {
5940 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5941 Err(e) => Some(crate::disk::unmeasured_in(
5942 probe,
5943 &e,
5944 &config.graph.language,
5945 )),
5946 };
5947 if reason.is_some() {
5948 return reason;
5949 }
5950 }
5951 None
5952}
5953
5954const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5961
5962const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5966
5967fn quota_wait(
5976 reset_at: Option<Timestamp>,
5977 now: Timestamp,
5978 fallback: Duration,
5979 cap: Duration,
5980) -> Duration {
5981 match reset_at {
5982 Some(at) if at > now => {
5983 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5984 Duration::from_secs(secs).min(cap)
5985 }
5986 _ => fallback,
5987 }
5988}
5989
5990fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
6004 parse_reset_hint_zoned(text, now)
6005 .or_else(|| parse_reset_hint_dated(text))
6006 .or_else(|| parse_reset_hint_relative(text, recorded))
6007}
6008
6009fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
6013 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
6014 let mut rest = rest.trim();
6015 if rest.is_empty() {
6016 return None;
6017 }
6018 let mut total: i64 = 0;
6019 let mut matched = false;
6020 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
6021 if let Some((digits, tail)) = rest.split_once(unit)
6022 && !digits.is_empty()
6023 && digits.bytes().all(|b| b.is_ascii_digit())
6024 {
6025 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
6026 rest = tail;
6027 matched = true;
6028 }
6029 }
6030 if !rest.is_empty() || !matched {
6031 return None;
6032 }
6033 recorded
6034 .checked_add(jiff::SignedDuration::from_secs(total))
6035 .ok()
6036}
6037
6038fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
6042 let clock = clock.trim().to_lowercase();
6043 let (digits, pm) = clock
6044 .strip_suffix("am")
6045 .map(|d| (d, false))
6046 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
6047 let (h, m) = digits.trim().split_once(':')?;
6048 let mut hour: i8 = h.trim().parse().ok()?;
6049 let minute: i8 = m.trim().parse().ok()?;
6050 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
6051 return None;
6052 }
6053 if pm && hour != 12 {
6054 hour += 12;
6055 } else if !pm && hour == 12 {
6056 hour = 0;
6057 }
6058 Some((hour, minute))
6059}
6060
6061fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
6066 let open = text.find('(')?;
6067 let close = text.rfind(')')?;
6068 if close <= open {
6069 return None;
6070 }
6071 let zone = text[open + 1..close].trim();
6072 let (hour, minute) = parse_12h_clock(&text[..open])?;
6073 let tz = jiff::tz::TimeZone::get(zone).ok()?;
6074 let candidate = now
6075 .to_zoned(tz)
6076 .with()
6077 .hour(hour)
6078 .minute(minute)
6079 .second(0)
6080 .millisecond(0)
6081 .microsecond(0)
6082 .nanosecond(0)
6083 .build()
6084 .ok()?;
6085 let mut at = candidate.timestamp();
6086 if at <= now {
6087 at += jiff::SignedDuration::from_hours(24);
6088 }
6089 Some(at)
6090}
6091
6092fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
6101 let words: Vec<&str> = text.split_whitespace().collect();
6102 if words.len() < 5 {
6103 return None;
6104 }
6105 (0..=words.len() - 5)
6106 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
6107}
6108
6109fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
6115 if trailing.is_some_and(|next| next.starts_with('(')) {
6116 return None;
6117 }
6118 let month = month_number(window[0])?;
6119 let day_token = window[1].strip_suffix(',')?.to_lowercase();
6120 let day_digits = ["st", "nd", "rd", "th"]
6121 .iter()
6122 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
6123 let day: i8 = day_digits.parse().ok()?;
6124 let year_token = window[2];
6125 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
6126 return None;
6127 }
6128 let year: i16 = year_token.parse().ok()?;
6129 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
6133 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
6134 let date = jiff::civil::Date::new(year, month, day).ok()?;
6135 let candidate = date
6136 .at(hour, minute, 0, 0)
6137 .to_zoned(jiff::tz::TimeZone::UTC)
6138 .ok()?;
6139 Some(candidate.timestamp())
6140}
6141
6142fn month_number(name: &str) -> Option<i8> {
6145 const NAMES: [&str; 12] = [
6146 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
6147 ];
6148 let lower = name.to_lowercase();
6149 NAMES
6150 .iter()
6151 .position(|n| *n == lower.as_str())
6152 .map(|i| i as i8 + 1)
6153}
6154
6155#[cfg(test)]
6157fn exhausted_review_budget(state: &RunState) -> bool {
6158 !crate::handover::can_resume(state)
6161 && state.status == RunStatus::Blocked
6162 && !state.released()
6163 && state.retired_to.is_none()
6164}
6165
6166fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
6203 unfinished_run_with(runs, short, RunState::load)
6204}
6205
6206pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
6209where
6210 F: FnMut(&str) -> Result<RunState>,
6211{
6212 let mut id = runs.last()?.clone();
6213 let mut visited = std::collections::BTreeSet::new();
6214 let mut repo = None;
6215 loop {
6216 if !runs.contains(&id) || !visited.insert(id.clone()) {
6217 tracing::warn!("invalid or cyclic recovery authority for task {short}");
6218 return None;
6219 }
6220 let state = match load(&id) {
6221 Ok(state) if state.id == id => state,
6222 Ok(_) => return None,
6223 Err(e) => {
6224 tracing::warn!("could not read run {id} for task {short}: {e:#}");
6225 return None;
6226 }
6227 };
6228 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
6229 return None;
6230 }
6231 repo = Some(state.repo.clone());
6232 if let Some(authority) = state.retired_to {
6233 id = authority;
6234 continue;
6235 }
6236 return (crate::handover::can_resume(&state)
6237 && state.liveness(false) != crate::run::Liveness::Live)
6238 .then_some(id);
6239 }
6240}
6241
6242fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
6250where
6251 F: FnOnce(&str) -> Result<RunState>,
6252{
6253 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
6255 || task.fresh_start
6256 || task.review_branch.is_some()
6257 {
6258 return false;
6259 }
6260 let Some(id) = task.execution_runs().last() else {
6261 return false;
6262 };
6263 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
6264}
6265
6266#[derive(Debug, Clone, PartialEq, Eq)]
6269enum Starter {
6270 Review(String),
6273 Resume(String),
6275 Start,
6277}
6278
6279fn retained_attempt_hold(
6284 task: &Task,
6285 starter: &Starter,
6286 limit: usize,
6287 mut load: impl FnMut(&str) -> Result<RunState>,
6288) -> Option<String> {
6289 if limit == 0 || matches!(starter, Starter::Resume(_)) {
6290 return None;
6291 }
6292 let mut retained = 0;
6293 for id in task
6294 .runs
6295 .iter()
6296 .chain(task.startup_failures.iter().map(|failure| &failure.run))
6297 .collect::<std::collections::BTreeSet<_>>()
6298 {
6299 let state = match load(id) {
6300 Ok(state) if state.id == *id => state,
6301 _ => {
6302 return Some(format!(
6303 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
6304 ));
6305 }
6306 };
6307 if state.retired_to.is_some() || state.released() {
6308 continue;
6309 }
6310 retained += 1;
6313 }
6314 (retained >= limit).then(|| format!(
6315 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
6316 ))
6317}
6318
6319fn selected_starter(
6321 selected: &crate::pr_recovery::Selection,
6322 previous: Option<&RunState>,
6323 unfinished: Option<&str>,
6324) -> Starter {
6325 if let Some(id) = unfinished
6326 && previous.is_some_and(|r| {
6327 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
6328 })
6329 {
6330 Starter::Resume(id.to_owned())
6331 } else {
6332 Starter::Review(selected.branch.clone())
6333 }
6334}
6335
6336fn take_divergence_answer(
6340 branch: &str,
6341 remote: &str,
6342 task: &mut Task,
6343) -> Option<crate::reconcile::Choice> {
6344 let summary = crate::reconcile::summary_for(branch, remote);
6345 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
6346 (a.question == summary)
6347 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
6348 .flatten()
6349 .map(|c| (i, c))
6350 })?;
6351 task.answers.remove(idx);
6352 Some(choice)
6353}
6354
6355fn choose_starter(
6367 review_branch: Option<&str>,
6368 branch_exists: bool,
6369 unfinished: Option<&str>,
6370) -> Starter {
6371 match review_branch {
6372 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
6373 Some(_) => Starter::Start,
6374 None => match unfinished {
6375 Some(id) => Starter::Resume(id.to_owned()),
6376 None => Starter::Start,
6377 },
6378 }
6379}
6380
6381fn review_retry_starter(
6385 task: &Task,
6386 previous: Option<&RunState>,
6387 branch: Option<&str>,
6388 exists: bool,
6389 unfinished: Option<&str>,
6390) -> Starter {
6391 if let Some(run) = previous
6392 && unfinished == Some(run.id.as_str())
6393 && run.reviewed_commits.is_some()
6394 && branch.is_some_and(|branch| {
6395 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
6396 })
6397 && may_resume_run(run, task)
6398 {
6399 return Starter::Resume(run.id.clone());
6400 }
6401 match (branch, unfinished, &task.review_of) {
6402 (None, None, Some(branch)) => Starter::Review(branch.clone()),
6403 _ => choose_starter(branch, exists, unfinished),
6404 }
6405}
6406
6407fn review_branch_for<'a>(task: &'a Task, previous: Option<&RunState>) -> Option<&'a str> {
6409 task.review_branch
6410 .as_deref()
6411 .filter(|&branch| match &task.review_instruction {
6412 Some(authored) => *authored == task.instruction,
6413 None => previous.map_or_else(
6416 || task.review_of.as_deref() == Some(branch),
6417 |run| compatible_run_instruction(run, task),
6418 ),
6419 })
6420}
6421
6422fn may_resume_run(run: &RunState, task: &Task) -> bool {
6425 if !task.assignment_accepts(run) {
6426 return false;
6427 }
6428 task.resume_override
6429 .as_ref()
6430 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
6431 || compatible_run_instruction(run, task)
6432}
6433
6434pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
6436 if let Some(authored) = &run.task_instruction {
6437 return compatible_instruction(authored, task);
6438 }
6439 if compatible_instruction(&run.instruction, task) {
6440 return true;
6441 }
6442 if run.reviewed_commits.is_some() {
6445 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
6446 return strip_answers_block(authored, task).trim() == task.instruction.trim();
6447 }
6448 if let Some(branch) = &task.review_of {
6449 let canonical = format!(
6450 "Review the branch `{branch}` (review-only: no implementation, no judging)."
6451 );
6452 let generated = format!(
6453 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
6454 );
6455 return task.instruction == canonical && run.instruction.starts_with(&generated);
6456 }
6457 }
6458 false
6459}
6460
6461pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
6463 strip_answers_block(old, task) == task.instruction
6464}
6465
6466fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
6469 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
6470 return fallback.to_path_buf();
6471 }
6472 task.repo.clone()
6473}
6474
6475const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
6479
6480fn answers_block(task: &Task, count: usize) -> String {
6482 let mut s = ANSWERS_HEADER.to_owned();
6483 for a in &task.answers[..count] {
6484 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
6485 }
6486 s
6487}
6488
6489fn resumption_block(r: &crate::queue::Resumption) -> String {
6492 format!(
6493 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
6494 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
6495 r.generation, r.at, r.instruction, r.answers_through,
6496 )
6497}
6498
6499fn append_answers(base: &str, task: &Task) -> String {
6501 let mut s = base.to_owned();
6502 if !task.answers.is_empty() {
6503 s.push_str(&answers_block(task, task.answers.len()));
6504 }
6505 if let Some(r) = &task.resumption {
6506 s.push_str(&resumption_block(r));
6507 }
6508 s
6509}
6510
6511fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
6515 let mut instruction = instruction;
6516 for approval in task
6517 .resumption
6518 .iter()
6519 .chain(task.resumption_history.iter().rev())
6520 {
6521 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
6522 instruction = base;
6523 break;
6524 }
6525 }
6526 for count in (1..=task.answers.len()).rev() {
6527 let block = answers_block(task, count);
6528 if let Some(base) = instruction.strip_suffix(&block) {
6529 return base;
6530 }
6531 }
6532 instruction
6533}
6534
6535fn instruction_for(task: &Task) -> String {
6543 append_answers(&task.instruction, task)
6544}
6545
6546fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
6558 append_answers(strip_answers_block(old_instruction, task), task)
6559}
6560
6561fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
6563 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
6564 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
6565 if let Some(cause) = task.hold_notice_cause.as_mut() {
6566 cause.target = failure.map(|f| f.target.clone());
6567 }
6568}
6569
6570#[derive(Debug, thiserror::Error)]
6571#[error("attachment `{name}` is recorded on the task but {path} is missing")]
6572struct MissingAttachment {
6573 name: String,
6574 path: PathBuf,
6575}
6576
6577fn task_attachments(
6579 queue: &Queue,
6580 task: &Task,
6581) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
6582 let paths = queue.attachment_paths(task);
6583 for (name, path) in task.attachments.iter().zip(&paths) {
6584 if !path.is_file() {
6585 return Err(MissingAttachment {
6586 name: name.clone(),
6587 path: path.clone(),
6588 });
6589 }
6590 }
6591 Ok(paths)
6592}
6593
6594fn prepare_instruction(
6605 starter: &Starter,
6606 old_instruction: Option<&str>,
6607 task: &Task,
6608) -> Option<String> {
6609 match starter {
6610 Starter::Start => Some(instruction_for(task)),
6611 Starter::Resume(_) => Some(resumed_instruction(
6612 old_instruction.expect("a resumed run always has a prior instruction"),
6613 task,
6614 )),
6615 Starter::Review(_) => None,
6616 }
6617}
6618
6619fn record(queue: &Queue, task: &mut Task) {
6623 if let Err(e) = queue.put(task) {
6624 tracing::error!("could not record task {}: {e:#}", task.short());
6625 let notice = Notice::error(
6626 "loop:record",
6627 format!(
6628 "Task {} state could not be saved: {}. Check the disk.",
6629 task.id,
6630 e.to_string().lines().next().unwrap_or("unknown save error")
6631 ),
6632 )
6633 .link(Link::Task {
6634 id: task.id.clone(),
6635 });
6636 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
6638 Some(home) => notices::raise_in(home, notice),
6639 None => notices::raise(notice),
6640 }
6641 }
6642}
6643
6644fn runnable(queue: &Queue) -> Vec<Task> {
6650 let mut tasks: Vec<Task> = queue
6651 .list()
6652 .into_iter()
6653 .filter(|t| t.status.runnable())
6654 .collect();
6655 tasks.sort_unstable_by(crate::queue::dispatch_order);
6656 tasks
6657}
6658
6659fn describe(state: &RunState) -> String {
6673 let p = phrases(&state.config.graph.language);
6674 let mut detail = if state.status == RunStatus::Stalled {
6675 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
6676 seats.sort_unstable();
6677 seats.dedup();
6678 if seats.is_empty() {
6679 p.quorum_lost.to_owned()
6680 } else {
6681 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
6682 }
6683 } else {
6684 format!("{}{}", p.run_ended, state.status.display_label())
6685 };
6686 if let Some(cause) = state.verification_blocking_cause() {
6687 detail.push_str(&format!(" ({cause})"));
6688 } else if let Some(last) = state.events.last() {
6689 let unanswered = state
6693 .reviews
6694 .last()
6695 .filter(|r| {
6696 state.status == RunStatus::Blocked
6697 && last.node == "review"
6698 && r.incomplete()
6699 && r.blocking == 0
6700 && r.round == state.config.graph.review_rounds
6701 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6702 })
6703 .map(|r| (r.expected - r.answered, r.round));
6704 match unanswered {
6705 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6706 detail.push_str(&format!(
6707 " ({}: {})",
6708 last.node,
6709 (p.reviewers_never_answered)(missing, rounds)
6710 ));
6711 }
6712 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6713 }
6714 }
6715 detail.push_str(&format!(" [run {}]", state.id));
6716 detail
6717}
6718
6719const DIAGNOSTIC_MAX: usize = 4_000;
6725
6726const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6731
6732fn diagnostic(state: &RunState) -> Option<String> {
6746 let mut parts: Vec<String> = Vec::new();
6747
6748 for o in state.gate.iter().filter(|o| !o.ok()) {
6750 parts.push(format!(
6751 "gate `{}` failed ({}):\n{}",
6752 o.command,
6753 notices::exit_status(o.code),
6754 crate::run::summarize_verify(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6755 ));
6756 }
6757
6758 if let Some(last) = state
6761 .events
6762 .iter()
6763 .rev()
6764 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6765 {
6766 parts.push(last.message.clone());
6767 }
6768
6769 if state.viable().is_empty() {
6776 for c in &state.candidates {
6777 if let Some(evidence) = &c.verified_noop {
6778 parts.push(format!(
6779 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6780 c.label
6781 ));
6782 } else if !c.summary.trim().is_empty() {
6783 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6784 } else if let Some(why) = &c.failed {
6785 parts.push(format!("candidate {}: {why}", c.label));
6786 }
6787 }
6788 }
6789
6790 if parts.is_empty() {
6791 return None;
6792 }
6793 Some(crate::run::tail(
6798 &parts.join("\n\n"),
6799 DIAGNOSTIC_MAX.saturating_sub(100),
6800 ))
6801}
6802
6803fn label(status: RunStatus) -> &'static str {
6811 status.as_str()
6812}
6813
6814pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6816 match mode {
6817 "none" => Ok(MergeMode::None),
6818 "local" => Ok(MergeMode::Local),
6819 "pr" => Ok(MergeMode::Pr),
6820 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6821 }
6822}
6823
6824fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6829 mutex
6830 .lock()
6831 .unwrap_or_else(std::sync::PoisonError::into_inner)
6832}
6833
6834#[cfg(test)]
6835mod tests {
6836 use super::*;
6837 use crate::queue::{Source, TaskStatus};
6838 use crate::run::{Candidate, CommandOutcome};
6839 use pretty_assertions::assert_eq;
6840
6841 fn task() -> Task {
6842 Task::new(
6843 "add retries".to_owned(),
6844 "add retries".to_owned(),
6845 PathBuf::from("/repo"),
6846 Source::Human,
6847 )
6848 }
6849
6850 #[test]
6851 fn dispatch_wait_aggregates_unique_skips_in_stable_count_order() {
6852 let skips = DispatchSkips::from([
6853 ("a".into(), SkipReason::CoolingDown),
6854 ("b".into(), SkipReason::WaitingOnLanding),
6855 ("c".into(), SkipReason::WaitingOnLanding),
6856 ("d".into(), SkipReason::OrdinaryCapacity),
6857 ]);
6858 let wait = aggregate_dispatch_wait(4, &skips).unwrap();
6859 assert_eq!(wait.runnable, 4);
6860 assert_eq!(
6861 wait.reasons
6862 .iter()
6863 .map(|r| (r.reason.as_str(), r.count))
6864 .collect::<Vec<_>>(),
6865 [
6866 ("waiting_on_landing", 2),
6867 ("cooling_down", 1),
6868 ("ordinary_capacity", 1)
6869 ]
6870 );
6871 assert_eq!(aggregate_dispatch_wait(0, &DispatchSkips::new()), None);
6872 assert!(
6873 aggregate_dispatch_wait(2, &DispatchSkips::new())
6874 .unwrap()
6875 .reasons
6876 .is_empty()
6877 );
6878 }
6879
6880 #[test]
6881 fn dispatch_wait_tracks_interrupt_withholding_and_set_changes() {
6882 let before = vec!["a".into(), "b".into(), "c".into()];
6883 let allowed = vec![task_with_id("b")];
6884 let skips = interrupt_skips(&before, &allowed);
6885 assert_eq!(
6886 skips,
6887 DispatchSkips::from([
6888 ("a".into(), SkipReason::InterruptGate),
6889 ("c".into(), SkipReason::InterruptGate),
6890 ])
6891 );
6892 let mut previous = DispatchSkips::new();
6893 assert!(!changed_dispatch_skips(&mut previous, DispatchSkips::new()));
6894 assert!(changed_dispatch_skips(&mut previous, skips.clone()));
6895 assert!(!changed_dispatch_skips(&mut previous, skips));
6896 assert!(changed_dispatch_skips(
6897 &mut previous,
6898 DispatchSkips::from([("a".into(), SkipReason::CoolingDown),])
6899 ));
6900 assert!(changed_dispatch_skips(&mut previous, DispatchSkips::new()));
6901 }
6902
6903 #[test]
6904 fn dispatch_wait_status_reads_legacy_and_future_reason_keys() {
6905 let mut value = serde_json::to_value(Status::new()).unwrap();
6906 value.as_object_mut().unwrap().remove("dispatch_wait");
6907 assert!(
6908 serde_json::from_value::<Status>(value.clone())
6909 .unwrap()
6910 .dispatch_wait
6911 .is_none()
6912 );
6913 assert!(
6914 serde_json::from_value::<Reading>(value.clone())
6915 .unwrap()
6916 .dispatch_wait
6917 .is_none()
6918 );
6919 value["dispatch_wait"] = serde_json::json!({
6920 "runnable": 3, "reasons": [{"reason": "future_reason", "count": 3}]
6921 });
6922 let reading = serde_json::from_value::<Reading>(value).unwrap();
6923 assert_eq!(
6924 reading.dispatch_wait.unwrap().reasons[0].reason,
6925 "future_reason"
6926 );
6927 }
6928
6929 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6930 let job = crate::worker::Job {
6931 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6932 sequence: 1,
6933 kind: "task".into(),
6934 key: format!("task:{}", request.task),
6935 payload: serde_json::to_value(request).unwrap(),
6936 generation: "a".repeat(64),
6937 version: None,
6938 status: crate::worker::Status::Starting,
6939 result: None,
6940 pid: None,
6941 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6942 finished_at: None,
6943 };
6944 let dir = home.join("workers/jobs").join(&job.id);
6945 std::fs::create_dir_all(&dir).unwrap();
6946 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6947 job
6948 }
6949
6950 #[test]
6951 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6952 let home = tempfile::tempdir().unwrap();
6953 let request = TaskWorker {
6954 opts: Opts::default(),
6955 queue: home.path().join("queue"),
6956 task: "startup".into(),
6957 lane: PermitKind::Ordinary,
6958 standalone: false,
6959 };
6960 let job = starting_task_reservation(home.path(), &request);
6961 let error = task_worker_spawn_error(
6962 home.path(),
6963 &request,
6964 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6965 )
6966 .expect_err("our own reservation must not turn a failed spawn into a resync");
6967 let active = crate::worker::active(home.path()).unwrap();
6968 assert_eq!(active.len(), 1);
6969 assert_eq!(active[0].id, job.id);
6970 assert_eq!(active[0].status, crate::worker::Status::Starting);
6971 assert!(
6972 crate::worker::control(home.path(), &job.key)
6973 .unwrap()
6974 .is_none()
6975 );
6976 let notices = notices::Notices::at(home.path().join("notifications")).list();
6977 assert_eq!(notices.len(), 1);
6978 assert_eq!(notices[0].key, "loop:worker-startup");
6979 assert!(!notices[0].message.contains("private-error-detail"));
6980 assert!(notices[0].message.contains(&request.task));
6981 assert!(
6982 notices[0]
6983 .message
6984 .contains("could not be started or acknowledged")
6985 );
6986 assert!(notices[0].message.contains("worker.log"));
6987 assert_eq!(
6988 notices[0].link,
6989 Some(Link::Task {
6990 id: request.task.clone()
6991 })
6992 );
6993 assert!(format!("{error:#}").contains("private-error-detail"));
6994 }
6995
6996 #[test]
6997 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6998 let home = tempfile::tempdir().unwrap();
6999 let mut request = TaskWorker {
7000 opts: Opts::default(),
7001 queue: home.path().join("queue"),
7002 task: "handoff".into(),
7003 lane: PermitKind::Ordinary,
7004 standalone: true,
7005 };
7006 let job = starting_task_reservation(home.path(), &request);
7007 request.standalone = false;
7008 task_worker_spawn_error(
7009 home.path(),
7010 &request,
7011 anyhow::anyhow!("operation key is already reserved for a different request"),
7012 )
7013 .expect("a different standalone request is the handoff race, not our startup failure");
7014 let active = crate::worker::active(home.path()).unwrap();
7015 assert_eq!(active.len(), 1);
7016 assert_eq!(active[0].id, job.id);
7017 assert_eq!(active[0].payload, job.payload);
7018 assert!(
7019 notices::Notices::at(home.path().join("notifications"))
7020 .list()
7021 .is_empty()
7022 );
7023 }
7024
7025 #[test]
7026 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
7027 let home = tempfile::tempdir().unwrap();
7028 let queue = Queue::at(home.path().join("queue"));
7029 let mut task = task();
7030 task.runs = vec!["previous".into()];
7031 queue.put(&mut task).unwrap();
7032 let request = TaskWorker {
7033 opts: Opts::default(),
7034 queue: home.path().join("queue"),
7035 task: task.id.clone(),
7036 lane: PermitKind::Ordinary,
7037 standalone: false,
7038 };
7039 let job = starting_task_reservation(home.path(), &request);
7040 let claims = current_work(home.path(), Timestamp::now());
7041 assert_eq!(claims.len(), 1);
7042 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
7043 assert_eq!(claims[0].run, "previous");
7044 assert!(other_claims_run(&claims, "previous", None));
7045 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
7048 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
7049 }
7050
7051 #[test]
7052 fn admission_excludes_only_the_callers_exact_reservation() {
7053 let own = Current {
7054 worker: Some("own".into()),
7055 task: "same-task".into(),
7056 run: "previous".into(),
7057 };
7058 assert!(!other_claims_run(
7059 std::slice::from_ref(&own),
7060 "previous",
7061 Some("own")
7062 ));
7063 assert!(other_claims_run(
7064 std::slice::from_ref(&own),
7065 "previous",
7066 None
7067 ));
7068 for worker in [None, Some("other".into())] {
7069 let other = Current {
7070 worker,
7071 ..own.clone()
7072 };
7073 assert!(other_claims_run(
7074 &[own.clone(), other],
7075 "previous",
7076 Some("own")
7077 ));
7078 }
7079 let legacy: Current =
7080 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
7081 assert!(other_claims_run(&[legacy], "previous", Some("own")));
7082 }
7083
7084 #[test]
7085 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
7086 let mut task = task();
7087 let mut run = run_state(RunStatus::Reviewing);
7088 run.task_instruction = Some(task.instruction.clone());
7089 run.reviewed_commits = Some(vec!["committed-fix".into()]);
7090 let branch = "magi/test/B";
7091 run.candidates.push(crate::run::Candidate {
7092 branch: branch.into(),
7093 ..candidate('B', "preserved work", false, None)
7094 });
7095 task.runs = vec![run.id.clone()];
7096 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
7097 assert_eq!(
7098 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
7099 Starter::Resume(run.id.clone())
7100 );
7101 assert!(matches!(
7102 review_retry_starter(
7103 &task,
7104 Some(&run),
7105 Some("different"),
7106 true,
7107 unfinished.as_deref()
7108 ),
7109 Starter::Review(_)
7110 ));
7111 assert_eq!(
7112 review_retry_starter(
7113 &task,
7114 Some(&run),
7115 Some(branch),
7116 false,
7117 unfinished.as_deref()
7118 ),
7119 Starter::Start
7120 );
7121 task.instruction.push_str(" changed criteria");
7122 assert!(matches!(
7123 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
7124 Starter::Review(_)
7125 ));
7126 assert!(matches!(
7127 review_retry_starter(&task, Some(&run), Some(branch), true, None),
7128 Starter::Review(_)
7129 ));
7130 }
7131
7132 #[test]
7133 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
7134 let dir = tempfile::tempdir().unwrap();
7135 let queue = Queue::at(dir.path().join("queue"));
7136 let mut task = task();
7137 task.status = TaskStatus::Running;
7138 task.attempts = 1;
7139 queue.put(&mut task).unwrap();
7140 let live = std::collections::BTreeSet::from([task.id.clone()]);
7141 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
7142 drop(queue.claim(&task.id).unwrap());
7143 let saved = queue.get(&task.id).unwrap();
7144 assert_eq!(saved.status, TaskStatus::Running);
7145 assert_eq!(saved.attempts, 1);
7146 assert!(saved.last_error.is_none());
7147 }
7148
7149 #[test]
7150 fn standalone_workers_do_not_consume_scheduler_lanes() {
7151 let request = |task: &str, lane, standalone| TaskWorker {
7152 opts: Opts::default(),
7153 queue: PathBuf::from("/queue"),
7154 task: task.to_owned(),
7155 lane,
7156 standalone,
7157 };
7158 let mut jobs = vec![
7159 request("ordinary", PermitKind::Ordinary, false),
7160 request("standalone-urgent", PermitKind::Urgent, true),
7161 request("standalone-ordinary", PermitKind::Ordinary, true),
7162 request("decision", PermitKind::None, false),
7163 ];
7164 assert_eq!(available_slots(&jobs, 2), (1, 1));
7165 jobs.push(request("urgent", PermitKind::Urgent, false));
7166 assert_eq!(available_slots(&jobs, 2), (1, 0));
7167 assert_eq!(available_slots(&jobs, 0), (0, 0));
7168 }
7169
7170 #[test]
7171 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
7172 let now = Timestamp::now();
7173 let mut reading = Reading {
7174 pid: Some(std::process::id()),
7175 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
7176 ..Reading::default()
7177 };
7178 assert!(!reading.running(now));
7179 assert!(legacy_scheduler_running(&reading, now, 0));
7180 reading.worker_protocol = Some(1);
7181 assert!(!legacy_scheduler_running(&reading, now, 0));
7182 }
7183
7184 #[test]
7185 fn selected_public_head_overrides_historical_resume_and_pin() {
7186 let mut t = task();
7187 let mut old = run_state(RunStatus::Blocked);
7188 let selection = crate::pr_recovery::Selection {
7189 url: "url".into(),
7190 branch: format!("magi/task-{}", t.id),
7191 head: "a".repeat(40),
7192 source_head: None,
7193 source_run: None,
7194 source_proof: None,
7195 disposition: Some(crate::pr_recovery::Disposition::Review),
7196 instruction: t.instruction.clone(),
7197 approved_at: Timestamp::now(),
7198 };
7199 t.resume_override = Some(crate::queue::OperatorResume {
7200 pinned_run: Some(old.id.clone()),
7201 question_id: "choice".into(),
7202 at: Timestamp::now(),
7203 conductor_rehold: None,
7204 forced: false,
7205 });
7206 assert_eq!(
7207 selected_starter(&selection, Some(&old), Some(&old.id)),
7208 Starter::Review(selection.branch.clone())
7209 );
7210 old.pr_recovery = Some(selection.clone());
7211 assert_eq!(
7212 selected_starter(&selection, Some(&old), Some(&old.id)),
7213 Starter::Review(selection.branch.clone())
7214 );
7215 old.reviewed_commits = Some(vec!["implementation".into()]);
7216 assert_eq!(
7217 selected_starter(&selection, Some(&old), Some(&old.id)),
7218 Starter::Resume(old.id.clone())
7219 );
7220 let mut changed = selection.clone();
7221 changed.head = "b".repeat(40);
7222 assert_eq!(
7223 selected_starter(&changed, Some(&old), Some(&old.id)),
7224 Starter::Review(changed.branch)
7225 );
7226 }
7227
7228 #[test]
7229 fn candidate_transplant_failure_is_one_actionable_hold() {
7230 let dir = tempfile::tempdir().unwrap();
7231 let q = Queue::at(dir.path().join("queue"));
7232 let mut t = task();
7233 let mut state = run_state(RunStatus::Blocked);
7234 state.event("prep", "cannot apply referenced commit: conflict");
7235 state.prep_failure = Some("conflict".into());
7236 t.start(state.id.clone());
7237 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
7238 assert_eq!(t.attempts, 1);
7239 assert_eq!(t.status, TaskStatus::Held);
7240 assert!(t.operator_held());
7241 assert!(!t.status.runnable());
7242 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
7243 let first = t.chat_notice_key().unwrap();
7244 t.chat_notices.push(first.clone());
7245 t.release_operator("test", None);
7246 state.id = "another-unprepared-run".into();
7247 t.start(state.id.clone());
7248 finish_attempt(
7249 3,
7250 &q,
7251 &mut t,
7252 &state,
7253 &[],
7254 Err(anyhow::anyhow!("still conflicts")),
7255 );
7256 assert_eq!(
7257 t.chat_notice_key().as_ref(),
7258 Some(&first),
7259 "no work happened"
7260 );
7261 t.release_operator("test", None);
7262 state.prep_failure = None;
7263 state.status = RunStatus::Failed;
7264 state.begin_pass();
7265 t.start(state.id.clone());
7266 finish_attempt(
7267 1,
7268 &q,
7269 &mut t,
7270 &state,
7271 &[],
7272 Err(anyhow::anyhow!("new gate failure")),
7273 );
7274 let failure = t.chat_notice_key().unwrap();
7275 assert_ne!(failure, first, "approved recovery reached a real failure");
7276 t.release_operator("test", None);
7277 state.begin_pass();
7278 t.start(state.id.clone());
7279 finish_attempt(
7280 1,
7281 &q,
7282 &mut t,
7283 &state,
7284 &[],
7285 Err(anyhow::anyhow!("new failure on resume")),
7286 );
7287 assert_ne!(
7288 t.chat_notice_key().as_ref(),
7289 Some(&failure),
7290 "a new settled pass"
7291 );
7292 }
7293
7294 #[test]
7295 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
7296 for source in ["cli", "web"] {
7297 let mut t = task();
7298 let mut run = run_state(RunStatus::Blocked);
7299 run.instruction = "Run A original criteria".into();
7300 run.task_instruction = Some(run.instruction.clone());
7301 t.start(run.id.clone());
7302 t.release_operator("choice", Some("resume-question"));
7303 t.resume_override = Some(crate::queue::OperatorResume {
7304 question_id: "resume-question".into(),
7305 at: Timestamp::now(),
7306 conductor_rehold: None,
7307 forced: true,
7308 pinned_run: Some(run.id.clone()),
7309 });
7310 t.hold_machine(Some("not enough free space to start a run".into()));
7311 let mut unchanged = t.clone();
7312 unchanged
7313 .edit("title changed".into(), unchanged.instruction.clone())
7314 .unwrap();
7315 unchanged.release_operator(source, None);
7316 assert_eq!(
7317 unchanged
7318 .resume_override
7319 .as_ref()
7320 .unwrap()
7321 .pinned_run
7322 .as_deref(),
7323 Some(run.id.as_str())
7324 );
7325 assert!(!compatible_run_instruction(&run, &unchanged));
7326 assert!(may_resume_run(&run, &unchanged));
7327
7328 t.edit(
7329 "approved recovery".into(),
7330 "Reuse measured PR head B; fix Clippy; review".into(),
7331 )
7332 .unwrap();
7333 assert!(t.resume_override.is_none());
7334 t.release_operator(source, None);
7335 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
7336 assert!(reloaded.resume_override.is_none());
7337 assert!(!may_resume_run(&run, &reloaded));
7338 let unfinished =
7339 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
7340 .filter(|_| may_resume_run(&run, &reloaded));
7341 assert_eq!(
7342 choose_starter(None, false, unfinished.as_deref()),
7343 Starter::Start
7344 );
7345 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
7346 assert_eq!(
7347 reloaded.resumption_history[0].question_id.as_deref(),
7348 Some("resume-question")
7349 );
7350
7351 unchanged.instruction = "Reuse measured PR head B".into();
7354 unchanged.hold_machine(Some("disk hold".into()));
7355 unchanged.release_operator(source, None);
7356 assert!(unchanged.resume_override.is_none());
7357 assert!(!may_resume_run(&run, &unchanged));
7358 }
7359 }
7360
7361 #[test]
7362 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
7363 let mut t = task();
7364 let mut run = run_state(RunStatus::Blocked);
7365 t.review_of = Some("approved-head".into());
7366 t.instruction =
7367 "Review the branch `approved-head` (review-only: no implementation, no judging)."
7368 .into();
7369 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
7370 run.reviewed_commits = Some(vec!["Commit log".into()]);
7371 t.start(run.id.clone());
7372 t.hold_machine(Some("interrupted review".into()));
7373 t.release_operator("cli", None);
7374 assert!(
7375 compatible_run_instruction(&run, &t),
7376 "legacy explicit review remains resumable"
7377 );
7378 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
7379 .filter(|_| compatible_run_instruction(&run, &t));
7380 assert_eq!(
7381 choose_starter(None, false, unfinished.as_deref()),
7382 Starter::Resume(run.id.clone())
7383 );
7384 run.task_instruction = Some(t.instruction.clone());
7385 assert!(compatible_run_instruction(&run, &t));
7386 let reloaded: RunState =
7387 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
7388 assert!(compatible_run_instruction(&reloaded, &t));
7389 t.edit(
7390 "new approved head".into(),
7391 "Reuse measured PR head B".into(),
7392 )
7393 .unwrap();
7394 assert!(!compatible_run_instruction(&run, &t));
7395
7396 t.review_of = None;
7398 t.instruction = "Fix Clippy on approved head".into();
7399 run.task_instruction = None;
7400 run.instruction = format!(
7401 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
7402 t.instruction
7403 );
7404 assert!(compatible_run_instruction(&run, &t));
7405 t.instruction.push('\n');
7406 assert!(
7407 compatible_run_instruction(&run, &t),
7408 "legacy generated review trimmed the original criteria"
7409 );
7410 t.hold_for_handover(Some("approved-head".into()), "refused".into());
7411 assert_eq!(review_branch_for(&t, Some(&run)), Some("approved-head"));
7412 t.edit("new plan".into(), "Reuse measured PR head B".into())
7413 .unwrap();
7414 assert!(review_branch_for(&t, Some(&run)).is_none());
7415 assert!(!compatible_run_instruction(&run, &t));
7416 t.review_branch = Some("approved-head".into());
7417 t.review_instruction = None;
7418 assert!(
7419 review_branch_for(&t, Some(&run)).is_none(),
7420 "legacy retained candidate cannot replace a revised approved head"
7421 );
7422 }
7423
7424 #[test]
7425 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
7426 let home = crate::run::test_home();
7427 crate::run::pin_test_home();
7428 let dir = tempfile::tempdir().unwrap();
7429 let queue = Queue::at(dir.path().join("queue"));
7430 for resume in [true, false] {
7431 let state = run_state(RunStatus::Blocked);
7432 let mut t = task();
7433 t.start(state.id.clone());
7434 let action = if resume {
7435 ask::ChoiceAction::Resume {
7436 run: state.id.clone(),
7437 }
7438 } else {
7439 ask::ChoiceAction::Requeue
7440 };
7441 let mut q = action_question(&state.id, action);
7442 q.status = ask::QuestionStatus::Open;
7443 q.answer = None;
7444 q.answered_at = None;
7445 t.fail("run stopped with a question", 3);
7446 hold_if_runnable(&queue, &mut t);
7447 assert!(t.operator_decided_at.is_none());
7448 assert!(t.operator_held());
7449 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
7450 let expected = if resume {
7451 ActionDecision::Resume(state.id.clone())
7452 } else {
7453 ActionDecision::Requeue
7454 };
7455 assert_eq!(
7456 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
7457 expected
7458 );
7459 t.hold_manual(Some("later explicit hold".into()));
7460 assert_eq!(
7461 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
7462 ActionDecision::Stale
7463 );
7464 }
7465 }
7466
7467 #[test]
7468 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
7469 let mut t = task();
7470 t.answers.push(crate::queue::AnsweredQuestion {
7471 question: "Inspect the remote?".into(),
7472 answer: "Inspect then wait".into(),
7473 });
7474 let old = instruction_for(&t);
7475 assert!(compatible_instruction(&old, &t));
7476 t.instruction = "Reuse approved PR head and review".into();
7477 t.release_operator("cli", None);
7478 assert!(!compatible_instruction(&old, &t));
7479 let approved = instruction_for(&t);
7480 assert!(approved.contains("waiting premises are superseded"));
7481 assert!(compatible_instruction(&approved, &t));
7482 assert_eq!(resumed_instruction(&approved, &t), approved);
7483 t.release_operator("web", None);
7484 assert!(compatible_instruction(&approved, &t));
7485 assert_eq!(t.resumption_history.len(), 1);
7486 t.start("run".into());
7487 assert!(instruction_for(&t).contains("Explicit operator resumption"));
7488 }
7489
7490 #[test]
7494 fn unrelated_ci_refunds_once_per_pass_and_holds_for_the_owner() {
7495 let mut state = RunState::new(
7496 PathBuf::from("/repo"),
7497 "main".into(),
7498 "base".into(),
7499 "task".into(),
7500 Config::default(),
7501 );
7502 state.status = RunStatus::Landing;
7503 state.parked = true;
7504 state.pr_pushed_head = Some("head".into());
7505 state.land_unrelated = Some(
7506 serde_json::from_value(serde_json::json!({
7507 "head": "head", "question": "ci-question",
7508 "evidence": {"head": "head", "base": "base", "changed": [], "checks": []}
7509 }))
7510 .unwrap(),
7511 );
7512 let mut t = task();
7513 t.start(state.id.clone());
7514 let verdict = Verdict {
7515 status: RunStatus::Landing,
7516 left_pr: true,
7517 quota_hit: false,
7518 parked: true,
7519 no_viable_candidates: false,
7520 };
7521 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
7522 assert_eq!(t.attempts, 0);
7523 assert_eq!(t.status, TaskStatus::Held);
7524 assert!(!t.operator_held());
7525 assert!(
7526 t.hold_reason
7527 .as_deref()
7528 .unwrap()
7529 .starts_with("[ci-unrelated]")
7530 );
7531 t.attempts = 1;
7533 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
7534 assert_eq!(t.attempts, 1);
7535 assert_eq!(t.passes.len(), 1);
7536 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
7537 state.pr_pushed_head = Some("new-head".into());
7538 state.parked = false;
7539 state.status = RunStatus::Blocked;
7540 let stale = Verdict {
7541 status: RunStatus::Blocked,
7542 parked: false,
7543 ..verdict
7544 };
7545 settle_and_diagnose(&mut t, stale, "stale CI hold", 2, &state);
7546 assert_eq!(t.attempts, 1);
7547 state.begin_pass();
7549 state.begin_pass();
7550 state.status = RunStatus::Landing;
7551 state.parked = true;
7552 state.pr_pushed_head = Some("head".into());
7553 settle_and_diagnose(&mut t, verdict, "CI hold on resume", 2, &state);
7554 assert_eq!(t.attempts, 0);
7555 assert_eq!(t.passes.len(), 2);
7556 }
7557
7558 #[test]
7559 fn orphan_recovery_does_not_refund_a_previous_pass_loss() {
7560 let mut state = run_state(RunStatus::Failed);
7561 state
7562 .quota
7563 .push(loss("advisor-1", "2026-10-22T14:38:00Z", None));
7564 state.begin_pass();
7565 let mut t = task();
7566 t.start(state.id.clone());
7567 reclaim(&mut t, Some(state), 2, "en");
7568 assert_eq!(t.attempts, 1);
7569 }
7570
7571 #[test]
7572 fn an_all_seats_unauthenticated_node_refunds_and_holds_without_retry() {
7573 let mut state = run_state(RunStatus::Blocked);
7574 let losses: Vec<_> = ["alpha", "beta"]
7575 .into_iter()
7576 .enumerate()
7577 .map(|(i, agent)| {
7578 QuotaLoss::from_report(
7579 format!("advisor-{i}"),
7580 "advise",
7581 Some(&crate::agent::Quota {
7582 cause: crate::agent::QuotaCause::Authentication,
7583 agent: agent.to_owned(),
7584 reset: None,
7585 }),
7586 )
7587 })
7588 .collect();
7589 crate::graph::block_machine_outage(&mut state, "advise", &losses);
7590 state.quota = losses;
7591 let mut t = task();
7592 t.start(state.id.clone());
7593 let verdict = Verdict {
7594 status: state.status,
7595 left_pr: false,
7596 quota_hit: true,
7597 parked: false,
7598 no_viable_candidates: true,
7599 };
7600 settle_and_diagnose(&mut t, verdict, "authentication unavailable", 2, &state);
7601 assert_eq!(t.attempts, 0);
7602 assert_eq!(t.status, TaskStatus::Held);
7603 assert!(!t.operator_held());
7604 let reason = t.hold_reason.as_deref().unwrap();
7605 assert!(reason.starts_with("[authentication]"));
7606 assert!(
7607 reason.contains("alpha")
7608 && reason.contains("beta")
7609 && reason.contains("re-authenticate")
7610 );
7611 assert!(cooldown_until(&state.quota, Timestamp::now()).is_none());
7612 let mut orphan = task();
7614 orphan.start(state.id.clone());
7615 reclaim(&mut orphan, Some(state), 2, "en");
7616 assert_eq!(orphan.attempts, 0);
7617 assert_eq!(orphan.status, TaskStatus::Held);
7618 }
7619
7620 #[test]
7621 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
7622 let mut state = RunState::new(
7623 PathBuf::from("/repo"),
7624 "main".to_owned(),
7625 "abc1234".to_owned(),
7626 "task".to_owned(),
7627 Config::default(),
7628 );
7629 state.status = RunStatus::Blocked;
7630 state.resource_block = Some(crate::run::ResourceBlock {
7631 cause: crate::run::ResourceBlockCause::Other,
7632 node: "graft".to_owned(),
7633 kind: "shared build cache".to_owned(),
7634 detail: "timed out".to_owned(),
7635 at: jiff::Timestamp::now(),
7636 });
7637 let mut t = task();
7638 t.start("20260902-000000-aaaa".to_owned());
7639 assert_eq!(t.attempts, 1);
7640 let verdict = Verdict {
7641 status: RunStatus::Blocked,
7642 left_pr: false,
7643 quota_hit: false,
7644 parked: false,
7645 no_viable_candidates: false,
7646 };
7647 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
7648 assert_eq!(t.status, TaskStatus::Held);
7649 assert_eq!(t.attempts, 0, "the attempt is refunded");
7650 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
7651 assert!(
7652 !t.operator_held(),
7653 "a machine hold, released without a human"
7654 );
7655
7656 let mut selected = task();
7657 selected.start("20260902-000000-cccc".into());
7658 selected.replacement = Some(crate::merge_decision::Replacement {
7659 question: "fixture".into(),
7660 run: "earlier".into(),
7661 pr: "https://example.invalid/pr/1".into(),
7662 agent: "selected".into(),
7663 context: "fixture".into(),
7664 });
7665 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
7666 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
7667 assert_eq!(selected.attempts, 0);
7668 assert!(
7669 !selected.operator_held(),
7670 "a selected agent also retains a machine resource hold"
7671 );
7672 assert!(
7673 selected
7674 .hold_reason
7675 .as_deref()
7676 .unwrap()
7677 .starts_with("[resource] ")
7678 );
7679
7680 let mut p = task();
7682 p.start("20260902-000000-bbbb".to_owned());
7683 settle_and_diagnose(
7684 &mut p,
7685 Verdict {
7686 parked: true,
7687 ..verdict
7688 },
7689 "parked",
7690 2,
7691 &state,
7692 );
7693 assert_eq!(p.status, TaskStatus::Parked);
7694 assert_eq!(
7695 p.attempts, 0,
7696 "parking refunds once rather than also settling the resource hold"
7697 );
7698 }
7699
7700 #[test]
7701 fn record_failure_notice_links_the_task() {
7702 let dir = tempfile::tempdir().unwrap();
7703 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
7706 let queue = Queue::at(dir.path().join("queue"));
7707 let mut t = interrupt_task("rec-fail-1");
7708 record(&queue, &mut t);
7709
7710 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
7711 assert_eq!(found.len(), 1);
7712 assert_eq!(found[0].key, "loop:record");
7713 assert_eq!(
7714 found[0].link,
7715 Some(Link::Task {
7716 id: "rec-fail-1".to_owned()
7717 })
7718 );
7719 }
7720
7721 fn interrupt_task(id: &str) -> Task {
7724 let mut t = task();
7725 t.id = id.to_owned();
7726 t.interrupt = true;
7727 t
7728 }
7729
7730 fn task_with_id(id: &str) -> Task {
7732 let mut t = task();
7733 t.id = id.to_owned();
7734 t
7735 }
7736
7737 fn urgent_task(id: &str) -> Task {
7739 let mut t = task();
7740 t.id = id.to_owned();
7741 t.urgent = true;
7742 t
7743 }
7744
7745 #[test]
7746 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
7747 let dir = tempfile::tempdir().unwrap();
7748 let queue = Queue::at(dir.path().join("queue"));
7749 let mut parked = task_with_id("parked");
7750 parked.start("run".into());
7751 parked.park("upgrade");
7752 parked.record_pass_at(
7753 "run",
7754 0,
7755 Some(Timestamp::now()),
7756 1,
7757 Some(crate::run::ParkCause::Upgrade {
7758 from: "old".into(),
7759 to: None,
7760 }),
7761 );
7762 let mut fresh = task_with_id("fresh");
7763 fresh.priority = 100;
7764 let mut urgent_task = urgent_task("urgent");
7765 for t in [&mut parked, &mut fresh, &mut urgent_task] {
7766 queue.put(t).unwrap();
7767 }
7768 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7769 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7770 let candidates = runnable(&queue);
7771 assert_eq!(candidates[0].id, parked.id);
7772 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
7773 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_err());
7774 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_ok());
7775 drop(first);
7776 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
7777 let next = runnable(&queue)
7778 .into_iter()
7779 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent).ok())
7780 .unwrap();
7781 assert_eq!(next.task.id, fresh.id);
7782 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
7783 drop(claimed_elsewhere);
7784 drop(next);
7785 assert!(matches!(
7786 claim_dispatch(&queue, &parked, true, &ordinary, &urgent),
7787 Err(SkipReason::CoolingDown)
7788 ));
7789 }
7790
7791 #[test]
7792 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
7793 let dir = tempfile::tempdir().unwrap();
7794 let queue = Queue::at(dir.path().join("queue"));
7795 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7796 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7797 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
7798 let mut first = task();
7799 queue.put(&mut first).unwrap();
7800 let stale_plain = queue.get(&first.id).unwrap();
7801 first.set_urgent(true).unwrap();
7802 queue.put(&mut first).unwrap();
7803 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
7804 let mut second = urgent_task("second");
7805 queue.put(&mut second).unwrap();
7806 assert!(matches!(
7807 claim_dispatch(&queue, &second, false, &ordinary, &urgent),
7808 Err(SkipReason::UrgentCapacity)
7809 ));
7810 drop(queue.claim(&second.id).unwrap());
7811 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
7812 drop(dispatched);
7813 let stale_urgent = queue.get(&first.id).unwrap();
7814 first.set_urgent(false).unwrap();
7815 queue.put(&mut first).unwrap();
7816 assert!(matches!(
7817 claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent),
7818 Err(SkipReason::OrdinaryCapacity)
7819 ));
7820 assert_eq!(urgent.available_permits(), 1);
7821 drop(queue.claim(&first.id).unwrap());
7822 drop(occupied);
7823 }
7824
7825 #[test]
7826 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
7827 let dir = tempfile::tempdir().unwrap();
7828 let queue = Queue::at(dir.path().join("queue"));
7829 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7830 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7831 let mut t = task();
7832 queue.put(&mut t).unwrap();
7833 for enabled in [true, false] {
7834 let stale = queue.get(&t.id).unwrap();
7835 t.set_urgent(enabled).unwrap();
7836 queue.put(&mut t).unwrap();
7837 assert_ne!(stale.urgent, enabled);
7838 let DispatchClaim {
7839 task: fresh,
7840 claim,
7841 permit,
7842 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
7843 assert_eq!(fresh.urgent, enabled);
7844 assert_eq!(ordinary.available_permits(), usize::from(enabled));
7845 assert_eq!(urgent.available_permits(), usize::from(!enabled));
7846 assert!(queue.claim(&t.id).is_err());
7847 assert_eq!(fresh.attempts, stale.attempts);
7848 drop((claim, permit));
7849 assert_eq!(ordinary.available_permits(), 1);
7850 assert_eq!(urgent.available_permits(), 1);
7851 }
7852 t.set_urgent(true).unwrap();
7853 queue.put(&mut t).unwrap();
7854 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
7855 assert!(matches!(
7856 claim_dispatch(&queue, &t, false, &ordinary, &urgent),
7857 Err(SkipReason::UrgentCapacity)
7858 ));
7859 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
7860 drop(queue.claim(&t.id).unwrap());
7861 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
7862 drop(occupied);
7863 assert!(matches!(
7864 claim_dispatch(&queue, &t, true, &ordinary, &urgent),
7865 Err(SkipReason::CoolingDown)
7866 ));
7867 drop(queue.claim(&t.id).unwrap());
7868 let held = queue.claim(&t.id).unwrap();
7869 assert!(matches!(
7870 claim_dispatch(&queue, &t, false, &ordinary, &urgent),
7871 Err(SkipReason::ClaimUnavailable)
7872 ));
7873 drop(held);
7874 t.status = TaskStatus::Held;
7875 queue.put(&mut t).unwrap();
7876 assert!(matches!(
7877 claim_dispatch(&queue, &t, false, &ordinary, &urgent),
7878 Err(SkipReason::NoLongerRunnable)
7879 ));
7880 drop(queue.claim(&t.id).unwrap());
7881 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
7882 assert!(matches!(
7883 claim_dispatch(&queue, &t, false, &ordinary, &urgent),
7884 Err(SkipReason::RereadFailed)
7885 ));
7886 drop(queue.claim(&t.id).unwrap());
7887 assert_eq!(urgent.available_permits(), 1);
7888 }
7889
7890 #[test]
7895 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
7896 assert_eq!(permit_kind(true, false), PermitKind::None);
7897 assert_eq!(permit_kind(true, true), PermitKind::None);
7898 }
7899
7900 #[test]
7905 fn permit_kind_separates_urgent_from_ordinary() {
7906 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
7907 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
7908 }
7909
7910 #[test]
7917 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
7918 let cfg = Config::default();
7919 let repo = Path::new("/any/repo/path");
7920
7921 let reason = disk_gate_with(repo, &cfg, PathBuf::new, |_| Ok(1024))
7922 .expect("must hold below the threshold");
7923 assert!(reason.contains("1024"), "{reason}");
7924 assert!(
7925 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7926 "{reason}"
7927 );
7928
7929 assert_eq!(
7930 disk_gate_with(repo, &cfg, PathBuf::new, |_| Ok(cfg.disk.min_free_bytes)),
7931 None,
7932 "exactly at the floor is open"
7933 );
7934 assert_eq!(
7935 disk_gate_with(
7936 repo,
7937 &cfg,
7938 PathBuf::new,
7939 |_| Ok(cfg.disk.min_free_bytes + 1)
7940 ),
7941 None,
7942 "comfortably above the floor is open"
7943 );
7944 }
7945
7946 #[test]
7947 fn disk_gate_with_checks_cache_and_worktree_volumes_and_measurement_errors() {
7948 let dir = tempfile::tempdir().unwrap();
7949 let repo = dir.path().join("repo");
7950 let cache = dir.path().join("cache");
7951 let worktrees = dir.path().join("worktrees");
7952 for path in [&repo, &cache, &worktrees] {
7953 std::fs::create_dir(path).unwrap();
7954 }
7955 let mut cfg = Config::default();
7956 cfg.graph.worktree_root = Some(worktrees.clone());
7957 cfg.verify.gate = vec![format!(
7958 "CARGO_TARGET_DIR='{}' cargo build",
7959 cache.display()
7960 )];
7961 for blocked in [&cache, &worktrees] {
7962 for unmeasured in [false, true] {
7963 assert!(
7964 disk_gate_with(&repo, &cfg, PathBuf::new, |path| {
7965 if path == blocked {
7966 if unmeasured {
7967 Err(anyhow::anyhow!("measurement failed"))
7968 } else {
7969 Ok(0)
7970 }
7971 } else {
7972 Ok(u64::MAX)
7973 }
7974 })
7975 .is_some(),
7976 "a blocked secondary output volume must close admission"
7977 );
7978 }
7979 }
7980 assert!(disk_gate_with(&repo, &cfg, PathBuf::new, |_| Ok(u64::MAX)).is_none());
7981 }
7982
7983 #[test]
7984 fn isolated_admission_measures_the_run_home_instead_of_the_shared_cache() {
7985 let dir = tempfile::tempdir().unwrap();
7986 let repo = dir.path().join("repo");
7987 let cache = dir.path().join("cache");
7988 let home_parent = dir.path().join("run-volume");
7989 let home = home_parent.join("new-home");
7990 for path in [&repo, &cache, &home_parent] {
7991 std::fs::create_dir(path).unwrap();
7992 }
7993 let mut cfg = Config::default();
7994 cfg.graph.worktree_root = Some(repo.clone());
7995 cfg.verify.gate = vec![format!(
7996 "CARGO_TARGET_DIR='{}' cargo build",
7997 cache.display()
7998 )];
7999 cfg.verify.build_cache = BuildCache::Isolated;
8000 let denied = |blocked: &Path| {
8001 disk_gate_with(
8002 &repo,
8003 &cfg,
8004 || home.clone(),
8005 |path| Ok(if path == blocked { 0 } else { u64::MAX }),
8006 )
8007 .is_some()
8008 };
8009 assert!(!denied(&cache));
8010 assert!(denied(&home_parent));
8011 }
8012
8013 #[test]
8014 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
8015 let mut cfg = Config::default();
8016 cfg.disk.min_free_bytes = 0;
8017 let repo = Path::new("/any/repo/path");
8018 assert_eq!(
8019 disk_gate_with(repo, &cfg, PathBuf::new, |_| Ok(0)),
8020 None,
8021 "a zero floor never measures at all"
8022 );
8023 }
8024
8025 #[test]
8026 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
8027 let cfg = Config::default();
8028 let repo = Path::new("/any/repo/path");
8029 let reason = disk_gate_with(repo, &cfg, PathBuf::new, |_| {
8030 Err(anyhow::anyhow!("no df on this box"))
8031 })
8032 .expect("a measurement failure must close the gate, not open it");
8033 assert!(reason.contains("could not measure"), "{reason}");
8034 }
8035
8036 #[test]
8037 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
8038 let ordinary = task();
8039 let next = advance_interrupt(
8040 Interrupt::Idle,
8041 std::slice::from_ref(&ordinary.id),
8042 std::slice::from_ref(&ordinary),
8043 );
8044 assert_eq!(next, Interrupt::Idle);
8045 }
8046
8047 #[test]
8048 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
8049 let marked = interrupt_task("marked");
8052 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
8053 assert_eq!(next, Interrupt::Idle);
8054 }
8055
8056 #[test]
8057 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
8058 let marked = interrupt_task("marked");
8059 let next = advance_interrupt(
8060 Interrupt::Idle,
8061 &["running".to_owned()],
8062 std::slice::from_ref(&marked),
8063 );
8064 assert_eq!(
8065 next,
8066 Interrupt::Parking {
8067 parked: vec!["running".to_owned()],
8068 interrupt_task: "marked".to_owned(),
8069 }
8070 );
8071 }
8072
8073 #[test]
8082 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
8083 let marked = interrupt_task("marked");
8084
8085 let two = advance_interrupt(
8086 Interrupt::Idle,
8087 &["a".to_owned(), "b".to_owned()],
8088 std::slice::from_ref(&marked),
8089 );
8090 assert_eq!(two, Interrupt::Idle);
8091
8092 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
8093 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
8094 }
8095
8096 #[test]
8097 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
8098 let state = Interrupt::Parking {
8099 parked: vec!["running".to_owned()],
8100 interrupt_task: "marked".to_owned(),
8101 };
8102 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
8104 assert_eq!(still_going, state);
8105
8106 let stopped_but_not_yet_dispatched =
8110 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
8111 assert_eq!(stopped_but_not_yet_dispatched, state);
8112
8113 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
8115 assert_eq!(
8116 dispatched,
8117 Interrupt::Running {
8118 parked: vec!["running".to_owned()],
8119 interrupt_task: "marked".to_owned(),
8120 }
8121 );
8122 }
8123
8124 #[test]
8125 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
8126 let state = Interrupt::Running {
8127 parked: vec!["running".to_owned()],
8128 interrupt_task: "marked".to_owned(),
8129 };
8130 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
8131 assert_eq!(still_running, state);
8132
8133 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
8140 assert_eq!(
8141 ended,
8142 Interrupt::Resuming {
8143 parked: vec!["running".to_owned()]
8144 }
8145 );
8146 }
8147
8148 #[test]
8149 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
8150 let state = Interrupt::Resuming {
8151 parked: vec!["running".to_owned()],
8152 };
8153 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
8154 assert_eq!(still_waiting, state);
8155
8156 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
8157 assert_eq!(dispatched, Interrupt::Idle);
8158 }
8159
8160 #[test]
8166 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
8167 {
8168 let state = Interrupt::Parking {
8169 parked: vec!["running".to_owned()],
8170 interrupt_task: "marked".to_owned(),
8171 };
8172 let next = advance_interrupt(state, &[], &[]);
8175 assert_eq!(
8176 next,
8177 Interrupt::Resuming {
8178 parked: vec!["running".to_owned()]
8179 },
8180 "abandoning the interrupt must not abandon the resume it owes"
8181 );
8182 }
8183
8184 #[test]
8187 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
8188 let state = Interrupt::Resuming {
8189 parked: vec!["running".to_owned()],
8190 };
8191 let next = advance_interrupt(state, &[], &[]);
8192 assert_eq!(
8193 next,
8194 Interrupt::Idle,
8195 "nothing is left to wait for; the loop must not stay wedged"
8196 );
8197 }
8198
8199 #[test]
8200 fn disabled_by_config_the_sequence_can_never_leave_idle() {
8201 let marked = interrupt_task("marked");
8202 let next = advance_interrupt_tick(
8203 false,
8204 Interrupt::Idle,
8205 &["running".to_owned()],
8206 std::slice::from_ref(&marked),
8207 );
8208 assert_eq!(
8209 next,
8210 Interrupt::Idle,
8211 "an unmarked, unconfigured daemon must behave exactly as before"
8212 );
8213 }
8214
8215 #[test]
8216 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
8217 let state = Interrupt::Parking {
8218 parked: vec!["running".to_owned()],
8219 interrupt_task: "marked".to_owned(),
8220 };
8221 let candidates = vec![interrupt_task("marked"), task()];
8222 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
8223 assert!(
8224 allowed.is_empty(),
8225 "nothing may dispatch - not even the interrupt task itself - \
8226 until the parked run has actually stopped"
8227 );
8228 }
8229
8230 #[test]
8244 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
8245 for state in [
8246 Interrupt::Parking {
8247 parked: vec!["running".to_owned()],
8248 interrupt_task: "marked".to_owned(),
8249 },
8250 Interrupt::Running {
8251 parked: vec!["running".to_owned()],
8252 interrupt_task: "marked".to_owned(),
8253 },
8254 Interrupt::Resuming {
8255 parked: vec!["running".to_owned()],
8256 },
8257 ] {
8258 let candidates = vec![
8259 interrupt_task("marked"),
8260 urgent_task("hot"),
8261 task_with_id("ordinary"),
8262 ];
8263 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
8264 assert!(
8265 !allowed.iter().any(|t| t.id == "hot"),
8266 "an urgent candidate must wait out the same gate as anything \
8267 else while the run it would run alongside has not actually \
8268 left flight, for state {state:?}: {allowed:?}"
8269 );
8270 }
8271 }
8272
8273 #[test]
8279 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
8280 let state = Interrupt::Resuming {
8281 parked: vec!["hot".to_owned()],
8282 };
8283 let candidates = vec![urgent_task("hot"), task()];
8284 let allowed = interrupt_gate(&state, &[], candidates);
8285 assert_eq!(
8286 allowed.iter().filter(|t| t.id == "hot").count(),
8287 1,
8288 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
8289 );
8290 }
8291
8292 #[test]
8293 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
8294 let state = Interrupt::Parking {
8295 parked: vec!["running".to_owned()],
8296 interrupt_task: "marked".to_owned(),
8297 };
8298 let other = task();
8299 let candidates = vec![interrupt_task("marked"), other.clone()];
8300 let allowed = interrupt_gate(&state, &[], candidates);
8301 assert_eq!(allowed.len(), 1);
8302 assert_eq!(allowed[0].id, "marked");
8303 }
8304
8305 #[test]
8306 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
8307 let state = Interrupt::Running {
8308 parked: vec!["running".to_owned()],
8309 interrupt_task: "marked".to_owned(),
8310 };
8311 let candidates = vec![task(), task()];
8312 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
8313 assert!(allowed.is_empty());
8314 }
8315
8316 #[test]
8323 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
8324 let state = Interrupt::Resuming {
8325 parked: vec!["a".to_owned(), "c".to_owned()],
8326 };
8327 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
8328 let allowed = interrupt_gate(&state, &[], candidates);
8329 assert_eq!(
8330 allowed.len(),
8331 1,
8332 "at most one candidate may be offered while resuming: {allowed:?}"
8333 );
8334 assert_eq!(allowed[0].id, "a");
8335 }
8336
8337 #[test]
8338 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
8339 let state = Interrupt::Resuming {
8340 parked: vec!["a".to_owned()],
8341 };
8342 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
8343 assert!(allowed.is_empty());
8344 }
8345
8346 #[test]
8352 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
8353 let running = task(); let marked = interrupt_task("marked");
8355
8356 let mut state = Interrupt::Idle;
8357 let in_flight = vec![running.id.clone()];
8359 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
8360 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
8361 assert!(gated.is_empty(), "still waiting on `running` to park");
8362
8363 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
8365 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
8366 assert_eq!(
8367 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
8368 vec!["marked"],
8369 "only the interrupt task may be offered to the dispatcher now"
8370 );
8371
8372 state = advance_interrupt_tick(
8374 true,
8375 state,
8376 &["marked".to_owned()],
8377 std::slice::from_ref(&running),
8378 );
8379 let gated = interrupt_gate(
8380 &state,
8381 &["marked".to_owned()],
8382 vec![marked.clone(), running.clone()],
8383 );
8384 assert!(
8385 gated.is_empty(),
8386 "the parked run must not be offered back while the interrupt \
8387 task is still running"
8388 );
8389
8390 let other = task_with_id("other");
8394 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
8395 assert_eq!(
8396 state,
8397 Interrupt::Resuming {
8398 parked: vec![running.id.clone()]
8399 }
8400 );
8401 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
8402 assert_eq!(
8403 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
8404 vec![running.id.as_str()],
8405 "exactly the parked run resumes - not the unrelated task, even \
8406 though it was offered first"
8407 );
8408
8409 state = advance_interrupt_tick(
8413 true,
8414 state,
8415 std::slice::from_ref(&running.id),
8416 std::slice::from_ref(&other),
8417 );
8418 assert_eq!(state, Interrupt::Idle);
8419 let gated = interrupt_gate(
8420 &state,
8421 std::slice::from_ref(&running.id),
8422 vec![other.clone()],
8423 );
8424 assert_eq!(
8425 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
8426 vec![other.id.as_str()],
8427 "ordinary dispatch is unrestricted again"
8428 );
8429 }
8430
8431 #[test]
8432 fn every_run_status_settles_the_task_it_came_from() {
8433 let table = [
8435 (RunStatus::Merged, TaskStatus::Done, 1),
8436 (RunStatus::Ready, TaskStatus::Done, 1),
8437 (RunStatus::Stalled, TaskStatus::Failed, 0),
8438 (RunStatus::Blocked, TaskStatus::Failed, 1),
8439 (RunStatus::Failed, TaskStatus::Failed, 1),
8440 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
8441 (RunStatus::Prep, TaskStatus::Failed, 1),
8442 (RunStatus::Implementing, TaskStatus::Failed, 1),
8443 (RunStatus::Judging, TaskStatus::Failed, 1),
8444 (RunStatus::Deliberating, TaskStatus::Failed, 1),
8445 (RunStatus::Voting, TaskStatus::Failed, 1),
8446 (RunStatus::Reviewing, TaskStatus::Failed, 1),
8447 (RunStatus::Gating, TaskStatus::Failed, 1),
8448 ];
8449 for (run, want, attempts) in table {
8450 let mut t = task();
8451 t.start("20260902-000000-aaaa".to_owned());
8452 settle(
8453 &mut t,
8454 Verdict {
8455 status: run,
8456 left_pr: false,
8457 parked: false,
8458 quota_hit: matches!(run, RunStatus::Stalled),
8459 no_viable_candidates: false,
8460 },
8461 "why",
8462 2,
8463 );
8464 assert_eq!(t.status, want, "task status after {}", label(run));
8465 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
8466 }
8467 }
8468
8469 #[test]
8470 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
8471 let mut stalled = task();
8472 stalled.start("20260902-000000-aaaa".to_owned());
8473 settle(
8474 &mut stalled,
8475 Verdict {
8476 status: RunStatus::Stalled,
8477 left_pr: false,
8478 parked: false,
8479 quota_hit: true,
8480 no_viable_candidates: false,
8481 },
8482 "quota",
8483 1,
8484 );
8485 assert_eq!(stalled.attempts, 0);
8486 assert!(
8487 stalled.status.runnable(),
8488 "a machine problem must leave the task in line"
8489 );
8490
8491 let mut blocked = task();
8492 blocked.start("20260902-000000-aaaa".to_owned());
8493 settle(
8494 &mut blocked,
8495 Verdict {
8496 status: RunStatus::Blocked,
8497 left_pr: false,
8498 parked: false,
8499 quota_hit: false,
8500 no_viable_candidates: false,
8501 },
8502 "findings open",
8503 1,
8504 );
8505 assert_eq!(blocked.attempts, 1);
8506 assert_eq!(
8507 blocked.status,
8508 TaskStatus::Held,
8509 "the last attempt hands the task to a human"
8510 );
8511 }
8512
8513 #[test]
8514 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
8515 let mut delivered = task();
8518 delivered.start("20260903-080619-01c2".to_owned());
8519 settle(
8520 &mut delivered,
8521 Verdict {
8522 status: RunStatus::Blocked,
8523 left_pr: true,
8524 parked: false,
8525 quota_hit: false,
8526 no_viable_candidates: false,
8527 },
8528 "no check status",
8529 4,
8530 );
8531 assert_eq!(
8532 delivered.status,
8533 TaskStatus::Held,
8534 "a pull request waiting on CI or a person is not a retryable failure"
8535 );
8536 assert!(
8537 !delivered.status.runnable(),
8538 "the loop must not pick this task up again"
8539 );
8540 assert_eq!(
8541 delivered.last_error.as_deref(),
8542 Some("no check status"),
8543 "the operator needs to be told what the gate was waiting for"
8544 );
8545
8546 let mut empty_handed = task();
8549 empty_handed.start("20260903-080619-01c2".to_owned());
8550 settle(
8551 &mut empty_handed,
8552 Verdict {
8553 status: RunStatus::Blocked,
8554 left_pr: false,
8555 parked: false,
8556 quota_hit: false,
8557 no_viable_candidates: false,
8558 },
8559 "findings open",
8560 4,
8561 );
8562 assert_eq!(empty_handed.status, TaskStatus::Failed);
8563 assert!(empty_handed.status.runnable());
8564 }
8565
8566 #[test]
8567 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
8568 let mut noop = task();
8574 noop.start("20260912-131304-391f".to_owned());
8575 settle(
8576 &mut noop,
8577 Verdict {
8578 status: RunStatus::VerifiedNoop,
8579 left_pr: false,
8580 parked: false,
8581 quota_hit: false,
8582 no_viable_candidates: true,
8583 },
8584 "candidate A: already fixed by b32cfc4, on main",
8585 4,
8586 );
8587 assert_eq!(
8588 noop.status,
8589 TaskStatus::Held,
8590 "an unverified claim is a request for a human, not a failure"
8591 );
8592 assert!(
8593 !noop.status.runnable(),
8594 "the loop must not requeue this on the same unverified claim"
8595 );
8596 assert_eq!(noop.attempts, 1);
8601 }
8602
8603 #[test]
8604 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
8605 let mut parked = task();
8610 parked.start("20260903-183634-2d98".to_owned());
8611 settle(
8612 &mut parked,
8613 Verdict {
8614 status: RunStatus::Implementing,
8615 left_pr: false,
8616 quota_hit: false,
8617 parked: true,
8618 no_viable_candidates: false,
8619 },
8620 "parked after `implementing`",
8621 2,
8622 );
8623 assert_eq!(parked.attempts, 0, "a park is refunded");
8624 assert!(
8625 parked.status.runnable(),
8626 "and the task stays in line so the next loop resumes its run"
8627 );
8628 assert_eq!(parked.status, TaskStatus::Parked);
8629 assert_eq!(
8630 parked.park_reason.as_deref(),
8631 Some("parked after `implementing`"),
8632 "the card says where it stopped"
8633 );
8634 assert_eq!(parked.last_error, None, "nothing failed");
8635
8636 let mut broken = task();
8640 broken.start("20260903-183634-2d98".to_owned());
8641 settle(
8642 &mut broken,
8643 Verdict {
8644 status: RunStatus::Implementing,
8645 left_pr: false,
8646 quota_hit: false,
8647 parked: false,
8648 no_viable_candidates: false,
8649 },
8650 "returned mid-flight",
8651 2,
8652 );
8653 assert_eq!(broken.attempts, 1);
8654 }
8655
8656 #[test]
8657 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
8658 let mut flaky = task();
8663 flaky.start("20260903-123023-e633".to_owned());
8664 settle(
8665 &mut flaky,
8666 Verdict {
8667 status: RunStatus::Stalled,
8668 left_pr: false,
8669 parked: false,
8670 quota_hit: false,
8671 no_viable_candidates: false,
8672 },
8673 "verdict rests on 1 of 3 judges",
8674 2,
8675 );
8676 assert_eq!(
8677 flaky.attempts, 1,
8678 "flakiness spends an attempt, so `max_attempts` still bounds it"
8679 );
8680 assert!(flaky.status.runnable(), "and it is still worth retrying");
8681
8682 let mut limited = task();
8684 limited.start("20260903-123023-e633".to_owned());
8685 settle(
8686 &mut limited,
8687 Verdict {
8688 status: RunStatus::Stalled,
8689 left_pr: false,
8690 parked: false,
8691 quota_hit: true,
8692 no_viable_candidates: false,
8693 },
8694 "judge-2, judge-3 out of quota",
8695 2,
8696 );
8697 assert_eq!(limited.attempts, 0, "a quota window is refunded");
8698 assert!(limited.status.runnable());
8699
8700 let mut worn = task();
8703 for _ in 0..2 {
8704 worn.release();
8705 }
8706 worn.start("20260903-123023-e633".to_owned());
8707 worn.attempts = 2;
8708 settle(
8709 &mut worn,
8710 Verdict {
8711 status: RunStatus::Stalled,
8712 left_pr: false,
8713 parked: false,
8714 quota_hit: false,
8715 no_viable_candidates: false,
8716 },
8717 "no quorum again",
8718 2,
8719 );
8720 assert_eq!(worn.status, TaskStatus::Held);
8721 assert!(!worn.status.runnable());
8722 }
8723
8724 #[test]
8725 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
8726 let mut wiped_out = task();
8733 wiped_out.start("20260907-025000-a1b2".to_owned());
8734 settle(
8735 &mut wiped_out,
8736 Verdict {
8737 status: RunStatus::Failed,
8738 left_pr: false,
8739 parked: false,
8740 quota_hit: true,
8741 no_viable_candidates: true,
8742 },
8743 "no candidate produced a change; nothing to judge",
8744 2,
8745 );
8746 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
8747 assert!(
8748 wiped_out.status.runnable(),
8749 "a machine problem must leave the task in line"
8750 );
8751
8752 let mut partial_progress = task();
8758 partial_progress.start("20260907-025500-c3d4".to_owned());
8759 settle(
8760 &mut partial_progress,
8761 Verdict {
8762 status: RunStatus::Failed,
8763 left_pr: false,
8764 parked: false,
8765 quota_hit: true,
8766 no_viable_candidates: false,
8767 },
8768 "gate failed on the winning candidate",
8769 2,
8770 );
8771 assert_eq!(
8772 partial_progress.attempts, 1,
8773 "a candidate that actually produced a change spends the attempt \
8774 even though some other seat hit its quota"
8775 );
8776 assert!(partial_progress.status.runnable());
8777 }
8778
8779 #[test]
8780 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
8781 let mut t = task();
8787 t.start("20260907-025000-a1b2".to_owned());
8788 let mut state = run_state(RunStatus::Failed);
8789 state.quota.push(QuotaLoss {
8790 cause: crate::agent::QuotaCause::UsageLimit,
8791 agent: String::new(),
8792 seat: "cand-a".to_owned(),
8793 node: "implement".to_owned(),
8794 at: Timestamp::now(),
8795 reset: None,
8796 });
8797 assert!(
8798 state.viable().is_empty(),
8799 "no candidate was added, so nothing is viable"
8800 );
8801 reclaim(&mut t, Some(state), 2, "en");
8802 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
8803 assert!(t.status.runnable());
8804 }
8805
8806 #[test]
8807 fn a_held_task_is_never_offered_to_the_loop() {
8808 let dir = tempfile::tempdir().unwrap();
8809 let queue = Queue::at(dir.path().to_path_buf());
8810 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
8811 let mut t = task();
8812 t.id = format!("2026090{n}-000000-000{n}");
8813 t.priority = priority;
8814 queue.put(&mut t).unwrap();
8815 }
8816 let mut held = task();
8817 held.id = "20260909-000000-9999".to_owned();
8818 held.priority = 99;
8819 held.hold_machine(None);
8820 queue.put(&mut held).unwrap();
8821
8822 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
8823 assert_eq!(order.len(), 3);
8824 assert!(!order.contains(&held.id));
8825 assert_eq!(
8826 order.first().cloned(),
8827 queue.next_runnable().map(|t| t.id),
8828 "the loop's first candidate is exactly what the queue offers"
8829 );
8830 assert_eq!(
8831 order,
8832 vec![
8833 "20260902-000000-0002".to_owned(),
8834 "20260903-000000-0003".to_owned(),
8835 "20260901-000000-0001".to_owned(),
8836 ],
8837 "priority first, then oldest, so nothing starves"
8838 );
8839 }
8840
8841 #[test]
8842 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
8843 let dir = tempfile::tempdir().unwrap();
8844 let queue = Queue::at(dir.path().to_path_buf());
8845 let mut old = task();
8846 old.id = "20260101-000000-old0".to_owned();
8847 queue.put(&mut old).unwrap();
8848 let mut fresh = task();
8849 fresh.id = "20260101-000000-new0".to_owned();
8850 queue.put(&mut fresh).unwrap();
8851
8852 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
8856 std::thread::sleep(Duration::from_millis(60));
8857 let live = queue.claim(&fresh.id).unwrap();
8858
8859 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8860 assert_eq!(swept, vec![old.id.clone()]);
8861 assert!(
8862 queue.claim(&old.id).is_ok(),
8863 "an unparseable lock older than the threshold is swept"
8864 );
8865 assert!(
8866 queue.claim(&fresh.id).is_err(),
8867 "a live pid protects its lock regardless of age"
8868 );
8869 drop(live);
8870 }
8871
8872 #[test]
8873 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
8874 let dir = tempfile::tempdir().unwrap();
8877 let queue = Queue::at(dir.path().to_path_buf());
8878 let mut t = task();
8879 t.id = "20260101-000000-live".to_owned();
8880 queue.put(&mut t).unwrap();
8881
8882 let claim = queue.claim(&t.id).unwrap();
8883 std::thread::sleep(Duration::from_millis(60));
8884
8885 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8886 assert!(
8887 swept.is_empty(),
8888 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
8889 );
8890 assert!(
8891 queue.claim(&t.id).is_err(),
8892 "the lock still protects its task"
8893 );
8894 drop(claim);
8895 }
8896
8897 fn injected_dead_pid() -> u32 {
8900 std::process::id().checked_add(1).unwrap_or(1)
8901 }
8902
8903 #[test]
8904 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
8905 let dir = tempfile::tempdir().unwrap();
8906 let queue = Queue::at(dir.path().to_path_buf());
8907 let mut t = task();
8908 t.id = "20260101-000000-dead".to_owned();
8909 queue.put(&mut t).unwrap();
8910 let dead_pid = injected_dead_pid();
8911
8912 std::fs::write(
8917 dir.path().join(format!("{}.lock", t.id)),
8918 dead_pid.to_string(),
8919 )
8920 .unwrap();
8921
8922 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8923 pid != dead_pid
8924 });
8925 assert_eq!(
8926 swept,
8927 vec![t.id.clone()],
8928 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
8929 );
8930 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
8931 }
8932
8933 #[test]
8934 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
8935 let dir = tempfile::tempdir().unwrap();
8936 let queue = Queue::at(dir.path().to_path_buf());
8937 let mut t = task();
8938 t.id = "20260101-000000-late".to_owned();
8939 queue.put(&mut t).unwrap();
8940 let dead_pid = injected_dead_pid();
8941
8942 assert!(
8945 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
8946 "nothing has claimed the task yet"
8947 );
8948
8949 std::fs::write(
8952 dir.path().join(format!("{}.lock", t.id)),
8953 dead_pid.to_string(),
8954 )
8955 .unwrap();
8956
8957 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8961 pid != dead_pid
8962 });
8963 assert_eq!(swept, vec![t.id.clone()]);
8964 }
8965
8966 #[test]
8967 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
8968 crate::run::pin_test_home();
8973 let dir = tempfile::tempdir().unwrap();
8974 let queue = Queue::at(dir.path().to_path_buf());
8975 let mut t = task();
8976 t.id = "20260101-000000-crsh".to_owned();
8977 t.status = TaskStatus::Running;
8978 t.attempts = 1;
8979 t.runs.push("20260904-000000-4043".to_owned());
8983 queue.put(&mut t).unwrap();
8984 let dead_pid = injected_dead_pid();
8985
8986 std::fs::write(
8989 dir.path().join(format!("{}.lock", t.id)),
8990 dead_pid.to_string(),
8991 )
8992 .unwrap();
8993
8994 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
9000 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
9001
9002 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
9003 pid != dead_pid
9004 });
9005 assert_eq!(swept, vec![t.id.clone()]);
9006
9007 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9008 assert_eq!(reclaimed, vec![t.id.clone()]);
9009 let after = queue.get(&t.id).unwrap();
9010 assert_eq!(
9011 after.status,
9012 TaskStatus::Held,
9013 "no run.json to recover from, so a human is asked"
9014 );
9015 assert_eq!(
9016 after.runs,
9017 vec!["20260904-000000-4043".to_owned()],
9018 "the crashed run's id is kept as evidence, not discarded"
9019 );
9020 }
9021
9022 #[test]
9023 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
9024 let dir = tempfile::tempdir().unwrap();
9025 let queue = Queue::at(dir.path().to_path_buf());
9026 let mut t = task();
9027 t.id = "20260101-000000-unknown".to_owned();
9028 queue.put(&mut t).unwrap();
9029 let dead_pid = injected_dead_pid();
9030 std::fs::write(
9031 dir.path().join(format!("{}.lock", t.id)),
9032 dead_pid.to_string(),
9033 )
9034 .unwrap();
9035
9036 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
9037 assert!(swept.is_empty(), "an unknown pid must keep its lock");
9038 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
9039 }
9040
9041 fn run_state_in(status: RunStatus, language: &str) -> RunState {
9042 let mut s = run_state(status);
9043 s.config.graph.language = language.to_owned();
9044 s
9045 }
9046
9047 fn unstarted_verdict(status: RunStatus) -> Verdict {
9048 Verdict {
9049 status,
9050 left_pr: false,
9051 quota_hit: false,
9052 parked: false,
9053 no_viable_candidates: false,
9054 }
9055 }
9056
9057 #[test]
9058 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
9059 let mut t = task();
9060 t.attempts = 1;
9061 settle_in(
9062 &mut t,
9063 unstarted_verdict(RunStatus::AlreadyInBase),
9064 "already in main",
9065 1,
9066 phrases("en"),
9067 );
9068 assert_eq!(t.status, TaskStatus::Done);
9069 assert_eq!(t.attempts, 0);
9070 }
9071
9072 #[test]
9073 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
9074 let reason = |language: &str| {
9075 let mut t = task();
9076 settle_in(
9077 &mut t,
9078 unstarted_verdict(RunStatus::Judging),
9079 "boom",
9080 1,
9081 phrases(language),
9082 );
9083 t.last_error.or(t.hold_reason).unwrap_or_default()
9084 };
9085 assert!(
9086 reason("en").starts_with("the graph stopped at `"),
9087 "{}",
9088 reason("en")
9089 );
9090 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
9091 assert!(reason("日本語").contains("boom"));
9092 assert_eq!(reason("fr"), reason("en"));
9093 }
9094
9095 #[test]
9096 fn describe_follows_the_run_language_and_keeps_the_run_id() {
9097 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
9098 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
9099 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
9100 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
9101 assert!(ja.contains("[run "), "{ja}");
9102 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
9103 assert!(ended.starts_with("run 終了: "), "{ended}");
9104 let mut de = run_state_in(RunStatus::Failed, "de");
9105 let mut en = run_state_in(RunStatus::Failed, "en");
9106 de.id = "same".to_owned();
9107 en.id = "same".to_owned();
9108 assert_eq!(describe(&de), describe(&en));
9109 }
9110
9111 #[test]
9112 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
9113 use crate::handover::Refused;
9114 let cases = [
9115 Refused::Foreign {
9116 branch: "b".into(),
9117 path: "/w/x".into(),
9118 why: "made by hand".into(),
9119 },
9120 Refused::Unsafe {
9121 branch: "b".into(),
9122 path: "/w/x".into(),
9123 why: "its worktree has uncommitted changes (a.rs)".into(),
9124 },
9125 Refused::ReleaseFailed {
9126 branch: "b".into(),
9127 path: "/w/x".into(),
9128 run: "ab12".into(),
9129 },
9130 ];
9131 for r in &cases {
9132 let en = (phrases("en").handover_refused)(r);
9133 assert_eq!(en, r.to_string());
9134 let ja = (phrases("ja").handover_refused)(r);
9135 assert!(
9136 !ja.contains("is checked out") && !ja.contains("try again"),
9137 "{ja}"
9138 );
9139 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
9140 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
9141 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
9142 }
9143 }
9144 assert!(phrases("en").handover_hint.contains("release the task"));
9145 assert!(phrases("ja").handover_hint.contains("解放"));
9146 }
9147
9148 #[test]
9149 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
9150 let build = |lang: &str| {
9151 let mut s = run_state_in(RunStatus::Blocked, lang);
9152 s.id = "same".to_owned();
9153 s.config.graph.review_rounds = 3;
9154 let mut r = review_round(3);
9155 r.expected = 3;
9156 r.answered = 1;
9157 s.reviews.push(r);
9158 s.event(
9159 "review",
9160 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
9161 );
9162 s
9163 };
9164 let en = describe(&build("en"));
9165 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
9166 let ja = describe(&build("ja"));
9167 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
9168 assert!(!ja.contains("never answered"), "{ja}");
9169 assert!(ja.contains("[run "), "{ja}");
9170
9171 let mut failed = build("ja");
9174 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
9175 command: "cargo test".to_owned(),
9176 code: Some(1),
9177 output_tail: String::new(),
9178 duration_ms: 0,
9179 resource_blocked: false,
9180 });
9181 failed.event("review", "stopped; e2e failed: cargo test");
9182 let ja = describe(&failed);
9183 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
9184 assert!(!ja.contains("席のレビュアー"), "{ja}");
9185 }
9186
9187 #[test]
9188 fn refusals_and_recovery_prose_follow_the_language() {
9189 let home = crate::run::test_home();
9190 let t = held_task_with("r1");
9191 let q = action_question(
9192 "r1",
9193 ask::ChoiceAction::Resume {
9194 run: "r1".to_owned(),
9195 },
9196 );
9197 let refuse = |p: &Phrases| match decide_action(&home, &t, &q, p, |_: &str| bail!("gone")) {
9198 ActionDecision::Refuse(s) => s,
9199 other => panic!("{other:?}"),
9200 };
9201 assert!(refuse(phrases("en")).contains("could not be read: gone"));
9202 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
9203 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
9204
9205 let mut held = task();
9206 reclaim(&mut held, None, 2, "ja");
9207 assert!(held.hold_reason.unwrap().contains("保留にしました"));
9208 let mut held = task();
9209 reclaim(&mut held, None, 2, "xx");
9210 assert!(held.hold_reason.unwrap().contains("held for a human"));
9211 }
9212
9213 fn run_state(status: RunStatus) -> RunState {
9214 let mut state = RunState::new(
9215 PathBuf::from("/repo"),
9216 "main".to_owned(),
9217 "abc1234def".to_owned(),
9218 "add retries".to_owned(),
9219 Config::default(),
9220 );
9221 state.status = status;
9222 state
9223 }
9224
9225 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
9226 Candidate {
9227 index: 0,
9228 label,
9229 agent: "claude".to_owned(),
9230 branch: format!("magi/x/{label}"),
9231 worktree: PathBuf::from("/repo"),
9232 summary: summary.to_owned(),
9233 stat: String::new(),
9234 files: 0,
9235 commits: usize::from(!empty),
9236 empty,
9237 failed: failed.map(str::to_owned),
9238 verified_noop: None,
9239 duration_ms: 0,
9240 folded: false,
9241 }
9242 }
9243
9244 #[test]
9245 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
9246 let mut state = run_state(RunStatus::Blocked);
9247 state.gate = vec![
9248 CommandOutcome {
9249 command: "cargo make check".to_owned(),
9250 code: Some(0),
9251 output_tail: "ok".to_owned(),
9252 duration_ms: 0,
9253 resource_blocked: false,
9254 },
9255 CommandOutcome {
9256 command: "cargo test".to_owned(),
9257 code: Some(101),
9258 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
9259 duration_ms: 0,
9260 resource_blocked: false,
9261 },
9262 ];
9263 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
9264 assert!(d.contains("cargo test"), "{d}");
9265 assert!(
9266 !d.contains("cargo make check"),
9267 "a passing check is not a diagnostic: {d}"
9268 );
9269 assert!(d.contains("assertion failed"), "{d}");
9270 }
9271
9272 #[test]
9273 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
9274 let mut state = run_state(RunStatus::Blocked);
9275 state.event(
9276 "land",
9277 "stopped: the fixer produced no commit while 2 check(s) were failing \
9278 (build, lint); stopping instead of looping on an unchanged tree",
9279 );
9280 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
9281 assert!(d.contains("build"), "{d}");
9282 assert!(d.contains("lint"), "{d}");
9283 assert!(d.contains("fixer produced no commit"), "{d}");
9284 }
9285
9286 #[test]
9287 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
9288 let state = run_state(RunStatus::VerifiedNoop);
9293 let d = describe(&state);
9294 assert!(
9295 d.contains("agent-verified no-op"),
9296 "expected the display label, not the wire spelling: {d}"
9297 );
9298 assert!(!d.contains("verified_noop"), "{d}");
9299 }
9300
9301 #[test]
9302 fn diagnostic_keeps_gate_panic_before_large_backtrace() {
9303 let mut state = run_state(RunStatus::Failed);
9304 state.gate.push(crate::run::CommandOutcome {
9305 command: "cargo test".into(),
9306 code: Some(101),
9307 output_tail: format!(
9308 "---- widget stdout ----\nwidget log\nthread 'widget' panicked at widget.rs:12:\nassertion failed: widget missing\n{}test result: FAILED\n",
9309 " 41: 0x1234 - backtrace frame\n".repeat(10000)
9310 ),
9311 duration_ms: 1,
9312 resource_blocked: false,
9313 });
9314 let d = diagnostic(&state).unwrap();
9315 assert!(d.contains("panicked at widget.rs:12"), "{d}");
9316 assert!(d.contains("assertion failed: widget missing"));
9317 assert!(d.contains("widget log"));
9318 }
9319
9320 #[test]
9321 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
9322 let mut state = run_state(RunStatus::Failed);
9328 state.candidates = vec![candidate(
9329 'A',
9330 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
9331 true,
9332 None,
9333 )];
9334 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
9335 assert!(d.contains("candidate A"), "{d}");
9336 assert!(d.contains("tagged v1.2.3"), "{d}");
9337 }
9338
9339 #[test]
9340 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
9341 let mut state = run_state(RunStatus::Failed);
9342 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
9343 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
9344 assert!(d.contains("candidate A"), "{d}");
9345 assert!(d.contains("agent timed out"), "{d}");
9346 }
9347
9348 #[test]
9349 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
9350 let mut state = run_state(RunStatus::Failed);
9353 state.candidates = vec![candidate('A', "did the work", false, None)];
9354 assert!(diagnostic(&state).is_none());
9355 }
9356
9357 #[test]
9358 fn diagnostic_is_bounded_however_much_a_run_printed() {
9359 let mut state = run_state(RunStatus::Blocked);
9360 state.gate = vec![
9361 CommandOutcome {
9362 command: "cargo test".to_owned(),
9363 code: Some(101),
9364 output_tail: "x".repeat(50_000),
9365 duration_ms: 0,
9366 resource_blocked: false,
9367 },
9368 CommandOutcome {
9369 command: "cargo clippy".to_owned(),
9370 code: Some(1),
9371 output_tail: "y".repeat(50_000),
9372 duration_ms: 0,
9373 resource_blocked: false,
9374 },
9375 ];
9376 state.candidates = vec![
9377 candidate('A', &"z".repeat(50_000), true, None),
9378 candidate('B', &"w".repeat(50_000), true, None),
9379 ];
9380 let d = diagnostic(&state).expect("plenty here to diagnose");
9381 assert!(
9382 d.len() <= DIAGNOSTIC_MAX,
9383 "diagnostic grew to {} bytes, unbounded",
9384 d.len()
9385 );
9386 }
9387
9388 #[test]
9389 fn selected_delivery_guards_survive_recovery_failure_accounting() {
9390 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
9391 let mut t = task();
9392 let mut state = run_state(status);
9393 t.pr_recovery = Some(crate::pr_recovery::Selection {
9394 url: "url".into(),
9395 branch: format!("magi/task-{}", t.id),
9396 head: "a".repeat(40),
9397 source_head: None,
9398 source_run: None,
9399 source_proof: None,
9400 disposition: Some(crate::pr_recovery::Disposition::Land),
9401 instruction: t.instruction.clone(),
9402 approved_at: Timestamp::now(),
9403 });
9404 state.pr_recovery = t.pr_recovery.clone();
9405 t.start(state.id.clone());
9406 settle_and_diagnose(
9407 &mut t,
9408 Verdict {
9409 status,
9410 left_pr: false,
9411 quota_hit: false,
9412 parked: false,
9413 no_viable_candidates: false,
9414 },
9415 "delivery stopped",
9416 5,
9417 &state,
9418 );
9419 assert_eq!(t.status, TaskStatus::Held);
9420 let reason = t.hold_reason.as_deref().unwrap();
9421 if status == RunStatus::Ready {
9422 assert!(reason.contains("Selected PR delivery is incomplete"));
9423 assert!(t.recovery_budget.is_none());
9424 } else {
9425 assert!(reason.contains("Selected recovery stopped"));
9426 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
9427 }
9428 assert!(t.pr_recovery.is_some());
9429 }
9430 }
9431
9432 #[test]
9433 fn base_recovery_settlement_holds_without_replacement_or_duplicate_notice_cause() {
9434 let mut state = run_state(RunStatus::Voting);
9435 state.base_sync = Some(crate::run::BaseSync {
9436 tip: "base-tip".into(),
9437 behind: 3,
9438 attempts: 4,
9439 conflict: Some("base stopped".into()),
9440 already_in: None,
9441 });
9442 state.record_base_sync_stop("candidate-head");
9443 let mut t = task();
9444 t.start(state.id.clone());
9445 let verdict = Verdict {
9446 status: RunStatus::Blocked,
9447 left_pr: false,
9448 quota_hit: false,
9449 parked: false,
9450 no_viable_candidates: false,
9451 };
9452 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
9453 assert_eq!(
9454 t.status,
9455 TaskStatus::Held,
9456 "unspent task attempts cannot start another competition"
9457 );
9458 let at = t.held_at;
9459 let cause = t.hold_notice_cause.clone();
9460 state.begin_pass();
9461 t.release_operator("test-resume", None);
9462 t.start(state.id.clone());
9463 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
9464 assert_eq!(t.status, TaskStatus::Held);
9465 assert_eq!(t.held_at, at);
9466 assert_eq!(t.hold_notice_cause, cause);
9467 }
9468
9469 #[test]
9470 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
9471 let mut state = run_state(RunStatus::Blocked);
9472 state.gate = vec![CommandOutcome {
9473 command: "cargo test".to_owned(),
9474 code: Some(101),
9475 output_tail: "assertion failed".to_owned(),
9476 duration_ms: 0,
9477 resource_blocked: false,
9478 }];
9479 let verdict = Verdict {
9480 status: RunStatus::Blocked,
9481 left_pr: false,
9482 quota_hit: false,
9483 parked: false,
9484 no_viable_candidates: false,
9485 };
9486
9487 let mut t = task();
9490 t.start("run-1".to_owned());
9491 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
9492 assert_eq!(t.status, TaskStatus::Failed);
9493 assert!(t.diagnostic.is_none());
9494
9495 t.start("run-2".to_owned());
9498 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
9499 assert_eq!(t.status, TaskStatus::Held);
9500 let d = t.diagnostic.expect("a held task must carry its diagnostic");
9501 assert!(d.contains("cargo test"), "{d}");
9502 }
9503
9504 #[test]
9505 fn a_held_task_names_the_open_question_it_is_waiting_on() {
9506 crate::run::pin_test_home();
9511 let home = crate::run::home();
9512 let state = run_state(RunStatus::VerifiedNoop);
9513 let mut q = ask::Question::new(
9514 state.id.clone(),
9515 "implement".to_owned(),
9516 "impl-A".to_owned(),
9517 "is this really a no-op?".to_owned(),
9518 String::new(),
9519 Vec::new(),
9520 );
9521 Questions::at(home.join("questions")).put(&mut q).unwrap();
9522
9523 let verdict = Verdict {
9524 status: RunStatus::VerifiedNoop,
9525 left_pr: false,
9526 quota_hit: false,
9527 parked: false,
9528 no_viable_candidates: false,
9529 };
9530 let mut t = task();
9531 t.start(state.id.clone());
9532 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
9533
9534 assert_eq!(t.status, TaskStatus::Held);
9535 let reason = t.hold_reason.expect("a held task must record why");
9536 assert!(
9537 reason.starts_with("run ended agent-verified no-op"),
9538 "the original settle reason must survive unchanged: {reason}"
9539 );
9540 assert!(
9541 reason.contains(q.short()),
9542 "the open question's id must be named so the notice is actionable: {reason}"
9543 );
9544 }
9545
9546 #[test]
9547 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
9548 crate::run::pin_test_home();
9549 let state = run_state(RunStatus::VerifiedNoop);
9550
9551 let verdict = Verdict {
9552 status: RunStatus::VerifiedNoop,
9553 left_pr: false,
9554 quota_hit: false,
9555 parked: false,
9556 no_viable_candidates: false,
9557 };
9558 let mut t = task();
9559 t.start(state.id.clone());
9560 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
9561
9562 assert_eq!(t.status, TaskStatus::Held);
9563 assert_eq!(
9564 t.hold_reason.as_deref(),
9565 Some("run ended agent-verified no-op"),
9566 "nothing to append when the question was already answered or never asked"
9567 );
9568 }
9569
9570 #[test]
9571 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
9572 crate::run::pin_test_home();
9573 let mut first = run_state(RunStatus::Blocked);
9574 first.id = "20260101-000000-sup1".to_owned();
9575 first.save().unwrap();
9576 let mut second = run_state(RunStatus::Merged);
9577 second.id = "20260101-000000-sup2".to_owned();
9578 second.save().unwrap();
9579
9580 let mut t = task();
9581 t.runs = vec![first.id.clone(), second.id.clone()];
9582 t.status = TaskStatus::Done;
9583
9584 supersede_prior_runs(&t, &crate::run::home());
9585
9586 assert_eq!(
9587 RunState::load(&first.id).unwrap().status,
9588 RunStatus::Superseded,
9589 "the first attempt's Blocked no longer needs anyone's attention"
9590 );
9591 assert_eq!(
9592 RunState::load(&second.id).unwrap().status,
9593 RunStatus::Merged,
9594 "the run that actually succeeded is left exactly as it was"
9595 );
9596 }
9597
9598 #[test]
9599 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
9600 crate::run::pin_test_home();
9606 let mut first = run_state(RunStatus::Blocked);
9607 first.id = "20260101-000000-sup9".to_owned();
9608 first.driver_pid = Some(std::process::id());
9611 first.driver_started_at = Some(
9612 crate::proc::process_started_at(std::process::id())
9613 .expect("this test process's own start time must be queryable"),
9614 );
9615 first.save().unwrap();
9616 let mut second = run_state(RunStatus::Merged);
9617 second.id = "20260101-000000-supa".to_owned();
9618 second.save().unwrap();
9619
9620 let mut t = task();
9621 t.runs = vec![first.id.clone(), second.id.clone()];
9622 t.status = TaskStatus::Done;
9623
9624 supersede_prior_runs(&t, &crate::run::home());
9625
9626 assert_eq!(
9627 RunState::load(&first.id).unwrap().status,
9628 RunStatus::Blocked,
9629 "a live driver_pid means something is still actually working this run, \
9630 even though no daemon claims it - rewriting under it would just be \
9631 undone the next time that process saves"
9632 );
9633 }
9634
9635 #[test]
9636 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
9637 let dir = tempfile::tempdir().unwrap();
9643 let home = dir.path().to_path_buf();
9644 let queue = Queue::at(dir.path().join("queue"));
9645
9646 let mut first = run_state(RunStatus::Blocked);
9647 first.id = "20260101-000000-supd".to_owned();
9648 first.driver_pid = Some(std::process::id());
9649 first.driver_started_at = Some(
9650 crate::proc::process_started_at(std::process::id())
9651 .expect("this test process's own start time must be queryable"),
9652 );
9653 first.save_under(&home).unwrap();
9654 let mut second = run_state(RunStatus::Merged);
9655 second.id = "20260101-000000-supe".to_owned();
9656 second.save_under(&home).unwrap();
9657
9658 let mut t = task();
9659 t.runs = vec![first.id.clone(), second.id.clone()];
9660 t.status = TaskStatus::Done;
9661 queue.put(&mut t).unwrap();
9662
9663 resweep_superseded_attempts(&queue, &home);
9664 assert_eq!(
9665 RunState::load_under(&first.id, &home).unwrap().status,
9666 RunStatus::Blocked,
9667 "still live on the first pass, so still untouched"
9668 );
9669
9670 let mut stale = RunState::load_under(&first.id, &home).unwrap();
9676 stale.driver_started_at = Some("1".to_owned());
9677 stale.save_under(&home).unwrap();
9678
9679 resweep_superseded_attempts(&queue, &home);
9680 assert_eq!(
9681 RunState::load_under(&first.id, &home).unwrap().status,
9682 RunStatus::Superseded,
9683 "the second pass catches up what the first one correctly skipped"
9684 );
9685 }
9686
9687 #[test]
9688 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
9689 crate::run::pin_test_home();
9690 let mut first = run_state(RunStatus::Blocked);
9691 first.id = "20260101-000000-sup3".to_owned();
9692 first.save().unwrap();
9693 let mut second = run_state(RunStatus::Blocked);
9694 second.id = "20260101-000000-sup4".to_owned();
9695 second.save().unwrap();
9696
9697 let mut t = task();
9698 t.runs = vec![first.id.clone(), second.id.clone()];
9699 t.status = TaskStatus::Failed;
9703
9704 supersede_prior_runs(&t, &crate::run::home());
9705
9706 assert_eq!(
9707 RunState::load(&first.id).unwrap().status,
9708 RunStatus::Blocked
9709 );
9710 assert_eq!(
9711 RunState::load(&second.id).unwrap().status,
9712 RunStatus::Blocked
9713 );
9714 }
9715
9716 #[test]
9717 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
9718 crate::run::pin_test_home();
9722 let mut first = run_state(RunStatus::Blocked);
9723 first.id = "20260101-000000-sup5".to_owned();
9724 first.save().unwrap();
9725
9726 let mut t = task();
9727 t.runs = vec![first.id.clone()];
9728 t.status = TaskStatus::Done;
9729
9730 supersede_prior_runs(&t, &crate::run::home());
9731
9732 assert_eq!(
9733 RunState::load(&first.id).unwrap().status,
9734 RunStatus::Blocked,
9735 "a single-attempt task has no earlier run to supersede"
9736 );
9737 }
9738
9739 #[test]
9740 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
9741 crate::run::pin_test_home();
9748 let mut first = run_state(RunStatus::Blocked);
9749 first.id = "20260101-000000-supb".to_owned();
9750 first.save().unwrap();
9751 let mut second = run_state(RunStatus::Failed);
9752 second.id = "20260101-000000-supc".to_owned();
9753 second.save().unwrap();
9754
9755 let mut t = task();
9756 t.runs = vec![first.id.clone(), second.id.clone()];
9757 t.status = TaskStatus::Done;
9758
9759 supersede_prior_runs(&t, &crate::run::home());
9760
9761 assert_eq!(
9762 RunState::load(&first.id).unwrap().status,
9763 RunStatus::Blocked,
9764 "the task's last attempt never landed, so there is nothing here \
9765 actually superseding it"
9766 );
9767 }
9768
9769 #[test]
9770 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
9771 crate::run::pin_test_home();
9775 let mut failed = run_state(RunStatus::Failed);
9776 failed.id = "20260101-000000-sup6".to_owned();
9777 failed.save().unwrap();
9778 let mut noop = run_state(RunStatus::VerifiedNoop);
9779 noop.id = "20260101-000000-sup7".to_owned();
9780 noop.save().unwrap();
9781 let mut winner = run_state(RunStatus::Ready);
9782 winner.id = "20260101-000000-sup8".to_owned();
9783 winner.save().unwrap();
9784
9785 let mut t = task();
9786 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
9787 t.status = TaskStatus::Done;
9788
9789 supersede_prior_runs(&t, &crate::run::home());
9790
9791 assert_eq!(
9792 RunState::load(&failed.id).unwrap().status,
9793 RunStatus::Failed
9794 );
9795 assert_eq!(
9796 RunState::load(&noop.id).unwrap().status,
9797 RunStatus::VerifiedNoop
9798 );
9799 }
9800
9801 fn approval_question(run: &str) -> ask::Question {
9802 ask::Question::new(
9803 run.to_owned(),
9804 land::APPROVAL_NODE.to_owned(),
9805 "land".to_owned(),
9806 "merge?".to_owned(),
9807 String::new(),
9808 vec!["merge".to_owned(), "hold".to_owned()],
9809 )
9810 }
9811
9812 #[test]
9813 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
9814 crate::run::pin_test_home();
9815 let mut state = run_state(RunStatus::Landing);
9816 state.id = "20260101-000000-fre1".to_owned();
9817 state.parked = true;
9818 state.save().unwrap();
9819 ask::Questions::open()
9820 .put(&mut approval_question(&state.id))
9821 .unwrap();
9822
9823 let mut t = task();
9824 t.runs.push(state.id.clone());
9825 assert_eq!(
9826 land_resume_state(&t),
9827 LandResume::StillWaiting,
9828 "nobody has answered and the timeout has not passed"
9829 );
9830 }
9831
9832 #[test]
9833 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
9834 crate::run::pin_test_home();
9835 let mut state = run_state(RunStatus::Gating);
9836 state.id = "20260101-000000-gt01".to_owned();
9837 state.parked = true;
9838 state.config.graph.answer_timeout = 60;
9839 state.github_text = Some(crate::run::GithubTextAsk {
9840 rewrite: None,
9841 fingerprint: "f".into(),
9842 title: true,
9843 body: false,
9844 categories: vec!["title-language".into()],
9845 question: None,
9846 asks: 1,
9847 resolved: false,
9848 chosen_title: None,
9849 choice: None,
9850 publication: None,
9851 });
9852 state.save().unwrap();
9853 let store = ask::Questions::open();
9854 let mut q = ask::Question::new(
9855 state.id.clone(),
9856 crate::github_text::ASK_NODE.to_owned(),
9857 crate::github_text::ASK_SEAT.to_owned(),
9858 "s".into(),
9859 String::new(),
9860 vec!["use fallback".into()],
9861 );
9862 store.put(&mut q).unwrap();
9863 let mut t = task();
9864 t.runs.push(state.id.clone());
9865 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
9866 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9867 store.put(&mut q).unwrap();
9868 assert_eq!(land_resume_state(&t), LandResume::Ready);
9869 assert!(!store.get(&q.id).unwrap().status.open());
9870 state.github_text.as_mut().unwrap().resolved = true;
9872 state.save().unwrap();
9873 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
9874 }
9875
9876 #[test]
9877 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
9878 crate::run::pin_test_home();
9883 let mut state = run_state(RunStatus::Landing);
9884 state.id = "20260101-000000-exp1".to_owned();
9885 state.parked = true;
9886 state.config.graph.answer_timeout = 60;
9887 state.save().unwrap();
9888
9889 let store = ask::Questions::open();
9890 let mut q = approval_question(&state.id);
9891 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9892 store.put(&mut q).unwrap();
9893
9894 let mut t = task();
9895 t.runs.push(state.id.clone());
9896 assert_eq!(
9897 land_resume_state(&t),
9898 LandResume::Ready,
9899 "an expired question must not be waited on forever"
9900 );
9901
9902 let after = store.get(&q.id).unwrap();
9903 assert!(
9904 !after.status.open(),
9905 "the question is abandoned, not silently ignored"
9906 );
9907 assert!(
9908 after.resolution().is_none(),
9909 "an abandoned question is not read as a decision"
9910 );
9911 }
9912
9913 #[test]
9914 fn reclaim_settles_a_running_task_against_its_last_run() {
9915 let mut t = task();
9916 t.start("20260904-000000-4043".to_owned());
9917 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
9918 assert_eq!(
9919 t.status,
9920 TaskStatus::Done,
9921 "a run that actually finished must not stay `running` forever"
9922 );
9923 }
9924
9925 #[test]
9926 fn reclaim_preserves_dispatched_inquiries_until_a_committed_pass() {
9927 let dir = tempfile::tempdir().unwrap();
9928 let queue = Queue::at(dir.path().join("queue"));
9929 let mut task = task();
9930 let mut run = run_state(RunStatus::Blocked);
9931 run.reserve_under(dir.path()).unwrap();
9932 run.driver_exited = true;
9933 run.save_under(dir.path()).unwrap();
9934 queue.put(&mut task).unwrap();
9935 let stale = task.clone();
9936 task.resume_override = Some(crate::queue::OperatorResume {
9937 question_id: "pending-resume".into(),
9938 at: Timestamp::now(),
9939 conductor_rehold: None,
9940 forced: true,
9941 pinned_run: Some(run.id.clone()),
9942 });
9943 task.start(run.id.clone());
9944 queue.put(&mut task).unwrap();
9945 let mut recovered = queue.get(&task.id).unwrap();
9947 let disk = RunState::load_under(&run.id, dir.path()).unwrap();
9948 reclaim(&mut recovered, Some(disk), 2, "en");
9949 queue.put(&mut recovered).unwrap();
9950 assert!(recovered.resume_override.is_none());
9951 assert_eq!(
9952 recovered.dispatched_resumes.get("pending-resume"),
9953 Some(&run.id)
9954 );
9955 let mut stale = stale;
9956 queue.put(&mut stale).unwrap();
9957 assert_eq!(
9958 stale.dispatched_resumes, recovered.dispatched_resumes,
9959 "a stale task writer cannot erase dispatch ownership"
9960 );
9961 let mut resumed = RunState::load_under(&run.id, dir.path()).unwrap();
9962 assert!(resumed.decision_starts.is_empty());
9963 resumed.carry_dispatched_resumes(&queue.get(&task.id).unwrap());
9964 assert!(
9965 resumed.decision_starts.is_empty(),
9966 "handoff is not execution proof"
9967 );
9968 resumed.begin_pass();
9969 resumed.save_under(dir.path()).unwrap();
9970 let mut disk = RunState::load_under(&run.id, dir.path()).unwrap();
9971 assert_eq!(disk.decision_starts.get("pending-resume"), Some(&0));
9972 disk.carry_dispatched_resumes(&recovered);
9973 disk.begin_pass();
9974 assert_eq!(
9975 disk.decision_starts.get("pending-resume"),
9976 Some(&0),
9977 "later execution must preserve the original pass receipt"
9978 );
9979 }
9980
9981 #[test]
9982 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
9983 let mut t = task();
9984 t.start("20260904-000000-4043".to_owned());
9985 t.last_error = Some("earlier trouble".to_owned());
9986 let mut state = run_state(RunStatus::Judging);
9987 state.parked = true;
9988 reclaim(&mut t, Some(state), 1, "en");
9989 assert_eq!(t.status, TaskStatus::Parked);
9990 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
9991 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9992 assert!(t.park_reason.is_some());
9993 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
9994 assert!(!t.fresh_start);
9995 assert!(t.status.runnable());
9996 }
9997
9998 #[test]
9999 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
10000 let mut t = task();
10001 t.start("20260904-000000-4043".to_owned());
10002 t.last_error = Some("earlier trouble".to_owned());
10003 settle(
10004 &mut t,
10005 Verdict {
10006 status: RunStatus::Judging,
10007 left_pr: false,
10008 quota_hit: false,
10009 parked: true,
10010 no_viable_candidates: false,
10011 },
10012 "parked after `judging`",
10013 1,
10014 );
10015 assert_eq!(t.status, TaskStatus::Parked);
10016 assert_eq!(t.attempts, 0);
10017 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
10018 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
10019 assert_eq!(t.schema, crate::queue::SCHEMA);
10020 assert!(
10021 crate::notices::task_held(&t).is_none(),
10022 "a park is not news"
10023 );
10024 t.start("20260904-000000-4043".to_owned());
10025 assert_eq!(t.park_reason, None);
10026 }
10027
10028 #[test]
10029 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
10030 let mut t = task();
10034 t.start("20260904-000000-4043".to_owned());
10035 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
10036 assert_eq!(t.status, TaskStatus::Failed);
10037 assert!(t.status.runnable());
10038 }
10039
10040 #[test]
10041 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
10042 let mut t = task();
10043 t.start("20260904-000000-4043".to_owned());
10044 assert_eq!(t.attempts, 1);
10045 let mut run = run_state(RunStatus::Implementing);
10046 run.id = "20260904-000000-4043".to_owned();
10047 run.begin_pass();
10048 let cause = crate::run::ParkCause::Upgrade {
10049 from: "0.121.0".to_owned(),
10050 to: Some("0.121.1".to_owned()),
10051 };
10052 run.end_pass_parked(cause.clone());
10053 run.parked = true;
10054 let verdict = Verdict {
10055 status: run.status,
10056 left_pr: false,
10057 quota_hit: false,
10058 parked: true,
10059 no_viable_candidates: false,
10060 };
10061 let mut orphan = t.clone();
10064 reclaim(&mut orphan, Some(run.clone()), 2, "en");
10065 assert_eq!(orphan.status, TaskStatus::Parked);
10066 assert_eq!(orphan.attempts, 0);
10067 assert_eq!(
10068 orphan.upgrade_resume_key(),
10069 Some((run.passes[0].started_at, orphan.id.as_str()))
10070 );
10071 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
10072 assert_eq!(t.attempts, 0);
10073 assert_eq!(t.passes.len(), 1);
10074 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
10075 assert_eq!(t.passes[0].parked, Some(cause));
10076 assert_eq!(t.passes[0].pass_index, 0);
10077 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
10078 assert!(t.upgrade_resume_key().is_some());
10079
10080 t.start("20260904-000000-4043".to_owned());
10083 run.begin_pass();
10084 run.status = RunStatus::Failed;
10085 let verdict = Verdict {
10086 status: RunStatus::Failed,
10087 left_pr: false,
10088 quota_hit: false,
10089 parked: false,
10090 no_viable_candidates: false,
10091 };
10092 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
10093 assert_eq!(t.passes.len(), 2);
10094 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
10095 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
10096 assert_eq!(t.passes[1].pass_index, 1);
10097 assert_eq!(t.attempts, 1);
10098 }
10099
10100 #[test]
10101 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
10102 let mut t = task();
10103 t.start("20260904-000000-4043".to_owned());
10104 reclaim(&mut t, None, 2, "en");
10105 assert_eq!(t.status, TaskStatus::Held);
10106 assert!(
10107 t.last_error
10108 .as_deref()
10109 .is_some_and(|e| e.contains("running")),
10110 "the operator needs to know why this task was held"
10111 );
10112 }
10113
10114 #[test]
10115 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
10116 let dir = tempfile::tempdir().unwrap();
10117 let queue = Queue::at(dir.path().to_path_buf());
10118
10119 let mut orphaned = task();
10121 orphaned.id = "20260904-000000-orph".to_owned();
10122 orphaned.status = TaskStatus::Running;
10123 orphaned.attempts = 1;
10124 queue.put(&mut orphaned).unwrap();
10125
10126 let mut alive = task();
10127 alive.id = "20260904-000000-live".to_owned();
10128 alive.status = TaskStatus::Running;
10129 alive.attempts = 1;
10130 queue.put(&mut alive).unwrap();
10131 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
10132
10133 let mut queued = task();
10134 queued.id = "20260904-000000-wait".to_owned();
10135 queue.put(&mut queued).unwrap();
10136
10137 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
10138 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
10139
10140 assert_eq!(
10141 queue.get(&orphaned.id).unwrap().status,
10142 TaskStatus::Held,
10143 "nothing was driving it and there was no run to recover"
10144 );
10145 assert_eq!(
10146 queue.get(&alive.id).unwrap().status,
10147 TaskStatus::Running,
10148 "a live claim must protect the task it belongs to"
10149 );
10150 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
10151 }
10152
10153 fn read_run_under(home: &Path, id: &str) -> RunState {
10159 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
10160 serde_json::from_str(&body).unwrap()
10161 }
10162
10163 #[test]
10164 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
10165 let dir = tempfile::tempdir().unwrap();
10166 let home = dir.path().to_path_buf();
10167 let now = Timestamp::now();
10168 let overrun_seat = || crate::run::ActiveSeat {
10169 node: "implement".to_owned(),
10170 started_at: now - jiff::SignedDuration::new(21_000, 0),
10171 timeout_secs: 3_600,
10172 attempt: 0,
10173 task: None,
10174 command: None,
10175 index: None,
10176 total: None,
10177 };
10178
10179 let mut dead = run_state(RunStatus::Implementing);
10180 dead.id = "20260101-000000-dead".to_owned();
10181 dead.active.insert("impl-A".to_owned(), overrun_seat());
10182 dead.driver_pid = Some(4242);
10185 dead.save_under(&home).unwrap();
10186
10187 let mut alive = run_state(RunStatus::Implementing);
10190 alive.id = "20260101-000000-aliv".to_owned();
10191 alive.active.insert("impl-A".to_owned(), overrun_seat());
10192 alive.save_under(&home).unwrap();
10193 let mut status = Status::new();
10194 status.current = vec![Current {
10195 worker: None,
10196 task: "20260101-000000-task".to_owned(),
10197 run: alive.id.clone(),
10198 }];
10199 write_status_to(&home.join("daemon.json"), &status).unwrap();
10200
10201 let questions = Questions::at(home.join("questions"));
10205 let mut q = ask::Question::new(
10206 dead.id.clone(),
10207 "implement".to_owned(),
10208 "impl-A".to_owned(),
10209 "Which storage backend?".to_owned(),
10210 String::new(),
10211 vec!["SQLite".to_owned(), "Redis".to_owned()],
10212 );
10213 questions.put(&mut q).unwrap();
10214
10215 let abandoned = reclaim_abandoned_runs_with(
10216 &home,
10217 now,
10218 |pid| if pid == 4242 { Some(false) } else { None },
10219 |_| panic!("a query answering Dead outright needs no identity corroboration"),
10220 );
10221 assert_eq!(abandoned, vec![dead.id.clone()]);
10222
10223 let reloaded = read_run_under(&home, &dead.id);
10224 assert_eq!(reloaded.status, RunStatus::Failed);
10225 assert!(reloaded.active.is_empty());
10226 assert!(
10227 !questions.get(&q.id).unwrap().status.open(),
10228 "the failed run's own open question must be settled in the same pass"
10229 );
10230
10231 let still_alive = read_run_under(&home, &alive.id);
10232 assert_eq!(
10233 still_alive.status,
10234 RunStatus::Implementing,
10235 "a live daemon's claim protects it"
10236 );
10237 assert!(!still_alive.active.is_empty());
10238 }
10239
10240 #[test]
10250 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
10251 let dir = tempfile::tempdir().unwrap();
10252 let home = dir.path().to_path_buf();
10253 let now = Timestamp::now();
10254
10255 let mut manual = run_state(RunStatus::Reviewing);
10256 manual.id = "20260101-000000-manl".to_owned();
10257 manual.active.insert(
10258 "review-1".to_owned(),
10259 crate::run::ActiveSeat {
10260 node: "review".to_owned(),
10261 started_at: now - jiff::SignedDuration::new(21_000, 0),
10262 timeout_secs: 3_600,
10263 attempt: 0,
10264 task: None,
10265 command: None,
10266 index: None,
10267 total: None,
10268 },
10269 );
10270 manual.driver_pid = Some(4242);
10274 manual.driver_started_at = Some("1790000000".to_owned());
10275 manual.save_under(&home).unwrap();
10276
10277 let abandoned = reclaim_abandoned_runs_with(
10278 &home,
10279 now,
10280 |pid| if pid == 4242 { Some(true) } else { None },
10281 |pid| {
10282 if pid == 4242 {
10283 Some("1790000000".to_owned())
10284 } else {
10285 None
10286 }
10287 },
10288 );
10289 assert!(
10290 abandoned.is_empty(),
10291 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
10292 );
10293
10294 let reloaded = read_run_under(&home, &manual.id);
10295 assert_eq!(reloaded.status, RunStatus::Reviewing);
10296 assert!(!reloaded.active.is_empty());
10297 }
10298
10299 #[test]
10300 fn an_already_claimed_task_is_skipped_rather_than_failed() {
10301 let dir = tempfile::tempdir().unwrap();
10302 let queue = Queue::at(dir.path().to_path_buf());
10303 let mut only = task();
10304 queue.put(&mut only).unwrap();
10305
10306 let _elsewhere = queue.claim(&only.id).unwrap();
10307 let candidates = runnable(&queue);
10308 assert_eq!(candidates.len(), 1, "the task is still runnable");
10309 assert!(
10310 queue.claim(&candidates[0].id).is_err(),
10311 "the loop cannot take a claim somebody else holds"
10312 );
10313
10314 let after = queue.get(&only.id).unwrap();
10315 assert_eq!(after.status, TaskStatus::Queued);
10316 assert_eq!(
10317 after.attempts, 0,
10318 "losing the race is not an attempt at the task"
10319 );
10320 assert_eq!(after.last_error, None);
10321 }
10322
10323 #[test]
10324 fn the_status_file_round_trips_and_its_heartbeat_advances() {
10325 let dir = tempfile::tempdir().unwrap();
10326 let path = dir.path().join("daemon.json");
10327
10328 let mut status = Status::new();
10329 status.idle = false;
10330 status.completed = 7;
10331 status.current = vec![Current {
10332 worker: None,
10333 task: "20260902-000000-t111".to_owned(),
10334 run: "20260902-000001-r111".to_owned(),
10335 }];
10336 write_status_to(&path, &status).unwrap();
10337 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
10338 assert_eq!(first.schema, SCHEMA);
10339 assert_eq!(first.pid, std::process::id());
10340 assert!(!first.idle);
10341 assert_eq!(first.completed, 7);
10342 assert_eq!(first.current, status.current);
10343 assert!(
10344 !path.with_extension("json.tmp").exists(),
10345 "the temp file is renamed, not left behind"
10346 );
10347
10348 std::thread::sleep(Duration::from_millis(5));
10349 status.updated_at = Timestamp::now();
10350 status.polls = 3;
10351 write_status_to(&path, &status).unwrap();
10352 let second: Status =
10353 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
10354 assert!(
10355 second.updated_at > first.updated_at,
10356 "a reader can only detect staleness if the heartbeat moves"
10357 );
10358 assert_eq!(
10359 second.started_at, first.started_at,
10360 "the start time is not a heartbeat"
10361 );
10362 assert_eq!(second.polls, 3);
10363 }
10364
10365 #[test]
10366 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
10367 let dir = tempfile::tempdir().unwrap();
10368
10369 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
10370
10371 let mut status = Status::new();
10372 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
10373 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10374 let stale = read_status(dir.path()).unwrap();
10375 assert!(
10376 !stale.running(Timestamp::now()),
10377 "a minute without a heartbeat is a dead daemon, not a busy one"
10378 );
10379 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
10380
10381 status.updated_at = Timestamp::now();
10382 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10383 let fresh = read_status(dir.path()).unwrap();
10384 assert!(fresh.running(Timestamp::now()));
10385 }
10386
10387 #[test]
10388 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
10389 let dir = tempfile::tempdir().unwrap();
10390 let now = Timestamp::now();
10391 let mine = "20260903-080619-01c2";
10392
10393 assert!(
10394 !is_working_on(dir.path(), mine, now),
10395 "no status file means nobody is working on anything"
10396 );
10397
10398 let mut status = Status::new();
10399 status.current = vec![Current {
10400 worker: None,
10401 task: "20260903-080340-0167".to_owned(),
10402 run: mine.to_owned(),
10403 }];
10404 status.updated_at = now;
10405 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10406 assert!(is_working_on(dir.path(), mine, now));
10407 assert!(
10408 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
10409 "a daemon busy with one run is not working on another"
10410 );
10411
10412 status.updated_at = now - jiff::SignedDuration::from_secs(600);
10415 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10416 assert!(
10417 !is_working_on(dir.path(), mine, now),
10418 "a stale heartbeat is a dead daemon, so its run is a leftover"
10419 );
10420 }
10421
10422 #[test]
10423 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
10424 let dir = tempfile::tempdir().unwrap();
10425 let now = Timestamp::now();
10426
10427 assert!(
10428 !is_working_on_short(dir.path(), "01c2", now),
10429 "no status file means nobody is working on anything"
10430 );
10431
10432 let mut status = Status::new();
10433 status.current = vec![Current {
10434 worker: None,
10435 task: "20260903-080340-0167".to_owned(),
10436 run: "20260903-080619-01c2".to_owned(),
10437 }];
10438 status.updated_at = now;
10439 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10440 assert!(
10441 is_working_on_short(dir.path(), "01c2", now),
10442 "the run's short id is the last block of its full id"
10443 );
10444 assert!(
10445 !is_working_on_short(dir.path(), "3cbf", now),
10446 "a daemon busy with one worktree bay is not working on another"
10447 );
10448 }
10449
10450 #[test]
10451 fn a_newer_status_file_still_yields_a_reading() {
10452 let dir = tempfile::tempdir().unwrap();
10453 std::fs::write(
10456 dir.path().join("daemon.json"),
10457 serde_json::json!({
10458 "schema": 2,
10459 "updated_at": Timestamp::now().to_string(),
10460 "idle": true,
10461 "surprise": { "nested": [1, 2, 3] },
10462 })
10463 .to_string(),
10464 )
10465 .unwrap();
10466
10467 let reading = read_status(dir.path()).expect("a forward-compatible read");
10468 assert!(reading.running(Timestamp::now()));
10469 assert!(reading.idle);
10470 assert!(reading.current.is_empty());
10471 }
10472
10473 #[test]
10474 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
10475 let dir = tempfile::tempdir().unwrap();
10481 std::fs::write(
10482 dir.path().join("daemon.json"),
10483 serde_json::json!({
10484 "schema": 1,
10485 "pid": 4242,
10486 "updated_at": Timestamp::now().to_string(),
10487 "idle": false,
10488 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
10489 "completed": 3,
10490 "polls": 9,
10491 })
10492 .to_string(),
10493 )
10494 .unwrap();
10495
10496 let reading = read_status(dir.path()).expect("an older shape must still parse");
10497 assert!(reading.running(Timestamp::now()));
10498 assert_eq!(
10499 reading.current,
10500 vec![Current {
10501 worker: None,
10502 task: "20260902-140501-aaaa".to_owned(),
10503 run: "20260902-140502-bbbb".to_owned(),
10504 }]
10505 );
10506 }
10507
10508 #[test]
10509 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
10510 let dir = tempfile::tempdir().unwrap();
10511 std::fs::write(
10512 dir.path().join("daemon.json"),
10513 serde_json::json!({
10514 "schema": 1,
10515 "updated_at": Timestamp::now().to_string(),
10516 "idle": true,
10517 "current": null,
10518 })
10519 .to_string(),
10520 )
10521 .unwrap();
10522 let with_null = read_status(dir.path()).expect("null must still parse");
10523 assert!(with_null.current.is_empty());
10524
10525 std::fs::write(
10526 dir.path().join("daemon.json"),
10527 serde_json::json!({
10528 "schema": 1,
10529 "updated_at": Timestamp::now().to_string(),
10530 "idle": true,
10531 })
10532 .to_string(),
10533 )
10534 .unwrap();
10535 let absent = read_status(dir.path()).expect("a missing field must still parse");
10536 assert!(absent.current.is_empty());
10537 }
10538
10539 #[test]
10540 fn a_task_without_a_repository_runs_in_the_daemons_default() {
10541 let fallback = Path::new("/default");
10542 let mut blank = task();
10543 blank.repo = PathBuf::new();
10544 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
10545 let mut dot = task();
10546 dot.repo = PathBuf::from(".");
10547 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
10548 assert_eq!(
10549 repo_for(&task(), fallback),
10550 PathBuf::from("/repo"),
10551 "a task that names a repository keeps it"
10552 );
10553 }
10554
10555 #[test]
10556 fn new_tasks_inherit_the_daemon_build_cache_but_historical_tasks_remain_shared() {
10557 let dir = tempfile::tempdir().unwrap();
10558 let repo = dir.path();
10559 let path = repo.join("explicit.toml");
10560 std::fs::write(
10561 &path,
10562 "[verify]\nbuild_cache = 'isolated'\n[graph]\njudges = 1\n",
10563 )
10564 .unwrap();
10565 let opts = Opts {
10566 config: Some(path),
10567 ..Opts::default()
10568 };
10569 let queue = Queue::at(repo.join("queue"));
10570 let mut new = task();
10571 queue.put(&mut new).unwrap();
10572 let resolved = |task: &Task| prepare_for(repo, &opts, task).unwrap();
10573 assert_eq!(
10574 resolved(&queue.get(&new.id).unwrap()).verify.build_cache,
10575 BuildCache::Isolated
10576 );
10577
10578 let mut old = task();
10579 old.schema = 22;
10580 queue.put(&mut old).unwrap();
10581 let mut old = queue.get(&old.id).unwrap();
10582 let config = resolved(&old);
10583 assert_eq!(config.verify.build_cache, BuildCache::Shared);
10584 assert_eq!(config.graph.judges, 1, "preserve daemon config inheritance");
10585 old.hold_manual(Some("wait".into()));
10586 queue.put(&mut old).unwrap();
10587
10588 let mut saved = queue.get(&old.id).unwrap();
10589 assert_eq!(resolved(&saved).verify.build_cache, BuildCache::Shared);
10590 saved
10591 .overrides
10592 .as_mut()
10593 .unwrap()
10594 .merge_over(&crate::queue::RunOverrides {
10595 build_cache: Some(BuildCache::Isolated),
10596 ..Default::default()
10597 });
10598 queue.put(&mut saved).unwrap();
10599 assert_eq!(
10600 resolved(&queue.get(&saved.id).unwrap()).verify.build_cache,
10601 BuildCache::Isolated
10602 );
10603 }
10604
10605 #[test]
10606 fn urgent_build_cache_precedes_config_but_not_an_explicit_choice() {
10607 let dir = tempfile::tempdir().unwrap();
10608 let path = dir.path().join("explicit.toml");
10609 let write = |verify: &str| {
10610 std::fs::write(&path, format!("[verify]\nbuild_cache = 'shared'\n{verify}")).unwrap();
10611 };
10612 write("e2e = [\"CARGO_TARGET_DIR=/tmp/shared-target cargo test\"]\n");
10613 let opts = Opts {
10614 config: Some(path.clone()),
10615 ..Opts::default()
10616 };
10617 let mut task = task();
10618 task.set_urgent(true).unwrap();
10619 let resolved = |task: &Task| {
10620 prepare_for(dir.path(), &opts, task)
10621 .unwrap()
10622 .verify
10623 .build_cache
10624 };
10625 assert_eq!(resolved(&task), BuildCache::Isolated);
10626 task.change_build_cache(Some(BuildCache::Shared)).unwrap();
10627 assert_eq!(resolved(&task), BuildCache::Shared);
10628 task.change_build_cache(None).unwrap();
10629 assert_eq!(resolved(&task), BuildCache::Isolated);
10630 task.set_urgent(false).unwrap();
10631 assert_eq!(resolved(&task), BuildCache::Shared);
10632
10633 task.set_urgent(true).unwrap();
10637 write("e2e = [\"npm test\"]\n");
10638 assert_eq!(resolved(&task), BuildCache::Shared);
10639 write("e2e = [\"CARGO_TARGET_DIR=/tmp/shared-target cargo test | tee log\"]\n");
10640 assert_eq!(resolved(&task), BuildCache::Shared);
10641 task.change_build_cache(Some(BuildCache::Isolated)).unwrap();
10643 assert_eq!(resolved(&task), BuildCache::Isolated);
10644 }
10645
10646 #[test]
10647 fn task_implementers_edit_inherits_daemon_config_and_merge() {
10648 let dir = tempfile::tempdir().unwrap();
10649 let path = dir.path().join("explicit.toml");
10650 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
10651 let opts = Opts {
10652 config: Some(path),
10653 merge: Some("none".into()),
10654 ..Default::default()
10655 };
10656 let mut t = task();
10657 t.set_implementers(
10658 2,
10659 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10660 .unwrap(),
10661 )
10662 .unwrap();
10663 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
10664 assert_eq!(cfg.graph.implementers, 2);
10665 assert_eq!(cfg.graph.judges, 1);
10666 assert_eq!(cfg.merge.mode, MergeMode::None);
10667 t.set_implementers(1, &cfg).unwrap();
10668 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
10669 apply_task_implementers(&mut cfg, &t);
10670 assert_eq!(cfg.graph.implementers, 1);
10671 assert_eq!(cfg.graph.judges, 1);
10672 assert_eq!(cfg.merge.mode, MergeMode::None);
10673 }
10674
10675 #[test]
10676 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
10677 let dir = tempfile::tempdir().unwrap();
10678 let repo = dir.path().join("repo");
10679 std::fs::create_dir(&repo).unwrap();
10680 std::fs::write(
10681 repo.join("magi.toml"),
10682 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
10683 )
10684 .unwrap();
10685 let daemon_config = dir.path().join("daemon.toml");
10686 std::fs::write(
10687 &daemon_config,
10688 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
10689 )
10690 .unwrap();
10691 let opts = Opts {
10692 config: Some(daemon_config),
10693 merge: Some("pr".into()),
10694 ..Default::default()
10695 };
10696 let q = Queue::at(dir.path().join("queue"));
10697 let validation: Config =
10698 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10699 .unwrap();
10700 for record in ["{}", r#"{"candidates":2}"#] {
10702 let overrides = serde_json::from_str(record).unwrap();
10703 let filing = crate::direct::Filing {
10704 instruction: "work".into(),
10705 title: "direct".into(),
10706 repo: repo.clone(),
10707 source: crate::queue::Source::Human,
10708 solo: false,
10709 overrides,
10710 review_of: None,
10711 sources: Default::default(),
10712 };
10713 let filed =
10714 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
10715 let crate::direct::Filed::Standalone {
10716 mut task,
10717 claim: _claim,
10718 } = filed
10719 else {
10720 panic!("local filing")
10721 };
10722 let before = prepare_for(&repo, &opts, &task).unwrap();
10723 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
10724 assert_eq!(
10725 before.merge.mode,
10726 MergeMode::None,
10727 "daemon --merge must not leak"
10728 );
10729 for n in [2, 1, 2] {
10730 task.set_implementers(n, &validation).unwrap();
10731 q.put(&mut task).unwrap();
10732 task = q.get(&task.id).unwrap();
10733 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
10734 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
10735 apply_task_implementers(&mut cfg, &task);
10736 assert_eq!(cfg.graph.implementers, usize::from(n));
10737 assert_eq!(cfg.graph.judges, 2);
10738 assert_eq!(cfg.merge.mode, MergeMode::None);
10739 }
10740 }
10741 }
10742
10743 #[test]
10744 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
10745 let cfg: Config =
10746 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10747 .unwrap();
10748 let mut t = task();
10749 t.review_of = Some("review-target".into());
10750 t.hold_manual(Some("retry".into()));
10751 let mut previous = RunState::new(
10752 PathBuf::from("/repo"),
10753 "main".into(),
10754 "abc1234".into(),
10755 "review".into(),
10756 cfg.clone(),
10757 );
10758 previous.config.graph.implementers = 2;
10759 for count in [1, 3] {
10760 t.set_implementers(count, &cfg).unwrap();
10761 t.release();
10762 assert_eq!(t.review_of.as_deref(), Some("review-target"));
10763 for proposed in [
10764 Starter::Resume(previous.id.clone()),
10765 Starter::Review("recovery-target".into()),
10766 ] {
10767 assert_eq!(
10768 starter_with_implementers(&t, Some(&previous), proposed),
10769 Starter::Review("review-target".into())
10770 );
10771 }
10772 let mut configured = cfg.clone();
10773 apply_task_implementers(&mut configured, &t);
10774 assert_eq!(configured.graph.implementers, usize::from(count));
10775 assert_eq!(previous.config.graph.implementers, 2);
10776 }
10777 }
10778
10779 #[test]
10780 fn task_implementers_dispatch_and_resume_agree() {
10781 let mut t = task();
10782 let cfg: crate::config::Config =
10783 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
10784 .unwrap();
10785 let mut run = RunState::new(
10786 PathBuf::from("/repo"),
10787 "main".into(),
10788 "abc1234".into(),
10789 "work".into(),
10790 cfg.clone(),
10791 );
10792 run.config.graph.implementers = 2;
10793 for n in [2, 1, 3, 2] {
10794 t.set_implementers(n, &cfg).unwrap();
10795 let mut dispatched = cfg.clone();
10796 apply_task_implementers(&mut dispatched, &t);
10797 assert_eq!(dispatched.graph.implementers, usize::from(n));
10798 assert_eq!(implementers_match(&t, &run), n == 2);
10799 for proposed in [
10800 Starter::Resume(run.id.clone()),
10801 Starter::Review("old-branch".into()),
10802 ] {
10803 let expected = if n == 2 {
10804 proposed.clone()
10805 } else {
10806 Starter::Start
10807 };
10808 assert_eq!(
10809 starter_with_implementers(&t, Some(&run), proposed),
10810 expected
10811 );
10812 }
10813 assert_eq!(
10814 run.config.graph.implementers, 2,
10815 "history must not be rewritten"
10816 );
10817 }
10818 t.solo = false;
10819 t.overrides = None;
10820 assert!(implementers_match(&t, &run));
10821 }
10822
10823 #[test]
10824 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
10825 let mut solo_cfg = Config::default();
10831 solo_cfg.graph.implementers = 3;
10832 let mut solo_task = task();
10833 solo_task.solo = true;
10834 apply_task_implementers(&mut solo_cfg, &solo_task);
10835 assert_eq!(solo_cfg.graph.implementers, 1);
10836
10837 let mut plain_cfg = Config::default();
10838 plain_cfg.graph.implementers = 3;
10839 let plain_task = task();
10840 assert!(!plain_task.solo);
10841 apply_task_implementers(&mut plain_cfg, &plain_task);
10842 assert_eq!(
10843 plain_cfg.graph.implementers, 3,
10844 "a task that did not ask to run alone keeps the config's candidates"
10845 );
10846 }
10847
10848 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
10849 QuotaLoss {
10850 cause: crate::agent::QuotaCause::UsageLimit,
10851 agent: String::new(),
10852 seat: seat.into(),
10853 node: "judge".into(),
10854 at: at.parse().unwrap(),
10855 reset: reset.map(str::to_string),
10856 }
10857 }
10858
10859 #[test]
10860 fn authentication_losses_never_arm_a_cooldown_even_with_reset_hints() {
10861 let now = Timestamp::now();
10862 for reset in [None, Some("in 1h")] {
10863 let auth = crate::agent::Quota {
10864 cause: crate::agent::QuotaCause::Authentication,
10865 agent: "alpha".into(),
10866 reset: reset.map(str::to_owned),
10867 };
10868 let loss = QuotaLoss::from_report("advisor-1".into(), "advise", Some(&auth));
10869 assert_eq!(cooldown_until(&[loss], now), None);
10870 }
10871 }
10872
10873 #[test]
10874 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
10875 let old: Vec<QuotaLoss> = (1..=4)
10876 .map(|i| {
10877 loss(
10878 &format!("judge-{i}"),
10879 "2026-09-23T05:23:00Z",
10880 Some("2:40pm (Asia/Tokyo)"),
10881 )
10882 })
10883 .collect();
10884 let fresh = losses_this_attempt(&old, &old);
10885 assert!(fresh.is_empty());
10886 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
10887 }
10889
10890 #[test]
10891 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
10892 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
10893 let now = Timestamp::now();
10894 let mut after = old.clone();
10895 after.push(loss("judge-2", &now.to_string(), None));
10896 let fresh = losses_this_attempt(&old, &after);
10897 assert_eq!(fresh, vec![after[1].clone()]);
10898 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
10899 assert_eq!(
10900 until,
10901 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
10902 );
10903 }
10904
10905 #[test]
10906 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
10907 let before = vec![
10910 loss("judge-1", "2026-09-23T05:23:00Z", None),
10911 loss("judge-2", "2026-09-23T05:24:00Z", None),
10912 ];
10913 let after = vec![
10914 loss("judge-2", "2026-09-23T05:24:00Z", None),
10915 loss("judge-1", "2026-09-24T01:00:00Z", None),
10916 ];
10917 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
10918 }
10919
10920 #[test]
10921 fn merge_overrides_are_parsed_or_refused() {
10922 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
10923 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
10924 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
10925 assert!(merge_mode("squash").is_err());
10926 }
10927
10928 #[test]
10929 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
10930 let now = Timestamp::now();
10931 let fallback = Duration::from_secs(300);
10932 let cap = Duration::from_secs(1800);
10933
10934 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
10936
10937 let soon = now + jiff::SignedDuration::from_secs(600);
10939 assert_eq!(
10940 quota_wait(Some(soon), now, fallback, cap),
10941 Duration::from_secs(600)
10942 );
10943
10944 let past = now - jiff::SignedDuration::from_secs(60);
10947 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
10948
10949 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
10952 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
10953 }
10954
10955 #[test]
10956 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
10957 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10958
10959 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
10960 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
10961
10962 let already_past =
10966 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
10967 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
10968
10969 assert!(
10970 parse_reset_hint("session limit reached", now, now).is_none(),
10971 "free text with no recognised shape is not guessed at"
10972 );
10973 assert!(
10974 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
10975 "an unresolvable zone name is not guessed at either"
10976 );
10977 }
10978
10979 #[test]
10980 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
10981 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10982
10983 let at = parse_reset_hint(
10984 "You've hit your usage limit. Visit \
10985 https://chatgpt.com/codex/settings/usage to purchase more \
10986 credits or try again at Sep 19th, 2026 5:10 PM.",
10987 now,
10988 now,
10989 )
10990 .expect("the codex reset wording is a recognised shape");
10991 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
10992
10993 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
10998 .expect("an explicit year needs no rollover");
10999 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
11000
11001 assert!(
11002 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
11003 "a two-digit year is not the documented shape and is not guessed at"
11004 );
11005 assert!(
11006 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
11007 "a four-letter month name is not the documented three-letter abbreviation"
11008 );
11009 assert!(
11010 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
11011 "an explicit zone on the dated shape is a format nobody has \
11012 documented, and is refused rather than guessed at as UTC"
11013 );
11014 }
11015
11016 #[test]
11017 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
11018 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
11019 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
11020
11021 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
11022 assert_eq!(at.as_second() - recorded.as_second(), 3769);
11023
11024 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
11025 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
11026
11027 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
11028 assert!(
11029 parse_reset_hint(bad, now, recorded).is_none(),
11030 "{bad:?} must not be guessed at"
11031 );
11032 }
11033 }
11034
11035 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
11039 let config = dir.join("magi.toml");
11040 std::fs::write(
11041 &config,
11042 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
11043 )
11044 .unwrap();
11045 let opts = Opts {
11046 poll: Duration::from_secs(30),
11047 config: Some(config),
11048 repo: dir.join("repo"),
11052 ..Opts::default()
11053 };
11054 let home = dir.join("home");
11063 let stamps = home.join("service-ticks");
11066 std::fs::create_dir_all(&stamps).unwrap();
11067 for service in ["waiter", "deputies", "release-watch"] {
11068 std::fs::write(
11069 stamps.join(service),
11070 Timestamp::now().as_second().to_string(),
11071 )
11072 .unwrap();
11073 }
11074 for operation in [
11075 "conductor",
11076 "maintenance",
11077 "housekeeping",
11078 "disk-pressure",
11079 "cache",
11080 "fetch",
11081 "chat-goals",
11082 ] {
11083 std::fs::write(
11084 stamps.join(format!("daemon-{operation}")),
11085 serde_json::to_vec(&Timestamp::now()).unwrap(),
11086 )
11087 .unwrap();
11088 }
11089 let worktrees = dir.join("wt");
11090 (
11091 opts,
11092 Queue::at(dir.join("queue")),
11093 home.join("daemon.json"),
11094 home,
11095 worktrees,
11096 )
11097 }
11098
11099 #[tokio::test]
11100 async fn disk_wait_maintenance_recovers_while_another_task_is_running() {
11101 let dir = tempfile::tempdir().unwrap();
11102 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
11103 let mut waiting = task();
11104 waiting.repo = opts.repo.clone();
11105 waiting.hold_machine(crate::disk::gate(1, 100));
11106 waiting.mark_disk_capacity();
11107 waiting.attempts = 2;
11108 queue.put(&mut waiting).unwrap();
11109 let mut busy = task();
11110 busy.status = TaskStatus::Running;
11111 queue.put(&mut busy).unwrap();
11112 let payload = serde_json::to_value(ServiceWorker {
11113 opts,
11114 queue: queue.root().to_owned(),
11115 worktrees_root: worktrees,
11116 operation: "maintenance".into(),
11117 })
11118 .unwrap();
11119 run_service_worker(&home, payload.clone()).await.unwrap();
11120 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Queued);
11121 assert_eq!(queue.get(&waiting.id).unwrap().attempts, 2);
11122 assert_eq!(queue.get(&busy.id).unwrap().status, TaskStatus::Running);
11123 run_service_worker(&home, payload).await.unwrap();
11124 assert!(
11125 crate::worker::active(&home).unwrap().is_empty(),
11126 "monitoring only queues; normal dispatch owns launches"
11127 );
11128 assert_eq!(queue.get(&waiting.id).unwrap().attempts, 2);
11129 }
11130
11131 #[tokio::test]
11132 async fn source_eviction_service_receipts_explain_idle_and_active_skips() {
11133 let dir = tempfile::tempdir().unwrap();
11134 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
11135 let cfg = Config::default();
11136 start_service(
11137 &home,
11138 &opts,
11139 &queue,
11140 &worktrees,
11141 "housekeeping",
11142 Some(&cfg),
11143 true,
11144 )
11145 .await
11146 .unwrap();
11147 let receipt = home.join("source-eviction-housekeeping-dispatch.json");
11148 let read = |path: &Path| -> serde_json::Value {
11149 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
11150 };
11151 assert_eq!(read(&receipt)["detail"], "cadence not due");
11152 std::fs::remove_file(home.join("service-ticks/daemon-housekeeping")).unwrap();
11153 start_service(
11154 &home,
11155 &opts,
11156 &queue,
11157 &worktrees,
11158 "housekeeping",
11159 Some(&cfg),
11160 true,
11161 )
11162 .await
11163 .unwrap();
11164 assert_eq!(
11165 read(&receipt)["detail"],
11166 "active work or runnable task queue"
11167 );
11168 assert_eq!(read(&receipt)["version"], env!("CARGO_PKG_VERSION"));
11169 assert_eq!(read(&receipt)["auto_fold"], cfg.disk.auto_fold);
11170 std::fs::remove_file(home.join("service-ticks/daemon-disk-pressure")).unwrap();
11171 start_service(
11172 &home,
11173 &opts,
11174 &queue,
11175 &worktrees,
11176 "disk-pressure",
11177 Some(&cfg),
11178 false,
11179 )
11180 .await
11181 .unwrap();
11182 assert_eq!(
11183 read(&home.join("source-eviction-disk-pressure-dispatch.json"))["detail"],
11184 "no active work; idle housekeeping owns scan"
11185 );
11186 assert!(crate::worker::active(&home).unwrap().is_empty());
11187 }
11188
11189 #[tokio::test]
11190 async fn goal_observation_service_does_not_block_scheduler_decisions() {
11191 let dir = tempfile::tempdir().unwrap();
11192 let (opts, _, _, home, worktrees) = idle_loop(dir.path());
11193 let queue = Queue::at(home.join("queue"));
11194 let mut task = task();
11195 task.repo = opts.repo.clone();
11196 task.chat = Some("decision-chat".into());
11197 task.hold_machine(Some("waiting for a decision".into()));
11198 queue.put(&mut task).unwrap();
11199 let questions = Questions::at(home.join("questions"));
11200 let mut question = action_question(&task.id, ask::ChoiceAction::Done);
11201 question.node = crate::conduct::NODE.into();
11202 question.status = ask::QuestionStatus::Open;
11203 question.answer = None;
11204 question.answered_at = None;
11205 questions.put(&mut question).unwrap();
11206 questions
11207 .update(&question.id, |q| {
11208 q.answer(ask::Answer::Choice("resume で続行する".into()))
11209 })
11210 .unwrap();
11211
11212 let talks = crate::talk::Talks::at(home.join("talks"));
11213 let mut decision_chat = crate::talk::seeded_for_test("decision-chat", opts.repo.clone());
11214 decision_chat.pending = "Keep this unsent status question".into();
11215 talks.put(&mut decision_chat).unwrap();
11216 let mut talk = crate::talk::seeded_for_test("observing-chat", opts.repo.clone());
11217 talk.goals.push(serde_json::from_value(serde_json::json!({
11218 "id":"goal","purpose":"follow the retained job","scope":"one task",
11219 "condition":{"description":"land the result","kind":"runs_merged"},
11220 "state":"runnable","step":"reading the forge","source_turn":0,"owner_input":"land this job",
11221 "tasks":[],"runs":[],"watches":[],"evidence":"","wake":1,"handled":0,
11222 "started":1,"wakes":1,"last_checked":null,"fingerprint":""
11223 })).unwrap());
11224 talks.put(&mut talk).unwrap();
11225
11226 let directory = home.join("workers/jobs/abcdef");
11229 std::fs::create_dir_all(&directory).unwrap();
11230 let lease = std::fs::File::create(directory.join("live.lock")).unwrap();
11231 crate::pr_head::try_lock_file(&lease).unwrap();
11232 let mut job = crate::worker::Job {
11233 id: "abcdef".into(),
11234 sequence: 1,
11235 kind: "daemon-service".into(),
11236 key: "service:daemon-chat-goals".into(),
11237 payload: serde_json::Value::Null,
11238 generation: "fixture".into(),
11239 version: None,
11240 status: crate::worker::Status::Running,
11241 result: None,
11242 pid: Some(std::process::id()),
11243 created_at: Timestamp::now().as_second() as u64,
11244 finished_at: None,
11245 };
11246 let record = directory.join("job.json");
11247 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
11248 let status = Arc::new(Mutex::new(Status::new()));
11249 let stop = Stop::new();
11250 let mut dispatch = Box::pin(poll(
11251 &opts,
11252 &queue,
11253 &status,
11254 &home,
11255 &worktrees,
11256 &stop,
11257 DispatchLimits {
11258 max_concurrent: 1,
11259 pause_for_interrupts: false,
11260 recovery_cfg: None,
11261 },
11262 ));
11263 std::future::poll_fn(|cx| {
11264 assert!(dispatch.as_mut().poll(cx).is_pending());
11265 std::task::Poll::Ready(())
11266 })
11267 .await;
11268 assert!(lock(&status).polls > 0);
11269 let saved = queue.get(&task.id).unwrap();
11270 assert_eq!(saved.status, TaskStatus::Done);
11271 assert!(saved.action_applied(&question.id));
11272 assert!(
11273 talks
11274 .get("decision-chat")
11275 .unwrap()
11276 .pending
11277 .contains("Keep this unsent status question")
11278 );
11279 let goal = talks.get(&talk.id).unwrap().goals.remove(0);
11280 assert_eq!(goal.state, crate::chat_goal::State::Runnable);
11281 assert_eq!(
11282 goal.started,
11283 Some(1),
11284 "only the observation worker may reconcile its goal"
11285 );
11286 assert_eq!(crate::worker::active(&home).unwrap()[0].id, job.id);
11287
11288 stop.stop();
11289 dispatch.await.unwrap();
11290 job.status = crate::worker::Status::Finished;
11291 std::fs::write(record, serde_json::to_vec(&job).unwrap()).unwrap();
11292 drop(lease);
11293 }
11294
11295 #[test]
11296 fn a_stop_is_idempotent_and_once_set_stays_set() {
11297 let stop = Stop::new();
11298 assert!(!stop.stopped());
11299
11300 stop.stop();
11301 assert!(stop.stopped());
11302 stop.stop();
11303 assert!(stop.stopped(), "a second stop is not a toggle");
11304
11305 let shared = stop.clone();
11306 assert!(
11307 shared.stopped(),
11308 "a clone is the same stop; that is how the loop and its caller share one"
11309 );
11310 }
11311
11312 #[tokio::test]
11313 async fn once_drain_rechecks_answers_after_queue_services_release_their_claims() {
11314 for operation in ["conductor", "maintenance", "housekeeping"] {
11315 let dir = tempfile::tempdir().unwrap();
11316 let (mut opts, queue, _, home, worktrees) = idle_loop(dir.path());
11317 opts.once = true;
11318 opts.poll = Duration::from_millis(1);
11319 let mut task = task();
11320 task.runs.push("fixture-run".into());
11321 queue.put(&mut task).unwrap();
11322 let claim = queue.claim(&task.id).unwrap();
11323
11324 let directory = home.join("workers/jobs/abcdef");
11327 std::fs::create_dir_all(&directory).unwrap();
11328 let lease = std::fs::File::create(directory.join("live.lock")).unwrap();
11329 crate::pr_head::try_lock_file(&lease).unwrap();
11330 let mut job = crate::worker::Job {
11331 id: "abcdef".into(),
11332 sequence: 1,
11333 kind: "daemon-service".into(),
11334 key: format!("service:daemon-{operation}"),
11335 payload: serde_json::Value::Null,
11336 generation: "fixture".into(),
11337 version: None,
11338 status: crate::worker::Status::Running,
11339 result: None,
11340 pid: Some(std::process::id()),
11341 created_at: Timestamp::now().as_second() as u64,
11342 finished_at: None,
11343 };
11344 let record = directory.join("job.json");
11345 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
11346
11347 let status = Arc::new(Mutex::new(Status::new()));
11348 let stop = Stop::new();
11349 let mut drain = Box::pin(poll(
11350 &opts,
11351 &queue,
11352 &status,
11353 &home,
11354 &worktrees,
11355 &stop,
11356 DispatchLimits {
11357 max_concurrent: 1,
11358 pause_for_interrupts: false,
11359 recovery_cfg: None,
11360 },
11361 ));
11362 std::future::poll_fn(|cx| {
11363 use std::future::Future;
11364 assert!(
11365 drain.as_mut().poll(cx).is_pending(),
11366 "--once must wait for {operation} to release its queue claim"
11367 );
11368 std::task::Poll::Ready(())
11369 })
11370 .await;
11371 assert!(lock(&status).polls > 0, "the dispatch scan was exercised");
11372
11373 let questions = Questions::at(home.join("questions"));
11376 let mut q = action_question("fixture-run", ask::ChoiceAction::Done);
11377 questions.put(&mut q).unwrap();
11378 drop(claim);
11379 let expired =
11383 serde_json::to_vec(&"2000-01-01T00:00:00Z".parse::<Timestamp>().unwrap()).unwrap();
11384 for service in [operation, "housekeeping"] {
11385 std::fs::write(
11386 home.join("service-ticks").join(format!("daemon-{service}")),
11387 &expired,
11388 )
11389 .unwrap();
11390 }
11391 job.status = crate::worker::Status::Finished;
11392 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
11393 drop(lease);
11394 tokio::time::timeout(Duration::from_secs(2), drain)
11395 .await
11396 .expect("the settled service must allow the drain to finish")
11397 .unwrap();
11398 let after = queue.get(&task.id).unwrap();
11399 assert_eq!(after.status, TaskStatus::Done, "answer after {operation}");
11400 assert!(after.action_applied(&q.id));
11401 for service in [operation, "housekeeping"] {
11402 assert_eq!(
11403 std::fs::read(home.join("service-ticks").join(format!("daemon-{service}")))
11404 .unwrap(),
11405 expired,
11406 "the final once rescan must not restart {service}"
11407 );
11408 }
11409 }
11410 }
11411
11412 #[test]
11413 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
11414 let stop = Stop::new();
11415 stop.busy.store(1, Ordering::SeqCst);
11416 assert!(
11417 !stop.finishing(),
11418 "a busy loop nobody has asked to stop is just running"
11419 );
11420
11421 stop.stop();
11422 assert!(
11423 stop.finishing(),
11424 "a stop asked for mid-run has not landed until the run is settled"
11425 );
11426
11427 stop.busy.store(0, Ordering::SeqCst);
11428 assert!(
11429 !stop.finishing(),
11430 "once the run is settled the stop has landed and there is nothing to finish"
11431 );
11432 }
11433
11434 #[test]
11435 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
11436 let stop = Stop::new();
11437 stop.busy.store(2, Ordering::SeqCst);
11438 stop.stop();
11439 assert!(stop.finishing(), "two runs still in flight");
11440
11441 stop.busy.store(1, Ordering::SeqCst);
11442 assert!(
11443 stop.finishing(),
11444 "one run finished, but a sibling is still working"
11445 );
11446
11447 stop.busy.store(0, Ordering::SeqCst);
11448 assert!(
11449 !stop.finishing(),
11450 "the last run out is what actually lands the stop"
11451 );
11452 }
11453
11454 #[tokio::test]
11455 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
11456 let dir = tempfile::tempdir().unwrap();
11457 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
11458 let stop = Stop::new();
11459 stop.stop();
11460
11461 let began = std::time::Instant::now();
11462 tokio::time::timeout(
11463 Duration::from_secs(2),
11464 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
11465 )
11466 .await
11467 .expect("a stopped loop must return, not sit out its poll interval")
11468 .expect("the loop's own setup and teardown must not fail");
11469 assert!(
11470 began.elapsed() < opts.poll,
11471 "returned only after {:?}, which is a poll interval, not a stop",
11472 began.elapsed()
11473 );
11474 }
11475
11476 #[tokio::test]
11477 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
11478 let dir = tempfile::tempdir().unwrap();
11479 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
11480 let stop = Stop::new();
11481
11482 let asker = {
11485 let stop = stop.clone();
11486 tokio::spawn(async move {
11487 tokio::time::sleep(Duration::from_millis(20)).await;
11488 stop.stop();
11489 })
11490 };
11491
11492 let began = std::time::Instant::now();
11493 tokio::time::timeout(
11494 Duration::from_secs(2),
11495 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
11496 )
11497 .await
11498 .expect("a stop asked for while idle must wake the wait")
11499 .expect("the loop's own setup and teardown must not fail");
11500 asker.await.unwrap();
11501 assert!(
11502 began.elapsed() < opts.poll,
11503 "returned only after {:?}, so the stop waited on the sleep",
11504 began.elapsed()
11505 );
11506 }
11507
11508 #[tokio::test]
11509 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
11510 let dir = tempfile::tempdir().unwrap();
11511 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
11512 let stop = Stop::new();
11513 stop.stop();
11514
11515 tokio::time::timeout(
11516 Duration::from_secs(2),
11517 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
11518 )
11519 .await
11520 .expect("a stopped loop must return")
11521 .expect("the loop's own setup and teardown must not fail");
11522
11523 assert!(
11524 home.is_dir(),
11525 "the loop did publish a status file, so its removal is the teardown and not an absence"
11526 );
11527 assert!(
11528 !status_file.exists(),
11529 "a stopped loop clears its status file"
11530 );
11531 assert!(
11532 read_status(&home).is_none(),
11533 "a reader must see no daemon at all, not a heartbeat that merely stopped"
11534 );
11535 }
11536
11537 #[tokio::test]
11538 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
11539 let dir = tempfile::tempdir().unwrap();
11540 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
11541
11542 let mut settled = RunState::new(
11543 dir.path().join("repo"),
11544 "main".to_owned(),
11545 "abc1234".to_owned(),
11546 "fixture".to_owned(),
11547 Config::default(),
11548 );
11549 settled.status = RunStatus::Ready;
11550 let run_dir = home.join("runs").join(&settled.id);
11551 std::fs::create_dir_all(&run_dir).unwrap();
11552 std::fs::write(
11553 run_dir.join("run.json"),
11554 serde_json::to_string_pretty(&settled).unwrap(),
11555 )
11556 .unwrap();
11557 let questions = Questions::at(home.join("questions"));
11558 let mut question = ask::Question::new(
11559 settled.id.clone(),
11560 "review".to_owned(),
11561 "reviewer-1".to_owned(),
11562 "Continue?".to_owned(),
11563 String::new(),
11564 Vec::new(),
11565 );
11566 questions.put(&mut question).unwrap();
11567
11568 run_service_worker(
11569 &home,
11570 serde_json::to_value(ServiceWorker {
11571 opts,
11572 queue: queue.root().to_path_buf(),
11573 worktrees_root: worktrees,
11574 operation: "housekeeping".to_owned(),
11575 })
11576 .unwrap(),
11577 )
11578 .await
11579 .unwrap();
11580
11581 assert_eq!(
11582 questions.get(&question.id).unwrap().status,
11583 ask::QuestionStatus::Abandoned,
11584 "the independent housekeeping worker settles ended-run questions"
11585 );
11586 }
11587
11588 #[test]
11589 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
11590 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
11591
11592 assert!(
11593 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
11594 "never checked before: due at once"
11595 );
11596
11597 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
11598 assert!(
11599 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
11600 "well inside the interval: not due yet"
11601 );
11602
11603 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
11604 assert!(
11605 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
11606 "exactly at the edge: not yet due, same convention as `clean::due`"
11607 );
11608
11609 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
11610 assert!(
11611 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
11612 "past the interval: due again"
11613 );
11614 }
11615
11616 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
11621 let config = dir.join("magi.toml");
11622 std::fs::write(
11628 &config,
11629 format!(
11630 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
11631 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
11632 cache_dir.display()
11633 ),
11634 )
11635 .unwrap();
11636 Opts {
11637 config: Some(config),
11638 repo: dir.join("repo"),
11639 ..Opts::default()
11640 }
11641 }
11642
11643 #[tokio::test]
11644 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
11645 let dir = tempfile::tempdir().unwrap();
11646 let home = dir.path().join("home");
11647 let cache_dir = dir.path().join("cache");
11648 std::fs::create_dir_all(&cache_dir).unwrap();
11649 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
11650 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
11651
11652 let running = Stop::new();
11655 let mut last_checked = None;
11656 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
11657 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
11658 .await;
11659 assert_eq!(
11660 crate::disk::dir_size(&cache_dir),
11661 0,
11662 "over the cap on the first check ever: pruned at once, no idle queue required"
11663 );
11664 assert_eq!(last_checked, Some(t0));
11665
11666 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
11668 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
11669 maybe_prune_cache_between_runs(
11670 &opts.repo,
11671 &opts,
11672 &home,
11673 &running,
11674 &mut last_checked,
11675 too_soon,
11676 )
11677 .await;
11678 assert_eq!(
11679 crate::disk::dir_size(&cache_dir),
11680 10,
11681 "too soon since the last check: left alone rather than rescanned every call"
11682 );
11683 assert_eq!(
11684 last_checked,
11685 Some(t0),
11686 "an idle check does not reset the clock"
11687 );
11688
11689 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
11691 maybe_prune_cache_between_runs(
11692 &opts.repo,
11693 &opts,
11694 &home,
11695 &running,
11696 &mut last_checked,
11697 due_again,
11698 )
11699 .await;
11700 assert_eq!(
11701 crate::disk::dir_size(&cache_dir),
11702 0,
11703 "due again: pruned back under the cap"
11704 );
11705 }
11706
11707 #[tokio::test]
11715 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
11716 let dir = tempfile::tempdir().unwrap();
11717 let home = dir.path().join("home");
11718 let cache_dir = dir.path().join("cache");
11719 std::fs::create_dir_all(&cache_dir).unwrap();
11720 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
11721 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
11722
11723 let stop = Stop::new();
11724 stop.stop();
11725 assert!(
11726 !stop.finishing(),
11727 "no run is in flight at a between-runs boundary, so nothing else \
11728 would tell the operator this stop had not taken effect yet"
11729 );
11730
11731 let mut last_checked = None;
11732 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
11733 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
11734 .await;
11735 assert_eq!(
11736 crate::disk::dir_size(&cache_dir),
11737 10,
11738 "over its cap, and due for the first check ever, but a stop outranks \
11739 it: the cap is a standing policy the next start measures again"
11740 );
11741 assert_eq!(
11742 last_checked, None,
11743 "a check that never happened must not claim the interval"
11744 );
11745 }
11746
11747 #[tokio::test]
11750 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
11751 let dir = tempfile::tempdir().unwrap();
11752 let cache_dir = dir.path().join("cache");
11753 std::fs::create_dir_all(&cache_dir).unwrap();
11754 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
11755
11756 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
11757 opts.poll = Duration::from_millis(20);
11758 opts.max_attempts = 1_000;
11759
11760 let queue = Queue::at(dir.path().join("queue"));
11761 for i in 0..128 {
11762 let mut task = Task::new(
11763 format!("queued work {i}"),
11764 format!("queued work {i}"),
11765 opts.repo.clone(),
11766 Source::Human,
11767 );
11768 queue.put(&mut task).unwrap();
11769 }
11770
11771 let home = dir.path().join("home");
11772 let worktrees = dir.path().join("wt");
11773 run_service_worker(
11774 &home,
11775 serde_json::to_value(ServiceWorker {
11776 opts,
11777 queue: queue.root().to_path_buf(),
11778 worktrees_root: worktrees,
11779 operation: "cache".to_owned(),
11780 })
11781 .unwrap(),
11782 )
11783 .await
11784 .unwrap();
11785 let after = queue.list();
11786 assert_eq!(after.len(), 128);
11787 assert!(
11788 after
11789 .iter()
11790 .all(|task| task.status.runnable() && task.attempts == 0)
11791 );
11792
11793 assert_eq!(
11794 crate::disk::dir_size(&cache_dir),
11795 0,
11796 "an oversized cache must not be left to grow unboundedly just because the \
11797 queue kept the loop busy the whole time"
11798 );
11799 }
11800
11801 #[test]
11802 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
11803 let dir = tempfile::tempdir().unwrap();
11804 let queue = Queue::at(dir.path().join("queue"));
11805 let questions = Questions::at(dir.path().join("questions"));
11806 let mut task = task();
11807 queue.put(&mut task).unwrap();
11808
11809 let mut task_question = ask::Question::new(
11810 task.id.clone(),
11811 crate::conduct::NODE.to_owned(),
11812 "conduct".to_owned(),
11813 "Which backend?".to_owned(),
11814 String::new(),
11815 Vec::new(),
11816 );
11817 questions.put(&mut task_question).unwrap();
11818 task.block(vec![task_question.id.clone()], None);
11819 queue.put(&mut task).unwrap();
11820
11821 let mut run_question = ask::Question::new(
11822 "20260101-000000-run1".to_owned(),
11823 "review".to_owned(),
11824 "reviewer-1".to_owned(),
11825 "Run question".to_owned(),
11826 String::new(),
11827 Vec::new(),
11828 );
11829 questions.put(&mut run_question).unwrap();
11830
11831 let mut coincidental = ask::Question::new(
11836 task.id.clone(),
11837 "review".to_owned(),
11838 "reviewer-1".to_owned(),
11839 "Unrelated review question".to_owned(),
11840 String::new(),
11841 Vec::new(),
11842 );
11843 questions.put(&mut coincidental).unwrap();
11844
11845 reconcile_task_questions(&queue, &questions);
11846 assert!(questions.get(&task_question.id).unwrap().status.open());
11847 assert!(questions.get(&run_question.id).unwrap().status.open());
11848 assert!(questions.get(&coincidental.id).unwrap().status.open());
11849
11850 task.release();
11851 queue.put(&mut task).unwrap();
11852 reconcile_task_questions(&queue, &questions);
11853 assert_eq!(
11854 questions.get(&task_question.id).unwrap().status,
11855 ask::QuestionStatus::Abandoned
11856 );
11857 assert!(
11858 questions.get(&run_question.id).unwrap().status.open(),
11859 "run questions remain the run janitor's responsibility"
11860 );
11861 assert!(
11862 questions.get(&coincidental.id).unwrap().status.open(),
11863 "a non-conductor question must not be abandoned just because its \
11864 run id coincides with a task id"
11865 );
11866 }
11867
11868 #[test]
11869 fn a_freshly_started_running_task_is_never_stalled() {
11870 let dir = tempfile::tempdir().unwrap();
11871 let mut t = task();
11872 t.start("run-1".to_owned());
11873 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
11876 }
11877
11878 #[test]
11879 fn a_long_running_task_with_no_live_daemon_is_stalled() {
11880 let dir = tempfile::tempdir().unwrap();
11881 let mut t = task();
11882 t.start("run-1".to_owned());
11883 t.updated_at = Timestamp::now()
11884 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
11885 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
11886 assert_eq!(
11887 stalled_tasks(
11888 &Queue::at(dir.path().join("q")),
11889 dir.path(),
11890 Timestamp::now()
11891 )
11892 .len(),
11893 0,
11894 "the task was never written to this queue"
11895 );
11896 }
11897
11898 #[test]
11899 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
11900 let dir = tempfile::tempdir().unwrap();
11901 let mut t = task();
11902 t.id = "20260903-080340-0167".to_owned();
11903 t.start("20260903-080619-01c2".to_owned());
11904 t.updated_at = Timestamp::now()
11905 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
11906
11907 let mut status = Status::new();
11908 status.current = vec![Current {
11909 worker: None,
11910 task: t.id.clone(),
11911 run: "20260903-080619-01c2".to_owned(),
11912 }];
11913 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
11914
11915 assert!(
11916 !is_stalled(&t, dir.path(), Timestamp::now()),
11917 "a live daemon's own heartbeat rules out stalled, however long the task has run"
11918 );
11919 }
11920
11921 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
11925 let path = queue.path_of(id);
11926 let body = std::fs::read_to_string(&path).unwrap();
11927 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
11928 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
11929 v["updated_at"] = serde_json::Value::String(old.to_string());
11930 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
11931 }
11932
11933 #[test]
11934 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
11935 let dir = tempfile::tempdir().unwrap();
11948 let queue = Queue::at(dir.path().join("queue"));
11949 let home = dir.path().join("home");
11950
11951 let mut t = task();
11952 t.id = "20260101-000001-lock".to_owned();
11953 t.start("run-1".to_owned());
11954 queue.put(&mut t).unwrap();
11955 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
11956 std::fs::write(
11957 dir.path().join("queue").join(format!("{}.lock", t.id)),
11958 "not a pid",
11959 )
11960 .unwrap();
11961
11962 let now = Timestamp::now();
11963 assert!(
11964 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
11965 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
11966 and reclaim must leave the task alone"
11967 );
11968 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
11969
11970 let stalled = stalled_tasks(&queue, &home, now);
11971 assert_eq!(
11972 stalled.len(),
11973 1,
11974 "reclaim's inability to claim it yet must not hide it from the conductor"
11975 );
11976 assert_eq!(stalled[0].id, t.id);
11977 }
11978
11979 #[test]
11980 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
11981 let dir = tempfile::tempdir().unwrap();
11982 crate::run::set_home(dir.path().join("run-home"));
11983 let queue = Queue::at(dir.path().join("queue"));
11984 let home = dir.path().join("home");
11985 let questions = Questions::at(dir.path().join("questions"));
11986
11987 let mut t = task();
11988 t.id = "20260101-000003-dead".to_owned();
11989 t.start("missing-run".to_owned());
11990 queue.put(&mut t).unwrap();
11991 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
11992
11993 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
11996 assert_eq!(
11997 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
11998 [&t.id]
11999 );
12000 assert_eq!(
12001 reclaim_orphaned_running(&queue, 2, &Default::default()),
12002 [t.id.clone()]
12003 );
12004 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12005
12006 crate::conduct::apply(
12009 &queue,
12010 &questions,
12011 &crate::conduct::Verdict {
12012 decisions: vec![crate::conduct::Decision {
12013 id: t.id.clone(),
12014 recovery: Some(crate::conduct::Recovery::Requeue),
12015 ..crate::conduct::Decision::default()
12016 }],
12017 },
12018 )
12019 .unwrap();
12020 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12021 }
12022
12023 #[test]
12024 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
12025 let dir = tempfile::tempdir().unwrap();
12026 let queue = Queue::at(dir.path().join("queue"));
12027 let home = dir.path().join("home");
12028
12029 let mut fresh = task();
12030 fresh.id = "20260101-000001-aaaa".to_owned();
12031 fresh.start("run-1".to_owned());
12032 queue.put(&mut fresh).unwrap();
12033
12034 let mut old = task();
12035 old.id = "20260101-000002-bbbb".to_owned();
12036 old.start("run-2".to_owned());
12037 queue.put(&mut old).unwrap();
12038 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
12039
12040 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
12041 assert_eq!(stalled.len(), 1);
12042 assert_eq!(stalled[0].id, old.id);
12043 }
12044
12045 #[test]
12046 fn queued_and_finished_task_views_partition_by_status() {
12047 let dir = tempfile::tempdir().unwrap();
12048 let queue = Queue::at(dir.path().join("queue"));
12049
12050 let mut queued = task();
12051 queued.id = "20260101-000001-aaaa".to_owned();
12052 queue.put(&mut queued).unwrap();
12053
12054 let mut failed = task();
12055 failed.id = "20260101-000002-bbbb".to_owned();
12056 failed.start("run-1".to_owned());
12057 failed.fail("gate red", 5);
12058 queue.put(&mut failed).unwrap();
12059
12060 let mut held = task();
12061 held.id = "20260101-000003-cccc".to_owned();
12062 held.hold_machine(None);
12063 queue.put(&mut held).unwrap();
12064
12065 let mut running = task();
12066 running.id = "20260101-000004-dddd".to_owned();
12067 running.start("run-2".to_owned());
12068 queue.put(&mut running).unwrap();
12069
12070 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
12071 assert_eq!(queued_ids, [queued.id.clone()]);
12072
12073 let mut finished_ids: Vec<String> =
12074 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
12075 finished_ids.sort_unstable();
12076 let mut want = vec![failed.id.clone(), held.id.clone()];
12077 want.sort_unstable();
12078 assert_eq!(finished_ids, want);
12079 }
12080
12081 #[test]
12082 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
12083 for short in [false, true] {
12084 let dir = tempfile::tempdir().unwrap();
12085 let queue = Queue::at(dir.path().join("queue"));
12086 let questions = ask::Questions::at(dir.path().join("questions"));
12087 let mut running = task();
12088 running.id = "20261013-192213-feb2".into();
12089 running.start("existing-run".into());
12090 queue.put(&mut running).unwrap();
12091 let mut middle = task();
12092 middle.id = "20261013-193611-ca5b".into();
12093 middle.block(
12094 vec![if short {
12095 "feb2".into()
12096 } else {
12097 running.id.clone()
12098 }],
12099 None,
12100 );
12101 queue.put(&mut middle).unwrap();
12102 let mut waiting = task();
12103 waiting.id = "20261013-202849-ee37".into();
12104 waiting.block(
12105 vec![if short {
12106 "ca5b".into()
12107 } else {
12108 middle.id.clone()
12109 }],
12110 Some("wait for implementation".into()),
12111 );
12112 queue.put(&mut waiting).unwrap();
12113 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
12114 let inv = crate::blockers::Inventory::load(&queue, &questions);
12115 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
12116 assert!(inv.stuck().is_empty());
12117 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
12118 assert!(report.quarantined.is_empty());
12119 assert!(report.asked.is_empty());
12120 resolve_blockers(&queue, &questions);
12121 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
12122 assert!(questions.list().is_empty());
12123 running.succeed();
12126 queue.put(&mut running).unwrap();
12127 resolve_blockers(&queue, &questions);
12128 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
12129 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
12130 middle = queue.get(&middle.id).unwrap();
12131 middle.succeed();
12132 queue.put(&mut middle).unwrap();
12133 resolve_blockers(&queue, &questions);
12134 let after = queue.get(&waiting.id).unwrap();
12135 assert_eq!(after.status, TaskStatus::Queued);
12136 assert!(after.blocked_by.is_empty());
12137 assert_eq!(after.instruction, waiting.instruction);
12138 }
12139 }
12140
12141 #[test]
12142 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
12143 for future_schema in [false, true] {
12144 let name = "20261013-193611-ca5b";
12145 let mut future = task();
12146 future.id = name.into();
12147 future.schema = crate::queue::SCHEMA + 1;
12148 let contents = if future_schema {
12149 serde_json::to_string(&future).unwrap()
12150 } else {
12151 "corrupt".into()
12152 };
12153 let dir = tempfile::tempdir().unwrap();
12154 let queue = Queue::at(dir.path().join("queue"));
12155 let questions = ask::Questions::at(dir.path().join("questions"));
12156 let mut waiting = task();
12157 waiting.id = "20261013-202849-ee37".into();
12158 waiting.block(vec!["ca5b".into()], None);
12159 queue.put(&mut waiting).unwrap();
12160 std::fs::write(queue.path_of(name), contents).unwrap();
12161 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
12162 assert!(
12163 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
12164 );
12165 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
12166 assert!(report.asked.is_empty());
12167 resolve_blockers(&queue, &questions);
12168 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
12169 assert!(questions.list().is_empty());
12170 }
12171 let dir = tempfile::tempdir().unwrap();
12172 let queue = Queue::at(dir.path().join("queue"));
12173 let questions = ask::Questions::at(dir.path().join("questions"));
12174 let mut waiting = task();
12175 waiting.id = "20261013-202849-ee37".into();
12176 waiting.block(vec!["ca5b".into()], None);
12177 queue.put(&mut waiting).unwrap();
12178 let mut done = task();
12179 done.id = "20261013-193611-ca5b".into();
12180 done.succeed();
12181 queue.put(&mut done).unwrap();
12182 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
12185 assert!(queue.get("ca5b").is_err());
12186 assert_eq!(
12187 crate::queue::dependency(&queue, &questions, "ca5b"),
12188 crate::queue::Dependency::Ambiguous
12189 );
12190 resolve_blockers(&queue, &questions);
12191 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
12192 assert!(report.asked.is_empty());
12193 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
12194 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
12195 }
12196
12197 #[test]
12198 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
12199 for held in [false, true] {
12200 for question in [false, true] {
12201 for reference in ["ca5b", "20261013-193611-ca5b"] {
12202 let dir = tempfile::tempdir().unwrap();
12203 let queue = Queue::at(dir.path().join("queue"));
12204 let questions = ask::Questions::at(dir.path().join("questions"));
12205 let mut prerequisite = task();
12206 prerequisite.id = "20261013-193611-ca5b".into();
12207 prerequisite.start("existing-run".into());
12208 queue.put(&mut prerequisite).unwrap();
12209 let mut waiting = task();
12210 waiting.id = "20261013-202849-ee37".into();
12211 if held {
12212 waiting.hold_machine(Some("waiting for separate consent".into()));
12213 }
12214 queue.put(&mut waiting).unwrap();
12215 let reason = "Deleting saved data requires separate owner consent";
12216 crate::conduct::apply(
12217 &queue,
12218 &questions,
12219 &crate::conduct::Verdict {
12220 decisions: vec![crate::conduct::Decision {
12221 id: waiting.id.clone(),
12222 blocked_by: vec![reference.into()],
12223 question: question.then(|| "May the saved data be deleted?".into()),
12224 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
12225 reason: Some(reason.into()),
12226 ..crate::conduct::Decision::default()
12227 }],
12228 },
12229 )
12230 .unwrap();
12231 let before = queue.get(&waiting.id).unwrap();
12232 if question {
12233 assert_eq!(before.status, TaskStatus::Blocked);
12234 let q = questions.get(&before.blocked_by[0]).unwrap();
12235 assert_eq!(q.summary, "May the saved data be deleted?");
12236 assert!(q.status.open());
12237 } else {
12238 assert_eq!(before.status, TaskStatus::Held);
12239 let recorded_reason = before.hold_reason.as_deref().unwrap();
12240 assert!(recorded_reason.starts_with(reason));
12241 if held {
12242 assert!(recorded_reason.contains("waiting for separate consent"));
12243 }
12244 }
12245 prerequisite.succeed();
12246 queue.put(&mut prerequisite).unwrap();
12247 resolve_blockers(&queue, &questions);
12248 let after = queue.get(&waiting.id).unwrap();
12249 assert_eq!(after.status, before.status);
12250 assert_eq!(after.blocked_by, before.blocked_by);
12251 assert_eq!(after.hold_reason, before.hold_reason);
12252 assert_eq!(after.instruction, waiting.instruction);
12253 if question {
12254 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
12255 }
12256 assert!(
12257 queue.next_runnable().is_none(),
12258 "dependency completion is not consent"
12259 );
12260 }
12261 }
12262 }
12263 }
12264
12265 #[test]
12266 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
12267 let dir = tempfile::tempdir().unwrap();
12268 let queue = Queue::at(dir.path().join("queue"));
12269 let questions = ask::Questions::at(dir.path().join("questions"));
12270
12271 let mut dep = task();
12272 dep.id = "20260101-000001-dep0".to_owned();
12273 dep.succeed();
12274 queue.put(&mut dep).unwrap();
12275
12276 let mut still_going = task();
12277 still_going.id = "20260101-000002-dep1".to_owned();
12278 queue.put(&mut still_going).unwrap();
12279
12280 let mut blocked = task();
12281 blocked.id = "20260101-000003-main".to_owned();
12282 blocked.block(
12283 vec![dep.id.clone(), still_going.id.clone()],
12284 Some("waits on both".to_owned()),
12285 );
12286 queue.put(&mut blocked).unwrap();
12287
12288 resolve_blockers(&queue, &questions);
12289
12290 let after = queue.get(&blocked.id).unwrap();
12291 assert_eq!(
12292 after.status,
12293 TaskStatus::Blocked,
12294 "one dependency is still outstanding"
12295 );
12296 assert_eq!(after.blocked_by, [still_going.id.clone()]);
12297 }
12298
12299 #[test]
12300 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
12301 let dir = tempfile::tempdir().unwrap();
12302 let queue = Queue::at(dir.path().join("queue"));
12303 let questions = ask::Questions::at(dir.path().join("questions"));
12304
12305 let mut q = crate::ask::Question::new(
12306 "20260101-000001-main".to_owned(),
12307 crate::conduct::NODE.to_owned(),
12308 "conduct".to_owned(),
12309 "Which backend?".to_owned(),
12310 String::new(),
12311 Vec::new(),
12312 );
12313 questions.put(&mut q).unwrap();
12314 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
12315 .unwrap();
12316 questions.put(&mut q).unwrap();
12317
12318 let mut blocked = task();
12319 blocked.id = "20260101-000001-main".to_owned();
12320 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
12321 queue.put(&mut blocked).unwrap();
12322
12323 resolve_blockers(&queue, &questions);
12324
12325 let after = queue.get(&blocked.id).unwrap();
12326 assert_eq!(
12327 after.status,
12328 TaskStatus::Queued,
12329 "the only blocker resolved"
12330 );
12331 assert_eq!(after.answers.len(), 1);
12332 assert_eq!(after.answers[0].question, "Which backend?");
12333 assert_eq!(after.answers[0].answer, "SQLite");
12334
12335 let instruction = instruction_for(&after);
12337 assert!(instruction.contains("Which backend?"));
12338 assert!(instruction.contains("SQLite"));
12339 }
12340
12341 #[test]
12342 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
12343 let dir = tempfile::tempdir().unwrap();
12344 let queue = Queue::at(dir.path().join("queue"));
12345 let questions = ask::Questions::at(dir.path().join("questions"));
12346
12347 let mut q = crate::ask::Question::new(
12348 "20260101-000001-main".to_owned(),
12349 crate::conduct::NODE.to_owned(),
12350 "conduct".to_owned(),
12351 "Is the setup done?".to_owned(),
12352 String::new(),
12353 Vec::new(),
12354 );
12355 q.abandon("no answer within 60s of asking");
12356 questions.put(&mut q).unwrap();
12357
12358 let mut blocked = task();
12359 blocked.id = "20260101-000001-main".to_owned();
12360 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
12361 queue.put(&mut blocked).unwrap();
12362
12363 resolve_blockers(&queue, &questions);
12364
12365 let after = queue.get(&blocked.id).unwrap();
12366 assert_eq!(
12367 after.status,
12368 TaskStatus::Held,
12369 "never left blocked on nothing"
12370 );
12371 assert!(!after.operator_held(), "a machine hold, for triage");
12372 assert!(
12373 after
12374 .hold_reason
12375 .as_deref()
12376 .unwrap_or_default()
12377 .contains("went unanswered")
12378 );
12379 }
12380
12381 #[test]
12382 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
12383 let dir = tempfile::tempdir().unwrap();
12389 let queue = Queue::at(dir.path().join("queue"));
12390 let questions = ask::Questions::at(dir.path().join("questions"));
12391
12392 let mut q = crate::ask::Question::new(
12393 "20260101-000001-main".to_owned(),
12394 crate::conduct::NODE.to_owned(),
12395 "conduct".to_owned(),
12396 "How should this be handled?".to_owned(),
12397 String::new(),
12398 Vec::new(),
12399 );
12400 questions.put(&mut q).unwrap();
12401 q.answer(crate::ask::Answer::Text(
12402 "leave it held, a human will look at it later".to_owned(),
12403 ))
12404 .unwrap();
12405 questions.put(&mut q).unwrap();
12406
12407 let mut held = task();
12408 held.id = "20260101-000001-main".to_owned();
12409 held.hold_machine(Some("out of attempts".to_owned()));
12410 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
12411 queue.put(&mut held).unwrap();
12412
12413 resolve_blockers(&queue, &questions);
12414
12415 let after = queue.get(&held.id).unwrap();
12416 assert_eq!(after.status, TaskStatus::Held);
12417 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
12418 assert_eq!(
12419 after.answers[0].answer,
12420 "leave it held, a human will look at it later"
12421 );
12422 }
12423
12424 #[test]
12425 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
12426 let dir = tempfile::tempdir().unwrap();
12427 let queue = Queue::at(dir.path().join("queue"));
12428 let questions = ask::Questions::at(dir.path().join("questions"));
12429 let mut dep = task();
12430 dep.id = "20260101-000001-gone".to_owned();
12431 queue.put(&mut dep).unwrap();
12432 let mut blocked = task();
12433 blocked.id = "20260101-000003-main".to_owned();
12434 blocked.block(vec![dep.id.clone()], None);
12435 queue.put(&mut blocked).unwrap();
12436
12437 let claim = queue.claim(&blocked.id).unwrap();
12439 queue.remove(&dep.id, false, &questions).unwrap();
12440 drop(claim);
12441 resolve_blockers(&queue, &questions);
12442
12443 let after = queue.get(&blocked.id).unwrap();
12444 assert_eq!(after.status, TaskStatus::Queued);
12445 assert!(after.blocked_by.is_empty());
12446 }
12447
12448 #[test]
12449 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
12450 let dir = tempfile::tempdir().unwrap();
12456 let queue = Queue::at(dir.path().join("queue"));
12457 let questions = ask::Questions::at(dir.path().join("questions"));
12458
12459 let mut still_going = task();
12460 still_going.id = "20260101-000002-dep1".to_owned();
12461 queue.put(&mut still_going).unwrap();
12462
12463 let mut blocked = task();
12464 blocked.id = "20260101-000003-main".to_owned();
12465 blocked.block(
12466 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
12467 Some("waits on both".to_owned()),
12468 );
12469 queue.put(&mut blocked).unwrap();
12470
12471 resolve_blockers(&queue, &questions);
12472
12473 let after = queue.get(&blocked.id).unwrap();
12474 assert_eq!(
12475 after.status,
12476 TaskStatus::Held,
12477 "a missing dependency must not leave the task blocked forever"
12478 );
12479 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
12480 assert!(after.blocked_by.is_empty());
12481 let reason = after.hold_reason.as_deref().unwrap_or_default();
12482 assert!(
12483 reason.contains("20260101-000001-gone"),
12484 "the missing id must be named so an operator can tell what happened: {reason}"
12485 );
12486 assert!(
12487 reason.contains(&still_going.id),
12488 "the still-valid dependency must not silently vanish from the record: {reason}"
12489 );
12490 }
12491
12492 #[test]
12493 fn instruction_for_is_unchanged_without_any_answers() {
12494 let t = task();
12495 assert_eq!(instruction_for(&t), t.instruction);
12496 }
12497
12498 #[test]
12499 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
12500 let dir = tempfile::tempdir().unwrap();
12501 let q = Queue::at(dir.path().join("queue"));
12502 let src = dir.path().join("shot.png");
12503 std::fs::write(&src, "x").unwrap();
12504 let mut t = task();
12505 q.attach(&mut t, &[src]).unwrap();
12506 let paths = task_attachments(&q, &t).unwrap();
12507 assert_eq!(paths.len(), 1);
12508 assert!(paths[0].is_absolute() && paths[0].is_file());
12509 std::fs::remove_file(&paths[0]).unwrap();
12510 let err = task_attachments(&q, &t).unwrap_err().to_string();
12511 assert!(err.contains("shot.png"), "{err}");
12512 }
12513
12514 #[tokio::test]
12515 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
12516 let dir = tempfile::tempdir().unwrap();
12517 let queue_path = dir.path().join("queue");
12518 let queue = Queue::at(queue_path.clone());
12519 let mut t = task();
12520 t.source = Source::Agent {
12521 run: "originating-chat".into(),
12522 node: crate::queue::CHAT_NODE.into(),
12523 };
12524 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
12525 stage: "config-fetch",
12526 target: "origin/main".into(),
12527 });
12528 record_config_failure(&mut t, &fetch);
12529 t.hold_machine(Some(format!("{fetch:#}")));
12530 let fetch_key = t.chat_notice_due().unwrap();
12531 t.chat_notices.push(fetch_key.clone());
12532 queue.put(&mut t).unwrap();
12533
12534 let queue = Queue::at(queue_path);
12535 let mut t = queue.get(&t.id).unwrap();
12536 let a = dir.path().join("a.toml");
12537 let b = dir.path().join("b.toml");
12538 for (path, text, new_cause) in [
12539 (&a, "[graph", true),
12540 (&a, "[verify", false),
12541 (&a, "[graph]\nimplementers = 'invalid-type'", true),
12542 (&b, "[graph]\nimplementers = 'invalid-type'", true),
12543 ] {
12544 t.release_operator("test", None);
12545 std::fs::write(path, text).unwrap();
12546 let error = Config::discover_fetched(dir.path(), Some(path))
12547 .await
12548 .unwrap_err();
12549 record_config_failure(&mut t, &error);
12550 t.hold_machine(Some(format!("{error:#}")));
12551 let due = t.chat_notice_due();
12552 assert_eq!(due.is_some(), new_cause, "{error:#}");
12553 if let Some(key) = due {
12554 assert_ne!(key, fetch_key);
12555 t.chat_notices.push(key);
12556 }
12557 }
12558 t.succeed();
12559 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
12560 }
12561
12562 #[test]
12563 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
12564 let dir = tempfile::tempdir().unwrap();
12565 let queue_path = dir.path().join("queue");
12566 let queue = Queue::at(queue_path.clone());
12567 let mut t = task();
12568 t.source = crate::queue::Source::Agent {
12569 run: "originating-chat".into(),
12570 node: crate::queue::CHAT_NODE.into(),
12571 };
12572 let a = dir.path().join("a.png");
12573 let b = dir.path().join("b.png");
12574 std::fs::write(&a, "a").unwrap();
12575 std::fs::write(&b, "b").unwrap();
12576 queue.attach(&mut t, &[a, b]).unwrap();
12577 let paths = task_attachments(&queue, &t).unwrap();
12578 for path in &paths {
12579 std::fs::remove_file(path).unwrap();
12580 }
12581 let missing = task_attachments(&queue, &t).unwrap_err();
12582 t.attempts += 1;
12583 let reason = missing.to_string();
12584 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
12585 let first = t.chat_notice_due().expect("first attachment notice");
12586 t.chat_notices.push(first.clone());
12587 queue.put(&mut t).unwrap();
12588
12589 let queue = Queue::at(queue_path.clone());
12590 let mut t = queue.get(&t.id).unwrap();
12591 assert!(t.chat_notice_due().is_none(), "restart does not replay");
12592 t.release_operator("test", None);
12593 t.attempts += 1;
12594 let missing = task_attachments(&queue, &t).unwrap_err();
12595 t.fail_startup_target(
12596 "startup-attachments",
12597 missing.name,
12598 "same missing attachment with regenerated wording",
12599 1,
12600 );
12601 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
12602
12603 std::fs::write(&paths[0], "restored").unwrap();
12604 t.release_operator("test", None);
12605 t.attempts += 1;
12606 let missing = task_attachments(&queue, &t).unwrap_err();
12607 let reason = missing.to_string();
12608 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
12609 let second = t
12610 .chat_notice_due()
12611 .expect("different attachment must notify");
12612 assert_ne!(first, second);
12613 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
12614 }
12615
12616 #[test]
12617 fn resumed_instruction_is_unchanged_without_any_answers() {
12618 let t = task();
12619 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
12620 }
12621
12622 #[test]
12623 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
12624 let mut t = task();
12625 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
12626 let old = t.instruction.clone();
12630
12631 let refreshed = resumed_instruction(&old, &t);
12632 assert!(refreshed.starts_with(&old), "the original text is kept");
12633 assert!(refreshed.contains("Which backend?"));
12634 assert!(refreshed.contains("SQLite"));
12635 }
12636
12637 #[test]
12638 fn resumed_instruction_keeps_an_original_answers_heading() {
12639 let mut t = task();
12640 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
12641 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
12642
12643 let refreshed = resumed_instruction(&t.instruction, &t);
12644
12645 assert!(
12646 refreshed.starts_with(&t.instruction),
12647 "an answers heading in the original instruction is not the appended block"
12648 );
12649 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
12650 assert!(refreshed.contains("Which backend?"));
12651 assert!(refreshed.contains("SQLite"));
12652
12653 let repeated = resumed_instruction(&refreshed, &t);
12654 assert_eq!(
12655 repeated, refreshed,
12656 "only the final appended block is refreshed"
12657 );
12658 }
12659
12660 #[test]
12661 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
12662 let mut t = task();
12663 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
12664
12665 let once = resumed_instruction(&t.instruction, &t);
12669 let twice = resumed_instruction(&once, &t);
12670 assert_eq!(once, twice);
12671 assert_eq!(once.matches("Which backend?").count(), 1);
12672
12673 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
12675 let refreshed = resumed_instruction(&once, &t);
12676 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
12677 assert!(refreshed.contains("Which backend?"));
12678 assert!(refreshed.contains("Which cache?"));
12679 }
12680
12681 #[test]
12682 fn prepare_instruction_covers_all_three_starters() {
12683 let mut t = task();
12684 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
12685
12686 assert_eq!(
12689 prepare_instruction(&Starter::Start, None, &t),
12690 Some(instruction_for(&t))
12691 );
12692
12693 let old = t.instruction.clone();
12696 assert_eq!(
12697 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
12698 Some(resumed_instruction(&old, &t))
12699 );
12700
12701 assert_eq!(
12705 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
12706 None
12707 );
12708 }
12709
12710 #[test]
12711 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
12712 assert_eq!(
12713 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
12714 Starter::Review("magi/eba2/A".to_owned())
12715 );
12716 }
12717
12718 #[test]
12719 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
12720 assert_eq!(
12721 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
12722 Starter::Start,
12723 "a vanished review branch must not fall back to resuming the old run either"
12724 );
12725 }
12726
12727 #[test]
12728 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
12729 let mut t = task();
12730 t.start("old-run".to_owned());
12731 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
12732 t.release();
12733 let branch = t.review_branch.take();
12734 assert_eq!(
12735 choose_starter(branch.as_deref(), true, Some("old-run")),
12736 Starter::Review("magi/eba2/A".to_owned()),
12737 "a review wins over resuming the old run"
12738 );
12739 }
12740
12741 #[test]
12742 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
12743 assert_eq!(
12744 choose_starter(None, false, Some("some-run")),
12745 Starter::Resume("some-run".to_owned())
12746 );
12747 assert_eq!(choose_starter(None, false, None), Starter::Start);
12748 }
12749
12750 #[test]
12751 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
12752 let mut released = task();
12753 released.start("stalled-run".to_owned());
12754 released.requeue();
12755 let unfinished = (!released.fresh_start)
12756 .then(|| Some("stalled-run".to_owned()))
12757 .flatten();
12758 assert_eq!(
12759 choose_starter(None, false, unfinished.as_deref()),
12760 Starter::Start,
12761 "release keeps run history but must not resume it"
12762 );
12763 assert_eq!(released.runs, ["stalled-run"]);
12764 }
12765
12766 #[test]
12767 fn landing_hold_receipt_survives_release_retry_and_checkpoint_removal() {
12768 let dir = tempfile::tempdir().unwrap();
12769 let queue = Queue::at(dir.path().join("queue"));
12770 let mut state = run_state(RunStatus::Blocked);
12771 state.merge_decision = Some(crate::merge_decision::Decision {
12772 question: "hold-inquiry".into(),
12773 target: crate::merge_decision::Target {
12774 pr: "https://example.test/pull/1".into(),
12775 head: "reviewed-head".into(),
12776 language: "en".into(),
12777 },
12778 choice: land::HOLD.into(),
12779 closed: false,
12780 replacement_run: None,
12781 replacement_pr: None,
12782 });
12783 let mut t = task();
12784 t.start(state.id.clone());
12785 settle_and_diagnose(
12786 &mut t,
12787 Verdict {
12788 status: RunStatus::Blocked,
12789 left_pr: true,
12790 quota_hit: false,
12791 parked: false,
12792 no_viable_candidates: false,
12793 },
12794 "held",
12795 2,
12796 &state,
12797 );
12798 queue.put(&mut t).unwrap();
12799 t.release_operator("chat", None);
12800 t.edit("revised".into(), "revised instruction".into())
12801 .unwrap();
12802 queue.put(&mut t).unwrap();
12803 drop(state);
12804 assert_eq!(
12805 queue
12806 .get(&t.id)
12807 .unwrap()
12808 .decision_effects
12809 .get("hold-inquiry"),
12810 Some(&format!("decision: {}", land::HOLD))
12811 );
12812 }
12813
12814 fn stopped_public_run(status: RunStatus) -> RunState {
12815 let mut state = run_state(status);
12816 state.driver_exited = true;
12817 state.config.graph.land = true;
12818 state.config.graph.review_rounds = 1;
12819 state.reviews = vec![review_round(1)];
12820 state.merge = Some(crate::run::MergeOutcome {
12821 mode: MergeMode::Pr,
12822 ok: false,
12823 detail: "landing stopped".into(),
12824 empty: false,
12825 });
12826 state.pr = Some(crate::run::PrRecord {
12827 url: "https://example.test/pull/1".into(),
12828 number: 1,
12829 state: "open".into(),
12830 checks: "red".into(),
12831 round: 1,
12832 rounds: 1,
12833 red_at_merge: Vec::new(),
12834 });
12835 state
12836 }
12837
12838 #[test]
12839 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
12840 for status in [RunStatus::Blocked, RunStatus::Gating] {
12841 let home = tempfile::tempdir().unwrap();
12842 let mut state = stopped_public_run(status);
12843 state.save_under(home.path()).unwrap();
12844 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
12845 assert!(!exhausted_review_budget(&state));
12846 let mut task = held_task_with(&state.id);
12847 task.release();
12848 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
12849 assert_eq!(
12850 choose_starter(None, false, unfinished.as_deref()),
12851 Starter::Resume(state.id.clone())
12852 );
12853 let question = action_question(&state.id, resume_action(&state.id));
12854 assert_eq!(
12855 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
12856 state.clone()
12857 )),
12858 ActionDecision::Resume(state.id.clone())
12859 );
12860 assert_eq!(
12862 crate::resume::validate(home.path(), &state.id).unwrap().id,
12863 state.id
12864 );
12865 let queue = Queue::at(home.path().join("queue"));
12866 let mut controlled = Task::new(
12867 "resume land".into(),
12868 state.instruction.clone(),
12869 state.repo.clone(),
12870 Source::Agent {
12871 run: "talk".into(),
12872 node: crate::queue::CHAT_NODE.into(),
12873 },
12874 );
12875 controlled.runs.push(state.id.clone());
12876 controlled.hold_machine(Some("land stopped".into()));
12877 queue.put(&mut controlled).unwrap();
12878 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
12879 assert_eq!(resumed.status, TaskStatus::Queued);
12880 assert_eq!(
12881 resumed.resume_override.unwrap().pinned_run,
12882 Some(state.id.clone())
12883 );
12884 assert_eq!(resumed.attempts, 0);
12885 }
12886 }
12887
12888 #[test]
12889 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
12890 for review_rounds in [0, 1] {
12891 let home = tempfile::tempdir().unwrap();
12892 let mut state = run_state(RunStatus::Blocked);
12893 state.config.graph.review_rounds = review_rounds;
12894 if review_rounds > 0 {
12895 let mut review = review_round(1);
12896 review.clean = true;
12897 state.reviews = vec![review];
12898 }
12899 state.gate_ran = true;
12900 state.gate = vec![crate::run::CommandOutcome {
12901 command: "verify".into(),
12902 code: Some(1),
12903 output_tail: "gate failed after repair budget exhausted".into(),
12904 duration_ms: 0,
12905 resource_blocked: false,
12906 }];
12907 state.save_under(home.path()).unwrap();
12908 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
12909 assert!(exhausted_review_budget(&state));
12910 let mut task = held_task_with(&state.id);
12911 task.release();
12912 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
12913 let question = action_question(&state.id, resume_action(&state.id));
12914 assert_ne!(
12915 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
12916 state.clone()
12917 )),
12918 ActionDecision::Resume(state.id.clone())
12919 );
12920 let error = crate::resume::validate(home.path(), &state.id)
12921 .unwrap_err()
12922 .to_string();
12923 assert!(!error.contains("review budget exhausted"), "{error}");
12924 assert!(
12925 error.contains(if review_rounds == 0 {
12926 "review is disabled"
12927 } else {
12928 "gate already ran"
12929 }),
12930 "{error}"
12931 );
12932
12933 state.gate_ran = false;
12935 state.gate.clear();
12936 assert!(!exhausted_review_budget(&state));
12937 assert_eq!(
12938 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
12939 Some(state.id)
12940 );
12941 }
12942 }
12943
12944 #[test]
12945 fn resource_blocked_verification_resumes_the_same_run() {
12946 for (kind, stage) in [
12947 ("cache", "clean"),
12948 ("coordination", "clean"),
12949 ("cache", "review-e2e"),
12950 ("coordination", "no-reviews"),
12951 ("coordination", "handoff"),
12952 ] {
12953 let home = tempfile::tempdir().unwrap();
12954 let mut state = run_state(RunStatus::Blocked);
12955 state.config.graph.review_rounds = 1;
12956 let mut review = review_round(1);
12957 review.clean = true;
12958 state.reviews = vec![review];
12959 if stage == "review-e2e" {
12960 let review = &mut state.reviews[0];
12961 review.clean = false;
12962 review.blocking = 1;
12963 review.e2e = vec![crate::run::CommandOutcome {
12964 command: "verify".into(),
12965 code: None,
12966 output_tail: "resource wait expired".into(),
12967 duration_ms: 0,
12968 resource_blocked: true,
12969 }];
12970 } else if stage == "no-reviews" {
12971 state.config.graph.review_rounds = 0;
12972 state.reviews.clear();
12973 } else if stage == "handoff" {
12974 let review = &mut state.reviews[0];
12977 review.clean = false;
12978 review.blocking = 1;
12979 review.e2e = vec![crate::run::CommandOutcome {
12980 command: "verify".into(),
12981 code: Some(0),
12982 output_tail: String::new(),
12983 duration_ms: 0,
12984 resource_blocked: false,
12985 }];
12986 }
12987 state.gate_ran = false;
12988 state.resource_block = Some(crate::run::ResourceBlock {
12989 cause: crate::run::ResourceBlockCause::Other,
12990 node: if stage == "review-e2e" {
12991 "review"
12992 } else {
12993 "gate"
12994 }
12995 .into(),
12996 kind: kind.into(),
12997 detail: "resource wait expired".into(),
12998 at: jiff::Timestamp::now(),
12999 });
13000 state.save_under(home.path()).unwrap();
13001 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
13002 assert!(!exhausted_review_budget(&state));
13003 let mut task = held_task_with(&state.id);
13004 task.release();
13005 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
13006 assert_eq!(
13007 choose_starter(None, false, unfinished.as_deref()),
13008 Starter::Resume(state.id.clone())
13009 );
13010 let question = action_question(&state.id, resume_action(&state.id));
13011 assert_eq!(
13012 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
13013 state.clone()
13014 )),
13015 ActionDecision::Resume(state.id.clone())
13016 );
13017 assert_eq!(
13018 crate::resume::validate(home.path(), &state.id).unwrap().id,
13019 state.id
13020 );
13021 let queue = Queue::at(home.path().join("queue"));
13022 let mut controlled = Task::new(
13023 "resume verification".into(),
13024 state.instruction.clone(),
13025 state.repo.clone(),
13026 Source::Agent {
13027 run: "talk".into(),
13028 node: crate::queue::CHAT_NODE.into(),
13029 },
13030 );
13031 controlled.runs.push(state.id.clone());
13032 controlled.hold_machine(Some("verification resource unavailable".into()));
13033 queue.put(&mut controlled).unwrap();
13034 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
13035 assert_eq!(resumed.status, TaskStatus::Queued);
13036 assert_eq!(
13037 resumed.resume_override.unwrap().pinned_run,
13038 Some(state.id.clone())
13039 );
13040 assert_eq!(resumed.attempts, 0);
13041
13042 state.config.graph.review_rounds = 1;
13044 let mut unresolved = review_round(1);
13045 unresolved.blocking = 1;
13046 state.reviews = vec![unresolved];
13047 state.resource_block = None;
13048 state.save_under(home.path()).unwrap();
13049 assert!(exhausted_review_budget(&state));
13050 assert!(crate::resume::validate(home.path(), &state.id).is_err());
13051 }
13052 }
13053
13054 #[tokio::test]
13055 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
13056 if !crate::standalone_test(
13057 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
13058 ) {
13059 return;
13060 }
13061 let home = tempfile::tempdir().unwrap();
13062 let mut owner = stopped_public_run(RunStatus::Blocked);
13063 owner.save_under(home.path()).unwrap();
13064 let history = vec![owner.id.clone()];
13065 let repo = owner.repo.clone();
13066 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
13067 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
13068 let initial =
13069 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
13070 assert!(
13071 initial.is_pending(),
13072 "a busy record must not hold or resume the task"
13073 );
13074 owner.retired_to = Some("20260101-000001-bbbb".into());
13075 owner.save_under(home.path()).unwrap();
13076 drop(lock);
13077 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
13078 .await
13079 .expect("ownership lock released")
13080 .expect("read settled owner");
13081 assert!(
13082 result.is_none(),
13083 "the retired owner must not be resumed from a stale read"
13084 );
13085 }
13086
13087 #[tokio::test]
13088 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
13089 if !crate::standalone_test(
13090 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
13091 ) {
13092 return;
13093 }
13094 let home = tempfile::tempdir().unwrap();
13095 let mut owner = stopped_public_run(RunStatus::Blocked);
13096 let repo = tempfile::tempdir().unwrap();
13097 crate::git::git(repo.path(), &["init"]).await.unwrap();
13098 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
13099 owner.id = "20260101-000000-aaaa".into();
13100 owner.save_under(home.path()).unwrap();
13101 let mut successor = run_state(RunStatus::Blocked);
13102 successor.id = "20260101-000001-bbbb".into();
13103 successor.save_under(home.path()).unwrap();
13104 let runs = vec![owner.id.clone(), successor.id.clone()];
13105 assert_eq!(
13106 public_resume_owner(home.path(), &runs, &owner.repo)
13107 .await
13108 .unwrap()
13109 .unwrap()
13110 .id,
13111 owner.id
13112 );
13113 let subdir = owner.repo.join("nested");
13114 std::fs::create_dir(&subdir).unwrap();
13115 assert_eq!(
13116 public_resume_owner(home.path(), &runs, &subdir)
13117 .await
13118 .unwrap()
13119 .unwrap()
13120 .id,
13121 owner.id
13122 );
13123 #[cfg(unix)]
13124 {
13125 let alias = home.path().join("repo-alias");
13126 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
13127 assert_eq!(
13128 public_resume_owner(home.path(), &runs, &alias.join("nested"))
13129 .await
13130 .unwrap()
13131 .unwrap()
13132 .id,
13133 owner.id
13134 );
13135 }
13136 let different = tempfile::tempdir().unwrap();
13137 crate::git::git(different.path(), &["init"]).await.unwrap();
13138 assert!(
13139 public_resume_owner(home.path(), &runs, different.path())
13140 .await
13141 .is_err()
13142 );
13143 std::fs::remove_file(
13144 home.path()
13145 .join("runs")
13146 .join(&successor.id)
13147 .join("run.json"),
13148 )
13149 .unwrap();
13150 assert!(
13151 public_resume_owner(home.path(), &runs, &owner.repo)
13152 .await
13153 .is_err()
13154 );
13155 }
13156
13157 #[test]
13158 fn an_ordinary_release_keeps_a_resumable_run_available() {
13159 let mut released = task();
13160 released.start("stalled-run".to_owned());
13161 released.release();
13162 let unfinished = (!released.fresh_start)
13163 .then(|| Some("stalled-run".to_owned()))
13164 .flatten();
13165 assert_eq!(
13166 choose_starter(None, false, unfinished.as_deref()),
13167 Starter::Resume("stalled-run".to_owned()),
13168 "manual release must preserve the normal resume path"
13169 );
13170 }
13171
13172 #[test]
13173 fn resume_refusals_explain_admission_without_changing_task_or_run() {
13174 use crate::handover::{ResumeRefusal, resume_refusal_under};
13175 let home = tempfile::tempdir().unwrap();
13176 let mut exhausted = run_state(RunStatus::Blocked);
13177 exhausted.driver_exited = true;
13178 exhausted.config.graph.review_rounds = 3;
13179 exhausted.reviews = (1..=3).map(review_round).collect();
13180 for refusal in [
13181 ResumeRefusal::ReviewBudget { used: 3, limit: 3 },
13182 ResumeRefusal::Status(RunStatus::Ready),
13183 ResumeRefusal::Released("20261022-000000-aaaa".into()),
13184 ResumeRefusal::Retired("retirement-successor".into()),
13185 ResumeRefusal::OwnershipTransferred("20261022-000000-bbbb".into()),
13186 ] {
13187 let mut state = exhausted.clone();
13188 let expected = match refusal {
13189 ResumeRefusal::Status(_) => {
13190 state.status = RunStatus::Ready;
13191 "status `ready`"
13192 }
13193 ResumeRefusal::Released(ref target) => {
13194 state.released_to = Some(target.clone());
13195 "execution was released"
13196 }
13197 ResumeRefusal::Retired(ref target) => {
13198 state.retired_to = Some(target.clone());
13199 "execution was retired"
13200 }
13201 ResumeRefusal::OwnershipTransferred(ref target) => {
13202 state.pr_taken_over_by = Some(target.clone());
13203 "PR ownership was transferred"
13204 }
13205 _ => "review budget exhausted (3/3 rounds used)",
13206 };
13207 assert_eq!(
13208 resume_refusal_under(&state, Some(home.path())),
13209 Some(refusal.clone())
13210 );
13211 assert_eq!(
13212 refusal.successor(),
13213 state
13214 .released_to
13215 .as_deref()
13216 .or(state.retired_to.as_deref())
13217 .or(state.pr_taken_over_by.as_deref())
13218 );
13219 state.save_under(home.path()).unwrap();
13220 let queue = Queue::at(home.path().join("queue"));
13221 let mut task = Task::new(
13222 "retained implementation".into(),
13223 state.instruction.clone(),
13224 state.repo.clone(),
13225 Source::Agent {
13226 run: "talk".into(),
13227 node: crate::queue::CHAT_NODE.into(),
13228 },
13229 );
13230 task.runs.push(state.id.clone());
13231 task.hold_machine(Some("stopped".into()));
13232 queue.put(&mut task).unwrap();
13233 let before_task = serde_json::to_value(queue.get(&task.id).unwrap()).unwrap();
13234 let before_run = serde_json::to_value(&state).unwrap();
13235 let error = crate::chat_control::resume(&queue, home.path(), &state.id)
13236 .unwrap_err()
13237 .to_string();
13238 assert!(error.contains(expected), "{error}");
13239 assert!(!error.contains("only a stalled or blocked"), "{error}");
13240 if matches!(refusal, ResumeRefusal::Status(_)) {
13241 assert!(
13242 error.contains("is final; there is nothing left to resume"),
13243 "{error}"
13244 );
13245 }
13246 if matches!(refusal, ResumeRefusal::ReviewBudget { .. }) {
13247 assert!(error.contains("magi review <saved-branch> --task <task-id> --merge none"));
13248 assert!(error.contains("review-only successor with a new review budget"));
13249 assert!(error.contains("does not resume this run"));
13250 }
13251 for target in [
13252 &state.released_to,
13253 &state.retired_to,
13254 &state.pr_taken_over_by,
13255 ]
13256 .into_iter()
13257 .flatten()
13258 {
13259 let short = if state.retired_to.as_ref() == Some(target) {
13260 target.as_str()
13261 } else {
13262 crate::run::short_of(target)
13263 };
13264 assert!(
13265 error.contains(&format!("run {short}; inspect `magi show {short}`")),
13266 "{error}"
13267 );
13268 if short != target {
13269 assert!(!error.contains(target), "full successor id leaked: {error}");
13270 }
13271 }
13272 assert_eq!(
13273 serde_json::to_value(queue.get(&task.id).unwrap()).unwrap(),
13274 before_task
13275 );
13276 assert_eq!(
13277 serde_json::to_value(RunState::load_under(&state.id, home.path()).unwrap())
13278 .unwrap(),
13279 before_run
13280 );
13281 let question = action_question(&state.id, resume_action(&state.id));
13282 for phrases in [&PHRASES_EN, &PHRASES_JA] {
13283 let decision = decide_action(home.path(), &task, &question, phrases, |id| {
13284 RunState::load_under(id, home.path())
13285 });
13286 let ActionDecision::Refuse(detail) = decision else {
13287 panic!(
13288 "refusal expected: decision={decision:?}, refusal={refusal:?}, run={}, home={}",
13289 state.id,
13290 home.path().display()
13291 );
13292 };
13293 assert!(detail.contains(expected), "{detail}");
13294 }
13295 }
13296 exhausted.status = RunStatus::Ready;
13298 exhausted.released_to = Some("released".into());
13299 exhausted.retired_to = Some("retired".into());
13300 exhausted.pr_taken_over_by = Some("owner".into());
13301 let reason = resume_refusal_under(&exhausted, Some(home.path())).unwrap();
13302 assert_eq!(reason, ResumeRefusal::Status(RunStatus::Ready));
13303 let detail = reason.diagnostic(&exhausted);
13304 for target in ["released", "retired", "owner"] {
13305 assert!(detail.contains(&format!("magi show {target}")));
13306 }
13307 }
13308
13309 #[test]
13310 fn resume_action_ignores_question_checkpoints_in_another_home() {
13311 crate::run::pin_test_home();
13312 let home = tempfile::tempdir().unwrap();
13313 let mut state = run_state(RunStatus::Blocked);
13314 state.id = format!(
13317 "resume-context-{}",
13318 home.path().file_name().unwrap().to_str().unwrap()
13319 );
13320 state.driver_exited = true;
13321 state.config.graph.review_rounds = 1;
13322 state.reviews = vec![review_round(1)];
13323 state.save_under(home.path()).unwrap();
13324 let task = held_task_with(&state.id);
13325 let answer = action_question(&state.id, resume_action(&state.id));
13326 let mut checkpoint = approval_question(&state.id);
13327 let foreign = Questions::open();
13328 foreign.put(&mut checkpoint).unwrap();
13329 let refusal = crate::handover::resume_refusal_under(&state, Some(home.path()));
13330 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
13331 RunState::load_under(id, home.path())
13332 });
13333 let matching: Vec<_> = foreign
13334 .list()
13335 .into_iter()
13336 .filter(|q| q.run == state.id && q.status.open())
13337 .collect();
13338 std::fs::remove_file(foreign.path_of(&checkpoint.id)).unwrap();
13339 assert!(
13340 matches!(decision, ActionDecision::Refuse(_)),
13341 "decision={decision:?}, refusal={refusal:?}, run={}, home={}, foreign_home={}, matching={matching:?}",
13342 state.id,
13343 home.path().display(),
13344 crate::run::test_home().display()
13345 );
13346 assert_eq!(
13347 refusal,
13348 Some(crate::handover::ResumeRefusal::ReviewBudget { used: 1, limit: 1 })
13349 );
13350 let local = Questions::at(home.path().join("questions"));
13351 for status in [
13352 ask::QuestionStatus::Open,
13353 ask::QuestionStatus::Answered,
13354 ask::QuestionStatus::Abandoned,
13355 ] {
13356 let mut question = checkpoint.clone();
13357 match status {
13358 ask::QuestionStatus::Open => {}
13359 ask::QuestionStatus::Answered => {
13360 question
13361 .answer(ask::Answer::Choice(question.choices[0].clone()))
13362 .unwrap();
13363 }
13364 ask::QuestionStatus::Abandoned => question.abandon("fixture closed"),
13365 }
13366 local.put(&mut question).unwrap();
13367 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
13368 RunState::load_under(id, home.path())
13369 });
13370 if status.open() {
13371 assert_eq!(decision, ActionDecision::Resume(state.id.clone()));
13372 } else {
13373 assert!(
13374 matches!(decision, ActionDecision::Refuse(ref detail) if detail.contains("review budget exhausted")),
13375 "status={status:?}, decision={decision:?}"
13376 );
13377 }
13378 }
13379 }
13380
13381 #[test]
13382 fn resume_action_checks_the_driver_lease_in_the_supplied_home() {
13383 let home = tempfile::tempdir().unwrap();
13384 let mut state = run_state(RunStatus::Stalled);
13385 state.id = format!(
13386 "resume-driver-{}",
13387 home.path().file_name().unwrap().to_str().unwrap()
13388 );
13389 state.driver_claimed = Some(false);
13390 state.save_under(home.path()).unwrap();
13391 let task = held_task_with(&state.id);
13392 let question = action_question(&state.id, resume_action(&state.id));
13393 let decide = || {
13394 decide_action(home.path(), &task, &question, &PHRASES_EN, |id| {
13395 RunState::load_under(id, home.path())
13396 })
13397 };
13398 let lease = crate::driver::maintenance(home.path(), &state.id).unwrap();
13399 assert_eq!(state.liveness_under(home.path(), false), Liveness::Live);
13400 assert_eq!(
13401 decide(),
13402 ActionDecision::Refuse("run already has a live driver".into())
13403 );
13404 drop(lease);
13405 assert_eq!(decide(), ActionDecision::Resume(state.id.clone()));
13406 }
13407
13408 #[test]
13409 fn resume_checkpoint_admission_uses_the_supplied_question_store() {
13410 let home = tempfile::tempdir().unwrap();
13411 let mut state = run_state(RunStatus::Blocked);
13412 state.driver_exited = true;
13413 state.config.graph.review_rounds = 1;
13414 state.reviews = vec![review_round(1)];
13415 assert!(crate::handover::resume_refusal_under(&state, Some(home.path())).is_some());
13416 for checkpoint in ["publication", "approval", "armed", "landing", "question"] {
13417 let mut saved = state.clone();
13418 match checkpoint {
13419 "publication" => saved.pr_publication_pending = true,
13420 "approval" => {
13421 saved.land_approval = Some(crate::run::LandApproval {
13422 question: "approval".into(),
13423 head: "head".into(),
13424 })
13425 }
13426 "armed" => saved.land_armed_head = Some("head".into()),
13427 "landing" => saved.status = RunStatus::Landing,
13428 _ => {
13429 let mut question = ask::Question::new(
13430 saved.id.clone(),
13431 "review".into(),
13432 "review-1".into(),
13433 "continue?".into(),
13434 String::new(),
13435 vec!["continue".into()],
13436 );
13437 ask::Questions::at(home.path().join("questions"))
13438 .put(&mut question)
13439 .unwrap();
13440 }
13441 }
13442 saved.save_under(home.path()).unwrap();
13443 assert!(
13444 crate::handover::resume_refusal_under(&saved, Some(home.path())).is_none(),
13445 "{checkpoint}"
13446 );
13447 assert!(
13448 crate::resume::validate(home.path(), &saved.id).is_ok(),
13449 "{checkpoint}"
13450 );
13451 }
13452 }
13453
13454 #[test]
13455 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
13456 let mut state = run_state(RunStatus::Blocked);
13457 state.config.graph.review_rounds = 3;
13458 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
13459 assert!(exhausted_review_budget(&state));
13460
13461 state.reviews.pop();
13463 assert!(!exhausted_review_budget(&state));
13464
13465 let mut stalled = run_state(RunStatus::Stalled);
13468 stalled.config.graph.review_rounds = 1;
13469 stalled.reviews = vec![review_round(1)];
13470 assert!(!exhausted_review_budget(&stalled));
13471 }
13472
13473 fn review_round(round: usize) -> crate::run::ReviewRound {
13474 crate::run::ReviewRound {
13475 round,
13476 head: "deadbeef".to_owned(),
13477 verified_head: None,
13478 verified_at: None,
13479 reviews: Vec::new(),
13480 e2e: Vec::new(),
13481 verify_retried: false,
13482 e2e_deferred: false,
13483 e2e_defer_reason: None,
13484 fix: None,
13485 blocking: 0,
13486 answered: 1,
13487 expected: 1,
13488 clean: false,
13489 progressed: true,
13490 vote_split: false,
13491 reconsideration: Vec::new(),
13492 verdict: None,
13493 }
13494 }
13495
13496 #[test]
13497 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
13498 let mut run = RunState::new(
13499 PathBuf::from("/repo"),
13500 "main".to_owned(),
13501 "abc1234def".to_owned(),
13502 "add retries".to_owned(),
13503 Config::default(),
13504 );
13505 run.status = RunStatus::Judging;
13506 run.parked = true;
13507 let mut task = Task::new(
13508 "add retries".to_owned(),
13509 "add retries".to_owned(),
13510 PathBuf::from("/repo"),
13511 crate::queue::Source::Human,
13512 );
13513 task.status = TaskStatus::Failed;
13514 task.runs = vec![run.id.clone()];
13515 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
13516 assert!(with(&task, &run), "parked after judging is the case");
13517 let mut parked_task = task.clone();
13518 parked_task.status = TaskStatus::Parked;
13519 assert!(with(&parked_task, &run), "the Parked status is the case");
13520
13521 let mut not_parked = run.clone();
13522 not_parked.parked = false;
13523 not_parked.status = RunStatus::Stalled;
13524 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
13525
13526 let mut fresh = task.clone();
13527 fresh.fresh_start = true;
13528 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
13529
13530 let mut review = task.clone();
13531 review.review_branch = Some("magi/x/A".to_owned());
13532 assert!(!with(&review, &run), "review is ranked before resume");
13533
13534 let mut held = task.clone();
13535 held.status = TaskStatus::Held;
13536 assert!(!with(&held, &run), "a hold stays visible to the conductor");
13537
13538 let mut released = run.clone();
13539 released.released_to = Some("20260901-000000-new1".to_owned());
13540 assert!(!with(&task, &released), "nothing left to resume into");
13541
13542 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
13543 "unreadable"
13544 )));
13545 }
13546
13547 #[test]
13548 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
13549 let mut released = RunState::new(
13550 PathBuf::from("/repo"),
13551 "main".to_owned(),
13552 "abc1234def".to_owned(),
13553 "add retries".to_owned(),
13554 Config::default(),
13555 );
13556 released.status = RunStatus::Blocked;
13557 assert_eq!(
13558 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
13559 Some(released.id.clone())
13560 );
13561 released.released_to = Some("20260901-000000-new1".to_owned());
13562 assert_eq!(
13563 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
13564 None,
13565 "there is nothing left to resume it into"
13566 );
13567 }
13568
13569 #[test]
13570 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
13571 let mut exhausted = RunState::new(
13580 PathBuf::from("/repo"),
13581 "main".to_owned(),
13582 "abc1234def".to_owned(),
13583 "add retries".to_owned(),
13584 Config::default(),
13585 );
13586 exhausted.status = RunStatus::Blocked;
13587 exhausted.config.graph.review_rounds = 1;
13588 exhausted.reviews = vec![review_round(1)];
13589
13590 assert_eq!(
13591 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
13592 None,
13593 "an exhausted `Blocked` run must not be offered as resumable"
13594 );
13595
13596 let mut has_budget_left = RunState::new(
13599 PathBuf::from("/repo"),
13600 "main".to_owned(),
13601 "abc1234def".to_owned(),
13602 "add retries".to_owned(),
13603 Config::default(),
13604 );
13605 has_budget_left.status = RunStatus::Blocked;
13606 has_budget_left.config.graph.review_rounds = 3;
13607 has_budget_left.reviews = vec![review_round(1)];
13608
13609 assert_eq!(
13610 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
13611 Ok(has_budget_left.clone())
13612 }),
13613 Some(has_budget_left.id.clone())
13614 );
13615 }
13616
13617 #[test]
13618 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
13619 let mut run = RunState::new(
13620 PathBuf::from("/repo"),
13621 "main".to_owned(),
13622 "abc1234def".to_owned(),
13623 "add retries".to_owned(),
13624 Config::default(),
13625 );
13626 run.status = RunStatus::Blocked;
13627 run.config.graph.review_rounds = 1;
13628 run.reviews = vec![review_round(1)];
13629 run.pr_publication_pending = true;
13630 assert_eq!(
13631 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
13632 Some(run.id.clone()),
13633 "a spent review budget must not turn a failed publication into a re-competition"
13634 );
13635 }
13636
13637 #[test]
13638 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
13639 let mut task = Task::new(
13640 "recover work".into(),
13641 "recover work".into(),
13642 PathBuf::from("/repo"),
13643 crate::queue::Source::Human,
13644 );
13645 let mut history = std::collections::BTreeMap::new();
13646 for index in 0..8 {
13647 let mut run = run_state(RunStatus::Blocked);
13648 run.id = format!("20260101-000000-{index:04}");
13649 run.base_commit = format!("upstream-{index}");
13650 if index > 0 {
13652 run.candidates.clear();
13653 }
13654 task.runs.extend([run.id.clone(), run.id.clone()]);
13655 history.insert(run.id.clone(), run);
13656 task.requeue();
13657 }
13658 let check = |task: &Task, starter: &Starter, limit| {
13659 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
13660 };
13661 assert!(
13662 check(&task, &Starter::Start, 8)
13663 .unwrap()
13664 .contains("8 unretired")
13665 );
13666 assert!(check(&task, &Starter::Start, 9).is_none());
13667 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
13668 task.release_operator("owner", None);
13669 assert_eq!(task.attempts, 0);
13670 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
13671 assert!(check(&restored, &Starter::Start, 8).is_some());
13672 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
13673 assert!(check(&restored, &Starter::Start, 0).is_none());
13674 }
13675
13676 #[test]
13677 fn retained_attempt_bound_includes_failed_startups_without_execution_history() {
13678 let tmp = tempfile::tempdir().unwrap();
13679 let mut task = Task::new(
13680 "recover selected PR".into(),
13681 "preserve implementation".into(),
13682 tmp.path().join("repo"),
13683 crate::queue::Source::Human,
13684 );
13685 let mut failed = run_state(RunStatus::Failed);
13686 failed.id = "20260101-000000-fail".into();
13687 failed.candidates.clear();
13688 failed.prep_complete = Some(false);
13689 failed.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13690 failed.startup = Some(crate::run::Startup {
13691 kind: "pr-recovery".into(),
13692 branch: Some("selected-public-head".into()),
13693 stage: "worktree-setup".into(),
13694 complete: false,
13695 failure: Some("setup modified a tracked file and failed".into()),
13696 });
13697 let worktree = failed.worktree_root().join("under-review");
13700 std::fs::create_dir_all(&worktree).unwrap();
13701 let retained_edit = worktree.join("implementation");
13702 std::fs::write(&retained_edit, "retained setup edit").unwrap();
13703 task.record_startup_failure(crate::run::StartupFailure {
13704 run: failed.id.clone(),
13705 stage: "worktree-setup".into(),
13706 });
13707 let task: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
13708 assert!(task.runs.is_empty() && task.passes.is_empty());
13709 assert_eq!(task.attempts, 0);
13710 let starter = Starter::Review("selected-public-head".into());
13711 assert!(
13712 retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone()))
13713 .unwrap()
13714 .contains("1 unretired")
13715 );
13716 assert!(retained_attempt_hold(&task, &starter, 2, |_| Ok(failed.clone())).is_none());
13717 assert!(
13718 retained_attempt_hold(&task, &starter, 2, |_| anyhow::bail!("unreadable"))
13719 .unwrap()
13720 .contains("Cannot verify")
13721 );
13722 let mut mixed = task.clone();
13724 mixed.runs.push(failed.id.clone());
13725 mixed
13726 .startup_failures
13727 .push(task.startup_failures[0].clone());
13728 assert!(retained_attempt_hold(&mixed, &starter, 2, |_| Ok(failed.clone())).is_none());
13729 failed.retired_to = Some("verified-recovery-owner".into());
13730 assert!(retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone())).is_none());
13731 assert!(
13732 retained_attempt_hold(&task, &Starter::Resume(failed.id.clone()), 1, |_| {
13733 panic!("resuming the existing owner must not allocate a new attempt")
13734 })
13735 .is_none()
13736 );
13737 assert_eq!(
13738 std::fs::read_to_string(retained_edit).unwrap(),
13739 "retained setup edit"
13740 );
13741 assert!(task.runs.is_empty() && task.passes.is_empty());
13742 assert_eq!(task.attempts, 0);
13743 }
13744
13745 #[test]
13746 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
13747 let mut task = Task::new(
13748 "recover work".into(),
13749 "recover work".into(),
13750 PathBuf::from("/repo"),
13751 crate::queue::Source::Human,
13752 );
13753 let mut run = run_state(RunStatus::Superseded);
13754 run.candidates.clear();
13755 task.runs.push(run.id.clone());
13756 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
13757 run.retired_to = Some("authority".into());
13758 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
13759 run.retired_to = None;
13760 run.released_to = Some("authority".into());
13761 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
13762 assert!(
13763 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
13764 .unwrap()
13765 .contains("Cannot verify")
13766 );
13767 assert!(
13769 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
13770 panic!("resume must not load historical attempts")
13771 })
13772 .is_none()
13773 );
13774 }
13775
13776 #[test]
13777 fn retired_resume_requires_a_valid_same_repository_authority() {
13778 let mut source = run_state(RunStatus::Blocked);
13779 let mut empty = run_state(RunStatus::Blocked);
13780 empty.retired_to = Some(source.id.clone());
13781 let runs = vec![source.id.clone(), empty.id.clone()];
13782 let resolve = |source: &RunState, empty: &RunState| {
13783 unfinished_run_with(&runs, "fixture", |id| {
13784 if id == source.id {
13785 Ok(source.clone())
13786 } else if id == empty.id {
13787 Ok(empty.clone())
13788 } else {
13789 bail!("unreadable authority")
13790 }
13791 })
13792 };
13793 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
13794 assert_eq!(
13795 choose_starter(None, false, resolve(&source, &empty).as_deref()),
13796 Starter::Resume(source.id.clone())
13797 );
13798 source.retired_to = Some(empty.id.clone());
13799 assert!(
13800 resolve(&source, &empty).is_none(),
13801 "cycles do not revive obsolete runs"
13802 );
13803 source.retired_to = None;
13804 source.repo = PathBuf::from("/another-repository");
13805 assert!(resolve(&source, &empty).is_none());
13806 source.repo = empty.repo.clone();
13807 source.released_to = Some("later".into());
13808 assert!(resolve(&source, &empty).is_none());
13809 empty.retired_to = Some("unlisted-run".into());
13810 assert!(resolve(&source, &empty).is_none());
13811 }
13812
13813 #[test]
13814 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
13815 let mut older_stalled = RunState::new(
13823 PathBuf::from("/repo"),
13824 "main".to_owned(),
13825 "abc1234def".to_owned(),
13826 "add retries".to_owned(),
13827 Config::default(),
13828 );
13829 older_stalled.status = RunStatus::Stalled;
13830
13831 let mut newest_exhausted = RunState::new(
13832 PathBuf::from("/repo"),
13833 "main".to_owned(),
13834 "abc1234def".to_owned(),
13835 "add retries".to_owned(),
13836 Config::default(),
13837 );
13838 newest_exhausted.status = RunStatus::Blocked;
13839 newest_exhausted.config.graph.review_rounds = 1;
13840 newest_exhausted.reviews = vec![review_round(1)];
13841
13842 assert_eq!(
13843 unfinished_run_with(
13844 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
13845 "t",
13846 |_| Ok(newest_exhausted.clone())
13847 ),
13848 None,
13849 "the newest run is exhausted, so nothing here is worth resuming - \
13850 least of all the older, already-superseded run"
13851 );
13852 }
13853
13854 #[test]
13855 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
13856 assert_eq!(
13857 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
13858 Err(anyhow::anyhow!("fixture is absent"))
13859 }),
13860 None
13861 );
13862 }
13863
13864 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
13865 let mut q = ask::Question::new(
13866 run.to_owned(),
13867 "implement".to_owned(),
13868 "impl-A".to_owned(),
13869 "continue?".to_owned(),
13870 String::new(),
13871 vec!["resume で続行する".to_owned(), "other".to_owned()],
13872 );
13873 q.actions.insert("resume で続行する".to_owned(), action);
13874 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
13875 .unwrap();
13876 q
13877 }
13878
13879 fn held_task_with(run: &str) -> Task {
13880 let mut t = task();
13881 t.runs = vec![run.to_owned()];
13882 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
13883 t
13884 }
13885
13886 fn resume_action(run: &str) -> ask::ChoiceAction {
13887 ask::ChoiceAction::Resume { run: run.into() }
13888 }
13889
13890 #[test]
13891 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
13892 let home = crate::run::test_home();
13893 for parked in [false, true] {
13894 let mut t = task();
13895 t.start("A".into());
13896 t.start("B".into());
13897 assert_eq!(t.runs, vec!["A", "B"]);
13898 t.start("A".into());
13899 let mut resumed = run_state(RunStatus::Blocked);
13900 resumed.id = "A".into();
13901 resumed.parked = parked;
13902 resumed.driver_exited = true;
13903 t.hold_machine(Some("resumed run stopped".into()));
13904 let q = action_question("A", resume_action("A"));
13905 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
13906 assert_eq!(
13907 decide_action(&home, &t, &q, &PHRASES_EN, |id| {
13908 assert_eq!(id, "A");
13909 Ok(resumed.clone())
13910 }),
13911 ActionDecision::Resume("A".into())
13912 );
13913 assert_eq!(
13914 action_standing(&t, &action_question("B", resume_action("B"))),
13915 ActionStanding::Stale
13916 );
13917 t.release_operator("cli", None);
13918 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
13919 assert_eq!(id, "A");
13920 Ok(resumed.clone())
13921 });
13922 assert_eq!(
13923 choose_starter(None, false, unfinished.as_deref()),
13924 Starter::Resume("A".into())
13925 );
13926 assert_eq!(t.runs, vec!["B", "A"]);
13927 }
13928 }
13929
13930 #[test]
13931 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
13932 let home = crate::run::test_home();
13933 let mut state = crate::land::tests::repair_state(true, true);
13934 state.status = RunStatus::Landing;
13935 state.parked = true;
13936 state.driver_exited = true;
13937 let mut q = ask::Question::new(
13938 state.id.clone(),
13939 land::APPROVAL_NODE.into(),
13940 "land".into(),
13941 "Merge?".into(),
13942 String::new(),
13943 vec![land::APPROVE.into(), land::HOLD.into()],
13944 );
13945 q.id = state.id.clone();
13946 state.land_approval = Some(crate::run::LandApproval {
13947 question: q.id.clone(),
13948 head: "current-head".into(),
13949 });
13950 assert!(land::upgrade_repair_choice(&mut q, &state));
13951 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
13952 let mut task = held_task_with(&state.id);
13953 task.status = TaskStatus::Parked;
13954 assert_eq!(
13955 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
13956 ActionDecision::Resume(state.id.clone())
13957 );
13958 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
13959 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
13960 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
13961 assert_eq!(
13962 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
13963 ActionDecision::Resume(state.id.clone()),
13964 "a repair answer newer than the manual hold resumes the existing owner"
13965 );
13966 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
13967 for answered_at in [q.answered_at, None] {
13968 let mut older = q.clone();
13969 older.answered_at = answered_at;
13970 let refused = decide_action(&home, &task, &older, &PHRASES_EN, |_| {
13971 panic!("a later manual hold must prevent dispatch")
13972 });
13973 let ActionDecision::Refuse(reason) = refused else {
13974 panic!("expected an actionable refusal, got {refused:?}");
13975 };
13976 assert!(reason.contains("later operator decision"));
13977 assert!(reason.contains(&format!("magi task release {}", task.id)));
13978 }
13979 task.operator_decided_at = None;
13980 let mut superseded = state.clone();
13981 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
13982 assert_eq!(
13983 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(superseded)),
13984 ActionDecision::Stale
13985 );
13986 for case in 1..4 {
13987 let mut changed = state.clone();
13988 match case {
13989 1 => changed.status = RunStatus::Blocked,
13990 2 => changed.released_to = Some("new-owner".into()),
13991 _ => changed.land_approval.as_mut().unwrap().head.clear(),
13992 }
13993 assert!(matches!(
13994 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(changed)),
13995 ActionDecision::Refuse(_)
13996 ));
13997 }
13998 assert!(matches!(
13999 decide_action(&home, &task, &q, &PHRASES_EN, |_| anyhow::bail!(
14000 "missing owner"
14001 )),
14002 ActionDecision::Refuse(_)
14003 ));
14004 task.runs.push("replacement".into());
14005 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
14006 assert_eq!(
14007 decide_action(&home, &task, &q, &PHRASES_EN, |_| panic!(
14008 "old owner must not load"
14009 )),
14010 ActionDecision::Stale
14011 );
14012 }
14013
14014 #[test]
14015 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
14016 let mut state = crate::land::tests::repair_state(true, true);
14017 state.status = RunStatus::Landing;
14018 state.parked = true;
14019 state.driver_exited = true;
14020 let mut q = ask::Question::new(
14021 state.id.clone(),
14022 land::APPROVAL_NODE.into(),
14023 "land".into(),
14024 "Merge?".into(),
14025 String::new(),
14026 vec![land::APPROVE.into(), land::HOLD.into()],
14027 );
14028 q.id = state.id.clone();
14029 state.land_approval = Some(crate::run::LandApproval {
14030 question: q.id.clone(),
14031 head: "current-head".into(),
14032 });
14033 state.save().unwrap();
14034 land::upgrade_repair_choice(&mut q, &state);
14035 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
14036 Questions::open().put(&mut q).unwrap();
14037 let tmp = tempfile::tempdir().unwrap();
14038 let queue = Queue::at(tmp.path().join("queue"));
14039 let mut task = held_task_with(&state.id);
14040 task.status = TaskStatus::Parked;
14041 queue.put(&mut task).unwrap();
14042 let empty = Arc::new(tokio::sync::Semaphore::new(0));
14043 let free = Arc::new(tokio::sync::Semaphore::new(1));
14044 assert!(matches!(
14045 claim_dispatch(&queue, &task, false, &empty, &free),
14046 Err(SkipReason::OrdinaryCapacity)
14047 ));
14048 assert!(matches!(
14049 claim_dispatch(&queue, &task, true, &free, &free),
14050 Err(SkipReason::CoolingDown)
14051 ));
14052 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
14053 assert!(claim.permit.is_some());
14054 assert!(matches!(
14055 claim_dispatch(&queue, &task, false, &free, &free),
14056 Err(SkipReason::ClaimUnavailable)
14057 ));
14058 drop(claim);
14059 Questions::open()
14060 .update(&q.id, |q| {
14061 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
14062 Ok(())
14063 })
14064 .unwrap();
14065 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
14066 assert!(claim.permit.is_none());
14067 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
14068 }
14069
14070 #[test]
14071 fn decide_action_resumes_only_the_latest_resumable_run() {
14072 let home = crate::run::test_home();
14073 let t = held_task_with("r1");
14074 let q = action_question("r1", resume_action("r1"));
14075 let load = |s: RunState| move |_: &str| Ok(s);
14076 assert_eq!(
14077 decide_action(
14078 &home,
14079 &t,
14080 &q,
14081 &PHRASES_EN,
14082 load(run_state(RunStatus::Blocked))
14083 ),
14084 ActionDecision::Resume("r1".into())
14085 );
14086 let q_other = action_question("r1", resume_action("r0"));
14088 assert!(matches!(
14089 decide_action(
14090 &home,
14091 &t,
14092 &q_other,
14093 &PHRASES_EN,
14094 load(run_state(RunStatus::Blocked))
14095 ),
14096 ActionDecision::Refuse(_)
14097 ));
14098 assert!(matches!(
14100 decide_action(
14101 &home,
14102 &t,
14103 &q,
14104 &PHRASES_EN,
14105 load(run_state(RunStatus::Ready))
14106 ),
14107 ActionDecision::Refuse(_)
14108 ));
14109 let mut released = run_state(RunStatus::Blocked);
14111 released.released_to = Some("elsewhere".into());
14112 assert!(matches!(
14113 decide_action(&home, &t, &q, &PHRASES_EN, load(released)),
14114 ActionDecision::Refuse(_)
14115 ));
14116 assert!(matches!(
14118 decide_action(&home, &t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
14119 ActionDecision::Refuse(_)
14120 ));
14121 }
14122
14123 #[test]
14124 fn decide_action_ignores_a_question_about_an_earlier_run() {
14125 let home = crate::run::test_home();
14126 let mut t = held_task_with("r1");
14127 t.runs.push("r2".to_owned());
14128 let never = |_: &str| -> Result<RunState> { bail!("not read") };
14129 assert_eq!(
14130 decide_action(
14131 &home,
14132 &t,
14133 &action_question("r1", ask::ChoiceAction::Done),
14134 &PHRASES_EN,
14135 never
14136 ),
14137 ActionDecision::Stale
14138 );
14139 }
14140
14141 #[test]
14142 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
14143 let home = crate::run::test_home();
14144 let mut t = held_task_with("r1");
14145 let never = |_: &str| -> Result<RunState> { bail!("not read") };
14146 assert_eq!(
14147 decide_action(
14148 &home,
14149 &t,
14150 &action_question("r1", ask::ChoiceAction::Requeue),
14151 &PHRASES_EN,
14152 never
14153 ),
14154 ActionDecision::Requeue
14155 );
14156 let done_q = action_question("r1", ask::ChoiceAction::Done);
14157 assert_eq!(
14158 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
14159 ActionDecision::Done
14160 );
14161 t.mark_action_applied(&done_q.id);
14162 assert_eq!(
14163 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
14164 ActionDecision::Skip
14165 );
14166
14167 let mut plain = action_question("r1", ask::ChoiceAction::Done);
14169 plain.actions.clear();
14170 assert_eq!(
14171 decide_action(&home, &held_task_with("r1"), &plain, &PHRASES_EN, never),
14172 ActionDecision::Skip
14173 );
14174 let mut running = held_task_with("r1");
14176 running.status = TaskStatus::Running;
14177 assert_eq!(
14178 decide_action(
14179 &home,
14180 &running,
14181 &action_question("r1", ask::ChoiceAction::Done),
14182 &PHRASES_EN,
14183 never
14184 ),
14185 ActionDecision::Skip
14186 );
14187 }
14188
14189 #[test]
14190 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
14191 crate::run::pin_test_home();
14192 for legacy in [false, true] {
14193 let dir = tempfile::tempdir().unwrap();
14194 let queue = Queue::at(dir.path().join("queue"));
14195 let questions = Questions::at(dir.path().join("questions"));
14196 let run = "20260101-000000-hold";
14197 let mut task = held_task_with(run);
14198 task.status = TaskStatus::Parked;
14199 let mut q = ask::Question::new(
14200 run.into(),
14201 land::APPROVAL_NODE.into(),
14202 "land".into(),
14203 "Repair?".into(),
14204 String::new(),
14205 vec![land::REPAIR.into()],
14206 );
14207 q.actions.insert(land::REPAIR.into(), resume_action(run));
14208 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
14209 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
14211 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
14212 questions.put(&mut q).unwrap();
14213 task.hold_manual(Some("Keep this PR paused".into()));
14214 if legacy {
14215 task.hold_source = None;
14216 }
14217 queue.put(&mut task).unwrap();
14218
14219 apply_task_choice_actions(&queue, &questions, dir.path());
14220 let after = queue.get(&task.id).unwrap();
14221 assert!(after.operator_held());
14222 assert_eq!(after.hold_source, task.hold_source);
14223 assert_eq!(after.operator_decided_at, task.operator_decided_at);
14224 assert_eq!(after.runs, task.runs);
14225 assert_eq!(after.instruction, task.instruction);
14226 assert!(after.resume_override.is_none());
14227 assert!(after.action_applied(&q.id));
14228 let reason = after.hold_reason.as_deref().unwrap();
14229 assert!(reason.starts_with("Keep this PR paused"));
14230 assert!(reason.contains("later operator decision"));
14231 assert!(reason.contains(&format!("magi task release {}", task.id)));
14232
14233 for recovery in [
14234 crate::conduct::Recovery::Review,
14235 crate::conduct::Recovery::Requeue,
14236 ] {
14237 crate::conduct::apply(
14238 &queue,
14239 &questions,
14240 &crate::conduct::Verdict {
14241 decisions: vec![crate::conduct::Decision {
14242 id: task.id.clone(),
14243 recovery: Some(recovery),
14244 ..Default::default()
14245 }],
14246 },
14247 )
14248 .unwrap();
14249 }
14250 apply_task_choice_actions(&queue, &questions, dir.path());
14253 let still_held = queue.get(&task.id).unwrap();
14254 assert!(still_held.operator_held());
14255 assert_eq!(still_held.hold_reason, after.hold_reason);
14256 assert_eq!(still_held.runs, task.runs);
14257 assert!(still_held.review_branch.is_none());
14258 assert!(!still_held.fresh_start);
14259 assert!(queue.next_runnable().is_none());
14260 }
14261 }
14262
14263 #[test]
14264 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
14265 for held in [false, true] {
14266 for retry in [false, true] {
14267 let dir = tempfile::tempdir().unwrap();
14268 let queue = Queue::at(dir.path().join("queue"));
14269 let questions = Questions::at(dir.path().join("questions"));
14270 let home = dir.path().join("home");
14271 let mut t = task();
14272 if held {
14273 t.hold_machine(Some("driver state unknown".into()));
14274 }
14275 queue.put(&mut t).unwrap();
14276 let verdict = serde_json::from_value(serde_json::json!({
14277 "decisions": [{
14278 "id": t.id,
14279 "question": "Driver state unknown; retry the checks or hold?",
14280 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
14281 "actions": {
14282 "daemon: recheck and retry": {"do": "requeue"},
14283 "daemon: hold the task": {"do": "hold"}
14284 }
14285 }]
14286 }))
14287 .unwrap();
14288 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
14289 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
14290 let mut q = questions.list().pop().unwrap();
14291 let choice = if retry {
14292 "daemon: recheck and retry"
14293 } else {
14294 "daemon: hold the task"
14295 };
14296 assert_eq!(q.actions.len(), 2);
14297 q.answer(ask::Answer::Choice(choice.into())).unwrap();
14298 questions.put(&mut q).unwrap();
14299
14300 questions.beat(&q.id, ask::WaiterKind::Deputy);
14302 for _ in 0..2 {
14303 resolve_blockers(&queue, &questions);
14304 apply_task_choice_actions(&queue, &questions, &home);
14305 let waiting = queue.get(&t.id).unwrap();
14306 assert_eq!(waiting.status, TaskStatus::Blocked);
14307 assert_eq!(waiting.blocked_by, [q.id.clone()]);
14308 assert!(waiting.answers.is_empty());
14309 assert!(!waiting.action_applied(&q.id));
14310 assert!(runnable(&queue).is_empty());
14311 let free = Arc::new(tokio::sync::Semaphore::new(1));
14312 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_err());
14313 }
14314 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
14315
14316 resolve_blockers(&queue, &questions);
14318 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
14319 apply_task_choice_actions(&queue, &questions, &home);
14320 let after = queue.get(&t.id).unwrap();
14321 assert_eq!(
14322 after.status,
14323 if retry {
14324 TaskStatus::Queued
14325 } else {
14326 TaskStatus::Held
14327 }
14328 );
14329 assert!(after.action_applied(&q.id));
14330 assert_eq!(after.operator_held(), !retry);
14331 assert_eq!(after.answers.len(), 1);
14332
14333 let mut changed = after;
14335 if retry {
14336 changed.hold_manual(Some("later hold".into()));
14337 } else {
14338 changed.release_operator("test", None);
14339 }
14340 let expected = changed.status;
14341 queue.put(&mut changed).unwrap();
14342 apply_task_choice_actions(&queue, &questions, &home);
14343 assert_eq!(queue.get(&t.id).unwrap().status, expected);
14344 }
14345 }
14346 }
14347
14348 #[test]
14349 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
14350 let dir = tempfile::tempdir().unwrap();
14351 let queue = Queue::at(dir.path().join("queue"));
14352 let questions = Questions::at(dir.path().join("questions"));
14353 let home = dir.path().join("home");
14354 let mut state = run_state(RunStatus::Blocked);
14355 state.id = "20260101-000000-act1".to_owned();
14356 state.save_under(&home).unwrap();
14357
14358 let mut t = held_task_with(&state.id);
14359 queue.put(&mut t).unwrap();
14360 let mut q = action_question(&state.id, resume_action(&state.id));
14361 questions.put(&mut q).unwrap();
14362
14363 apply_task_choice_actions(&queue, &questions, &home);
14364 let after = queue.get(&t.id).unwrap();
14365 assert_eq!(after.status, TaskStatus::Queued);
14366 assert!(!after.fresh_start);
14367 assert!(after.action_applied(&q.id));
14368 let pin = after.resume_override.clone().unwrap();
14369 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
14370 assert!(pin.forced);
14371
14372 let mut again = queue.get(&t.id).unwrap();
14374 again.hold_machine(Some("later".into()));
14375 queue.put(&mut again).unwrap();
14376 apply_task_choice_actions(&queue, &questions, &home);
14377 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
14378 }
14379
14380 #[test]
14381 fn a_delivered_answer_is_still_acted_on_exactly_once() {
14382 let dir = tempfile::tempdir().unwrap();
14383 let queue = Queue::at(dir.path().join("queue"));
14384 let questions = Questions::at(dir.path().join("questions"));
14385 let home = dir.path().join("home");
14386 let mut state = run_state(RunStatus::Blocked);
14387 state.id = "20260101-000000-act2".to_owned();
14388 state.save_under(&home).unwrap();
14389
14390 let mut t = held_task_with(&state.id);
14391 queue.put(&mut t).unwrap();
14392 let mut q = action_question(&state.id, resume_action(&state.id));
14393 q.answer_delivered = true;
14394 questions.put(&mut q).unwrap();
14395
14396 apply_task_choice_actions(&queue, &questions, &home);
14399 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
14400 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
14401
14402 let mut again = queue.get(&t.id).unwrap();
14404 again.hold_machine(Some("later".into()));
14405 queue.put(&mut again).unwrap();
14406 apply_task_choice_actions(&queue, &questions, &home);
14407 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
14408 }
14409
14410 #[test]
14411 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
14412 let dir = tempfile::tempdir().unwrap();
14413 let queue = Queue::at(dir.path().join("queue"));
14414 let questions = Questions::at(dir.path().join("questions"));
14415 let home = dir.path().join("home");
14416 let mut state = run_state(RunStatus::Blocked);
14417 state.id = "20260101-000000-act3".to_owned();
14418 state.save_under(&home).unwrap();
14419 let mut t = held_task_with(&state.id);
14420 queue.put(&mut t).unwrap();
14421 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
14422 questions.put(&mut q).unwrap();
14423
14424 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
14425 apply_task_choice_actions(&queue, &questions, &home);
14426 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
14427
14428 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
14429 apply_task_choice_actions(&queue, &questions, &home);
14430 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
14431 }
14432
14433 #[test]
14434 fn action_standing_tells_the_reasons_apart() {
14435 let mut t = held_task_with("r1");
14436 let q = action_question("r1", ask::ChoiceAction::Requeue);
14437 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
14438 let mut plain = q.clone();
14439 plain.actions.clear();
14440 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
14441
14442 t.mark_action_applied(&q.id);
14444 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
14445
14446 let mut t = held_task_with("r1");
14448 t.start("r2".to_owned());
14449 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
14450 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
14452 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
14453 t.status = TaskStatus::Blocked;
14454 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
14455
14456 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
14458 c.node = crate::conduct::NODE.to_owned();
14459 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
14460 assert!(!ActionStanding::Busy.daemon_owns());
14461 }
14462
14463 #[test]
14464 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
14465 let dir = tempfile::tempdir().unwrap();
14466 let queue = Queue::at(dir.path().join("queue"));
14467 let questions = Questions::at(dir.path().join("questions"));
14468 let home = dir.path().join("home");
14469 let mut state = run_state(RunStatus::Blocked);
14470 state.id = "20260101-000000-act4".to_owned();
14471 state.save_under(&home).unwrap();
14472 let mut t = held_task_with(&state.id);
14473 t.status = TaskStatus::Running;
14474 queue.put(&mut t).unwrap();
14475 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
14476 q.cwd = Some(dir.path().display().to_string());
14477 questions.put(&mut q).unwrap();
14478
14479 let running = queue.get(&t.id).unwrap();
14482 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
14483 assert!(matches!(
14484 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
14485 crate::waiter::Action::Deliver(_)
14486 ));
14487 apply_task_choice_actions(&queue, &questions, &home);
14489 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
14490 let mut back = queue.get(&t.id).unwrap();
14491 back.hold_machine(Some("later".into()));
14492 queue.put(&mut back).unwrap();
14493 apply_task_choice_actions(&queue, &questions, &home);
14494 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
14495 let mut again = queue.get(&t.id).unwrap();
14496 again.hold_machine(Some("again".into()));
14497 queue.put(&mut again).unwrap();
14498 apply_task_choice_actions(&queue, &questions, &home);
14499 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
14500 }
14501}