1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.runs.last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id,
519 run: task.runs.last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if q.chosen_action().is_none() {
887 ActionStanding::NoAction
888 } else if task.action_applied(&q.id) {
889 ActionStanding::Applied
890 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 ActionStanding::Stale
892 } else if matches!(
893 task.status,
894 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
895 ) {
896 ActionStanding::Busy
897 } else {
898 ActionStanding::Pending
899 }
900}
901
902#[derive(Debug, Clone, PartialEq, Eq)]
904enum ActionDecision {
905 Skip,
907 Resume(String),
909 Requeue,
911 Hold,
913 Done,
915 Stale,
918 Refuse(String),
921}
922
923fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
931where
932 F: FnOnce(&str) -> Result<RunState>,
933{
934 let Some(action) = q.chosen_action() else {
935 return ActionDecision::Skip;
936 };
937 if task.action_applied(&q.id)
938 || matches!(
939 task.status,
940 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
941 )
942 {
943 return ActionDecision::Skip;
944 }
945 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
948 return ActionDecision::Stale;
949 }
950 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
951 let stopping = q.node == land::APPROVAL_NODE
952 && q.resolution()
953 .as_deref()
954 .is_some_and(crate::merge_decision::stops);
955 let decision_at = if repair || stopping {
959 q.answered_at.unwrap_or(q.asked_at)
960 } else {
961 q.asked_at
962 };
963 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
964 return if repair || stopping {
965 ActionDecision::Refuse(format!(
966 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
967 if repair { "repair" } else { "stop" },
968 q.id,
969 q.run,
970 task.id
971 ))
972 } else {
973 ActionDecision::Stale
974 };
975 }
976 match action {
977 a if a.maintenance() => ActionDecision::Skip,
978 ask::ChoiceAction::ResumeRun { .. }
979 | ask::ChoiceAction::ReleaseTask { .. }
980 | ask::ChoiceAction::RerunChecks { .. }
981 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
982 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
983 ask::ChoiceAction::Hold => ActionDecision::Hold,
984 ask::ChoiceAction::Done => ActionDecision::Done,
985 ask::ChoiceAction::Resume { run } => {
986 if task.runs.last() != Some(run) {
987 return ActionDecision::Refuse((p.resume_not_latest)(
988 q.short(),
989 ask::short_id(run),
990 ));
991 }
992 match load(run) {
993 Ok(s) if stopping => {
994 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
995 || !s
996 .land_approval
997 .as_ref()
998 .is_some_and(|binding| binding.question == q.id)
999 {
1000 ActionDecision::Stale
1001 } else if s.liveness(false) == Liveness::Live {
1002 ActionDecision::Skip
1003 } else if s.status == RunStatus::Landing
1004 && !s.released()
1005 && s.retired_to.is_none()
1006 {
1007 ActionDecision::Resume(run.clone())
1008 } else {
1009 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1010 }
1011 }
1012 Ok(s)
1013 if q.node == land::APPROVAL_NODE
1014 && q.resolution().as_deref() == Some(land::REPAIR) =>
1015 {
1016 if s.merge_repairs.iter().any(|r| r.question == q.id)
1017 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1018 {
1019 ActionDecision::Stale
1022 } else if s.liveness(false) == Liveness::Live {
1023 ActionDecision::Skip
1024 } else if s.status != RunStatus::Landing
1025 || s.released()
1026 || s.reviews.is_empty()
1027 || !implementers_match(task, &s)
1028 || !s
1029 .land_approval
1030 .as_ref()
1031 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1032 {
1033 ActionDecision::Refuse(format!(
1034 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1035 q.id, run, run
1036 ))
1037 } else {
1038 ActionDecision::Resume(run.clone())
1039 }
1040 }
1041 Ok(s) if s.liveness(false) == Liveness::Live => {
1042 ActionDecision::Refuse("run already has a live driver".into())
1043 }
1044 Ok(s) if crate::handover::can_resume(&s) => ActionDecision::Resume(run.clone()),
1045 Ok(_) => ActionDecision::Refuse((p.resume_cannot_progress)(
1046 q.short(),
1047 ask::short_id(run),
1048 )),
1049 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1050 q.short(),
1051 ask::short_id(run),
1052 &format!("{e:#}"),
1053 )),
1054 }
1055 }
1056 }
1057}
1058
1059struct Phrases {
1073 graph_stopped: fn(&str, &str) -> String,
1075 quorum_lost: &'static str,
1076 quota_took_out: &'static str,
1078 run_ended: &'static str,
1080 waiting_for_answer: &'static str,
1082 recovered_running: &'static str,
1083 no_run_to_recover: &'static str,
1084 could_not_start: &'static str,
1085 resume_not_latest: fn(&str, &str) -> String,
1086 resume_cannot_progress: fn(&str, &str) -> String,
1087 resume_unreadable: fn(&str, &str, &str) -> String,
1088 handover_refused: fn(&crate::handover::Refused) -> String,
1091 handover_hint: &'static str,
1093 reviewers_never_answered: fn(usize, usize) -> String,
1096}
1097
1098const PHRASES_EN: Phrases = Phrases {
1099 graph_stopped: |status, detail| {
1100 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1101 },
1102 quorum_lost: "the judging panel lost its quorum",
1103 quota_took_out: "; quota took out ",
1104 run_ended: "run ended ",
1105 waiting_for_answer: " - waiting for operator answer to question ",
1106 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1107 no_run_to_recover: "task was `running` with no live daemon and no readable \
1108 run to recover; held for a human to check what happened",
1109 could_not_start: "could not start the run: ",
1110 resume_not_latest: |q, run| {
1111 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1112 },
1113 resume_cannot_progress: |q, run| {
1114 format!("question {q} asked to resume run {run}, which cannot make progress")
1115 },
1116 resume_unreadable: |q, run, e| {
1117 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1118 },
1119 handover_refused: |r| r.to_string(),
1120 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1121 reviewers_never_answered: |missing, rounds| {
1122 format!(
1123 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1124 refusing to call it clean"
1125 )
1126 },
1127};
1128
1129const PHRASES_JA: Phrases = Phrases {
1130 graph_stopped: |status, detail| {
1131 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1132 },
1133 quorum_lost: "審査パネルが定足数を失いました",
1134 quota_took_out: "。クォータで脱落: ",
1135 run_ended: "run 終了: ",
1136 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1137 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1138 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1139 何が起きたか人が確認するため保留にしました",
1140 could_not_start: "run を開始できませんでした: ",
1141 resume_not_latest: |q, run| {
1142 format!(
1143 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1144 )
1145 },
1146 resume_cannot_progress: |q, run| {
1147 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません")
1148 },
1149 resume_unreadable: |q, run, e| {
1150 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1151 },
1152 handover_refused: |r| {
1153 use crate::handover::Refused;
1154 match r {
1155 Refused::Foreign { branch, path, why } => format!(
1156 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1157 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1158 から、やり直してください(詳細: {why})"
1159 ),
1160 Refused::Unsafe { branch, path, why } => format!(
1161 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1162 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1163 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1164 ),
1165 Refused::ReleaseFailed { branch, path, run } => format!(
1166 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1167 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1168 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1169 ),
1170 }
1171 },
1172 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1173 reviewers_never_answered: |missing, rounds| {
1174 format!(
1175 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1176 クリーンとは認めません"
1177 )
1178 },
1179};
1180
1181fn phrases(language: &str) -> &'static Phrases {
1182 if crate::lang::is_japanese(language) {
1183 &PHRASES_JA
1184 } else {
1185 &PHRASES_EN
1186 }
1187}
1188
1189fn language_of(task: &Task, fallback: &Path) -> String {
1193 crate::lang::of_repo(&repo_for(task, fallback))
1194}
1195
1196pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1200 if q.node == crate::conduct::NODE {
1201 return tasks.iter().find(|t| t.id == q.run);
1202 }
1203 tasks.iter().find(|t| t.runs.contains(&q.run))
1204}
1205
1206fn repair_choice_offers(questions: &Questions) {
1209 for q in questions.list().into_iter().filter(|q| {
1210 q.status.open()
1211 && q.node == crate::conduct::NODE
1212 && q.choices
1213 .iter()
1214 .any(|c| crate::choice_effect::describe(q, c).is_none())
1215 }) {
1216 let _ = questions.update(&q.id, |current| {
1217 if current.status.open() {
1218 crate::choice_effect::executable_conductor(current);
1219 if let Some(deputy) = &mut current.deputy {
1220 deputy.brief = crate::deputy::brief(
1221 ¤t.run,
1222 ¤t.detail,
1223 ¤t.choices,
1224 ¤t.actions,
1225 );
1226 }
1227 }
1228 Ok(())
1229 });
1230 }
1231}
1232
1233pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1236 apply_maintenance_choices(queue, questions, home).await;
1237 apply_task_choice_actions(queue, questions, home);
1238}
1239
1240async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1243 for q in questions.list() {
1244 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1245 continue;
1246 };
1247 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1250 continue;
1251 }
1252 let dir = home.join("choice-actions");
1253 if std::fs::create_dir_all(&dir).is_err() {
1254 continue;
1255 }
1256 let Ok(lock) = std::fs::OpenOptions::new()
1257 .create(true)
1258 .truncate(false)
1259 .write(true)
1260 .open(dir.join(format!("{}.lock", q.id)))
1261 else {
1262 continue;
1263 };
1264 if crate::pr_head::try_lock_file(&lock).is_err() {
1265 continue;
1266 }
1267 let fresh = !home
1268 .join("choice-actions")
1269 .join(format!("{}.json", q.id))
1270 .exists();
1271 let result = async {
1272 let receipt = dir.join(format!("{}.json", q.id));
1273 if receipt.exists() {
1274 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1275 anyhow::ensure!(
1276 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1277 "previous maintenance attempt did not confirm completion; inspect its receipt"
1278 );
1279 return Ok::<_, anyhow::Error>(());
1280 }
1281 let record = |status: &str| -> anyhow::Result<()> {
1284 let tmp = receipt.with_extension("tmp");
1285 std::fs::write(
1286 &tmp,
1287 serde_json::to_vec(&serde_json::json!({
1288 "action": action, "status": status,
1289 }))?,
1290 )?;
1291 std::fs::rename(tmp, &receipt)?;
1292 Ok(())
1293 };
1294 let applied = async {
1295 match action {
1296 ask::ChoiceAction::ResumeRun { run } => {
1297 crate::resume::validate(home, run)?;
1298 record("started")?;
1299 if !crate::resume::pin_queued(queue, run, Some(q.asked_at), Some(&q.id))? {
1300 crate::resume::validate_unqueued(home, run)?;
1301 crate::resume::start(home, run.clone()).await?;
1302 }
1303 }
1304 ask::ChoiceAction::ReleaseTask {
1305 task,
1306 held_at,
1307 reason,
1308 } => {
1309 anyhow::ensure!(
1310 queue.resolve_id(task)? == *task,
1311 "task action requires an exact task ID"
1312 );
1313 let _claim = queue.claim(task)?;
1314 let mut task = queue.get(task)?;
1315 if task.action_applied(&q.id) {
1316 return Ok(());
1317 }
1318 if q.node == crate::conduct::NODE
1323 && q.run == task.id
1324 && task.status == TaskStatus::Blocked
1325 && task.blocked_from == Some(TaskStatus::Held)
1326 && task.blocked_by.as_slice() == [q.id.as_str()]
1327 {
1328 task.unblock(&q.id);
1329 }
1330 anyhow::ensure!(
1331 task.status == TaskStatus::Held
1332 && task.held_at == Some(*held_at)
1333 && task.hold_reason.as_deref() == reason.as_deref()
1334 && task.operator_decided_at.is_none_or(|at| at <= q.asked_at),
1335 "task no longer has the hold this question names"
1336 );
1337 task.release_operator("choice", Some(&q.id));
1338 task.mark_action_applied(&q.id);
1339 queue.put(&mut task)?;
1340 }
1341 ask::ChoiceAction::AbandonRun { run } => {
1342 record("started")?;
1343 crate::driver::abandon_proven_dead(home, run)?;
1344 }
1345 ask::ChoiceAction::RerunChecks {
1346 repo,
1347 pr,
1348 head,
1349 runs,
1350 } => {
1351 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1352 .await?;
1353 }
1354 _ => unreachable!(),
1355 }
1356 Ok::<_, anyhow::Error>(())
1357 }
1358 .await;
1359 match applied {
1360 Ok(()) => record("applied"),
1361 Err(e) => {
1362 record(&format!("refused or failed: {e:#}"))?;
1363 Err(e)
1364 }
1365 }
1366 }
1367 .await;
1368 if home
1373 .join("choice-actions")
1374 .join(format!("{}.json", q.id))
1375 .exists()
1376 && let Some(listed) = task_of_question(&queue.list(), &q)
1377 && !listed.action_applied(&q.id)
1378 && let Ok(_claim) = queue.claim(&listed.id)
1379 && let Ok(mut origin) = queue.get(&listed.id)
1380 {
1381 if !origin.operator_held()
1382 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1383 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1384 {
1385 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1386 origin.hold_machine(Some(format!(
1387 "Maintenance choice {} settled; inspect its target before releasing this task",
1388 q.id
1389 )));
1390 } else {
1391 origin.unblock(&q.id);
1392 }
1393 }
1394 origin.mark_action_applied(&q.id);
1395 let _ = queue.put(&mut origin);
1396 }
1397 if let Err(e) = result
1398 && fresh
1399 {
1400 notices::raise_in(
1401 home,
1402 Notice::warn(
1403 &format!("action:{}", q.id),
1404 format!("Choice action {} was not completed: {e:#}", q.id),
1405 )
1406 .link(if q.run_names_task() {
1407 Link::Task { id: q.run.clone() }
1408 } else {
1409 Link::Run { id: q.run.clone() }
1410 }),
1411 );
1412 }
1413 }
1414}
1415
1416pub(crate) async fn rerun_checks_once(
1423 home: &Path,
1424 request: &str,
1425 repo: &Path,
1426 pr: &str,
1427 head: &str,
1428 runs: &[String],
1429 started: impl FnOnce() -> anyhow::Result<()>,
1430) -> anyhow::Result<()> {
1431 use crate::release_watch::{GhForge, ReleaseForge};
1432 anyhow::ensure!(
1433 !runs.is_empty()
1434 && runs.len() <= 20
1435 && runs
1436 .iter()
1437 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1438 && !pr.starts_with('-'),
1439 "invalid rerun targets"
1440 );
1441 let snap = GhForge.snapshot(repo, pr).await?;
1442 anyhow::ensure!(
1443 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1444 "pull request head or lifecycle changed"
1445 );
1446 anyhow::ensure!(
1447 runs.iter().all(|r| snap
1448 .checks
1449 .iter()
1450 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1451 "named workflow is no longer failed on this pull request"
1452 );
1453 started()?;
1454 let dir = home.join("choice-actions");
1455 std::fs::create_dir_all(&dir)?;
1456 let rerun_lock = std::fs::OpenOptions::new()
1457 .create(true)
1458 .truncate(false)
1459 .write(true)
1460 .open(dir.join("reruns.lock"))?;
1461 crate::pr_head::try_lock_file(&rerun_lock)?;
1462 let ledger = dir.join("reruns.json");
1463 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1464 serde_json::from_slice(&std::fs::read(&ledger)?)?
1465 } else {
1466 Default::default()
1467 };
1468 for run in runs {
1469 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1470 if !recorded.insert(key) {
1471 continue;
1472 }
1473 let tmp = ledger.with_extension("tmp");
1474 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1475 std::fs::rename(tmp, &ledger)?;
1476 GhForge.rerun(repo, run).await?;
1477 }
1478 Ok(())
1479}
1480
1481pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1490 let tasks = queue.list();
1491 for q in questions.list() {
1492 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1494 continue;
1495 }
1496 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1497 continue;
1498 }
1499 let Some(listed) = task_of_question(&tasks, &q) else {
1500 if q.node == land::APPROVAL_NODE
1501 && q.resolution().as_deref().is_some_and(|choice| {
1502 choice == land::REPAIR || crate::merge_decision::stops(choice)
1503 })
1504 {
1505 notices::raise_in(
1506 home,
1507 Notice::warn(
1508 &format!("repair-owner:{}", q.id),
1509 format!(
1510 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1511 q.run, q.run
1512 ),
1513 ).link(Link::Run { id: q.run.clone() }),
1514 );
1515 }
1516 continue;
1517 };
1518 if listed.action_applied(&q.id) {
1519 continue;
1520 }
1521 let Ok(_claim) = queue.claim(&listed.id) else {
1522 continue;
1523 };
1524 let Ok(mut task) = queue.get(&listed.id) else {
1525 continue;
1526 };
1527 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1530 let blockers: Vec<_> = task
1531 .blocked_by
1532 .iter()
1533 .filter(|id| {
1534 crate::queue::dependency(queue, questions, id)
1535 == crate::queue::Dependency::Question(q.id.clone())
1536 })
1537 .cloned()
1538 .collect();
1539 if !blockers.is_empty() {
1540 for id in blockers {
1541 task.unblock(&id);
1542 }
1543 let answer = match &q.answer {
1544 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1545 None => String::new(),
1546 };
1547 task.record_answer(q.summary.clone(), answer);
1548 }
1549 }
1550 let language = language_of(&task, Path::new("."));
1551 let decision = decide_action(&task, &q, phrases(&language), |id| {
1552 RunState::load_under(id, home)
1553 });
1554 let ran = matches!(
1555 decision,
1556 ActionDecision::Resume(_)
1557 | ActionDecision::Requeue
1558 | ActionDecision::Hold
1559 | ActionDecision::Done
1560 );
1561 if ran && asker_may_still_read(questions, &q, home) {
1562 continue;
1571 }
1572 let done = matches!(decision, ActionDecision::Done);
1573 match decision {
1574 ActionDecision::Skip => continue,
1575 ActionDecision::Resume(run) => {
1576 task.release_operator("choice", Some(&q.id));
1577 task.resume_override = Some(crate::queue::OperatorResume {
1578 question_id: q.id.clone(),
1579 at: Timestamp::now(),
1580 conductor_rehold: None,
1581 forced: true,
1582 pinned_run: Some(run),
1583 });
1584 }
1585 ActionDecision::Stale => {}
1586 ActionDecision::Requeue => {
1587 task.release_operator("choice-requeue", Some(&q.id));
1588 task.requeue();
1589 }
1590 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1591 ActionDecision::Done => task.succeed(),
1592 ActionDecision::Refuse(why) => {
1593 if task.operator_held() {
1594 task.hold_reason = Some(match task.hold_reason.take() {
1597 Some(reason) => format!("{reason}\n\n{why}"),
1598 None => why,
1599 });
1600 } else {
1601 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1602 task.hold_machine(Some(why));
1603 }
1604 notices::raise(
1605 Notice::warn(
1606 &format!("action:{}", q.id),
1607 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1608 )
1609 .link(Link::Task {
1610 id: task.id.clone(),
1611 }),
1612 );
1613 }
1614 }
1615 task.mark_action_applied(&q.id);
1616 record(queue, &mut task);
1619 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1620 supersede_prior_runs(&task, home);
1621 }
1622 let _ = questions.update(&q.id, |r| {
1625 r.answer_delivered = true;
1626 Ok(())
1627 });
1628 }
1629}
1630
1631fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1635 let now = Timestamp::now();
1636 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1637 return true;
1638 }
1639 q.node != crate::conduct::NODE
1640 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1641 s.seats_active()
1642 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1643 })
1644}
1645
1646fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1657 let tasks = queue.list();
1658 let by_id: std::collections::BTreeMap<&str, &Task> =
1659 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1660 let referenced: Vec<&str> = tasks
1661 .iter()
1662 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1663 .collect();
1664
1665 for mut question in questions.list() {
1666 if !question.status.open() || question.node != crate::conduct::NODE {
1667 continue;
1668 }
1669 if referenced
1674 .iter()
1675 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1676 {
1677 continue;
1678 }
1679 let Some(task) = by_id.get(question.run.as_str()) else {
1680 continue;
1681 };
1682 question.abandon(format!(
1683 "task {} no longer waits for this answer",
1684 task.short()
1685 ));
1686 if let Err(e) = questions.put(&mut question) {
1687 tracing::warn!(
1688 "could not retire question {} for task {}: {e:#}",
1689 question.short(),
1690 task.short()
1691 );
1692 }
1693 }
1694}
1695
1696#[derive(Debug, Clone, Copy)]
1703pub struct Verdict {
1704 pub status: RunStatus,
1706 pub left_pr: bool,
1708 pub quota_hit: bool,
1710 pub parked: bool,
1712 pub no_viable_candidates: bool,
1719}
1720
1721pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1778 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1779}
1780
1781fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1783 if verdict.parked {
1789 task.park(detail);
1790 return;
1791 }
1792 match verdict.status {
1793 RunStatus::Merged | RunStatus::Ready
1794 if task.pr_recovery.as_ref().is_some_and(|s| {
1795 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1796 }) =>
1797 {
1798 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1799 }
1800 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1801 RunStatus::AlreadyInBase => task.already_landed(detail),
1802 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1803 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1804 task.stall(detail)
1805 }
1806 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1807 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1808 RunStatus::Blocked => task.fail(detail, max_attempts),
1809 RunStatus::VerifiedNoop => task.handed_off(detail),
1810 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1811 }
1812}
1813
1814pub fn supersede_prior_runs(task: &Task, home: &Path) {
1863 let now = Timestamp::now();
1864 let last_run_succeeded = task
1871 .runs
1872 .last()
1873 .and_then(|id| RunState::load_under(id, home).ok())
1874 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1875 for id in task.superseded_attempts(last_run_succeeded) {
1876 let mut state = match RunState::load_under(id, home) {
1877 Ok(s) => s,
1878 Err(e) => {
1879 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1880 continue;
1881 }
1882 };
1883 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1884 continue;
1885 }
1886 let daemon_claims = is_working_on(home, id, now);
1887 if state.liveness(daemon_claims) == Liveness::Live {
1888 continue;
1889 }
1890 state.status = RunStatus::Superseded;
1891 if let Err(e) = state.save_under(home) {
1892 tracing::warn!("could not mark run {id} superseded: {e:#}");
1893 }
1894 }
1895}
1896
1897fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1919 for task in queue.list() {
1920 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1921 continue;
1922 }
1923 supersede_prior_runs(&task, home);
1924 }
1925}
1926
1927fn settle_and_diagnose(
1934 task: &mut Task,
1935 verdict: Verdict,
1936 detail: &str,
1937 max_attempts: usize,
1938 state: &RunState,
1939) {
1940 let p = phrases(&state.config.graph.language);
1941 if let Some(d) = &state.merge_decision {
1944 use crate::merge_decision as md;
1945 if task.runs.last().is_some_and(|id| id != &state.id) {
1946 return;
1947 }
1948 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1949 task.succeed();
1950 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1951 if state.liveness(false) != Liveness::Dead {
1952 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1953 return;
1954 }
1955 let replacement = md::Replacement {
1956 question: d.question.clone(),
1957 run: state.id.clone(),
1958 pr: d.target.pr.clone(),
1959 agent: agent.to_owned(),
1960 context: format!(
1961 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1962 d.target.pr,
1963 state.id,
1964 serde_json::to_string(&state.reviews).unwrap_or_default()
1965 ),
1966 };
1967 task.requeue();
1968 task.pr_recovery = None;
1969 task.review_of = None;
1970 task.replacement = Some(replacement);
1971 } else {
1972 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1973 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1974 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1975 }
1976 task.mark_action_applied(&d.question);
1977 if let Some(pass) = state.passes.last() {
1978 task.record_pass_at(
1979 &state.id,
1980 pass.index,
1981 Some(pass.started_at),
1982 task.attempts,
1983 None,
1984 );
1985 }
1986 return;
1987 }
1988 if let Some(replacement) = &task.replacement
1989 && !state.parked
1990 && state.resource_block.is_none()
1991 && (state.viable().is_empty()
1992 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1993 {
1994 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1995 return;
1996 }
1997 let attempts_before = task.attempts;
1998 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2005 task.notice_cause(
2006 &format!("resource:{}:{}", block.kind, block.node),
2007 None,
2008 None,
2009 );
2010 task.last_error = Some(detail.to_owned());
2011 task.diagnostic = None;
2012 task.attempts = task.attempts.saturating_sub(1);
2013 if block.kind == "disk pressure" {
2014 task.hold_machine(Some(format!(
2015 "{}{}",
2016 crate::disk::RUNNING_PRESSURE,
2017 block.detail
2018 )));
2019 } else {
2020 task.hold_machine(Some(format!(
2021 "{}{}; no attempt was spent; resume to retry the affected operation",
2022 crate::cache::RESOURCE_HOLD,
2023 block.description()
2024 )));
2025 }
2026 if let Some(pass) = state.passes.last() {
2027 task.record_pass_at(
2028 &state.id,
2029 pass.index,
2030 Some(pass.started_at),
2031 attempts_before,
2032 None,
2033 );
2034 }
2035 return;
2036 }
2037 if let Some(hold) = state.land_unrelated.as_ref().filter(|hold| {
2038 state.parked
2039 && state.status == RunStatus::Landing
2040 && state
2041 .pr_pushed_head
2042 .as_ref()
2043 .is_none_or(|h| h == &hold.head)
2044 && task.runs.last() == Some(&state.id)
2045 }) {
2046 let pass = state.passes.last().map_or(0, |p| p.index);
2049 if !task
2050 .passes
2051 .iter()
2052 .any(|p| p.run == state.id && p.pass_index == pass)
2053 {
2054 task.attempts = task.attempts.saturating_sub(1);
2055 }
2056 task.hold_machine(Some(format!(
2057 "[ci-unrelated] CI failure reproduced on base; owner question {}",
2058 hold.question
2059 )));
2060 task.last_error = Some(detail.to_owned());
2061 task.diagnostic = None;
2062 task.record_pass_at(
2063 &state.id,
2064 pass,
2065 state.passes.last().map(|p| p.started_at),
2066 attempts_before,
2067 Some(crate::run::ParkCause::Question),
2068 );
2069 return;
2070 }
2071 let unresolved = task.hold_notice_cause.clone();
2072 if !verdict.parked
2073 && task
2074 .pr_recovery
2075 .as_ref()
2076 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2077 && matches!(
2078 verdict.status,
2079 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2080 )
2081 {
2082 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2083 task.succeed();
2084 } else {
2085 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2086 }
2087 } else if !verdict.parked
2088 && task.pr_recovery.is_some()
2089 && matches!(
2090 verdict.status,
2091 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2092 )
2093 && !verdict.quota_hit
2094 {
2095 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2096 } else {
2097 settle_in(task, verdict, detail, max_attempts, p);
2098 }
2099 if !verdict.parked
2100 && !verdict.quota_hit
2101 && matches!(
2102 verdict.status,
2103 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2104 )
2105 {
2106 task.recovery_failed(state);
2107 }
2108 if state.prep_failure.is_some() {
2109 task.hold_notice_cause = unresolved;
2112 task.startup_notice_cause("startup-candidates");
2113 }
2114 if let Some(pass) = state.passes.last() {
2117 let parked = match &pass.end {
2118 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2119 _ => None,
2120 };
2121 task.record_pass_at(
2122 &state.id,
2123 pass.index,
2124 Some(pass.started_at),
2125 attempts_before,
2126 parked,
2127 );
2128 }
2129 if task.status == TaskStatus::Held {
2130 if state.prep_failure.is_none() {
2131 task.notice_cause(
2132 label(verdict.status),
2133 Some(&state.id),
2134 state.passes.last().map(|p| p.index),
2135 );
2136 }
2137 task.diagnostic = diagnostic(state);
2138 note_open_question(task, &state.id, p);
2139 }
2140}
2141
2142fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2157 let Some(home) = crate::run::try_home() else {
2158 return;
2159 };
2160 let open = Questions::at(home.join("questions")).open_for(run);
2161 let Some(q) = open.first() else {
2162 return;
2163 };
2164 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2165 decisions.sort_unstable();
2166 task.notice_cause(
2167 &format!("questions:{}", decisions.join(",")),
2168 Some(run),
2169 None,
2170 );
2171 let base = task.hold_reason.clone().unwrap_or_default();
2172 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2173}
2174
2175fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2186 match last_run {
2187 Some(state) => {
2188 let verdict = Verdict {
2189 status: state.status,
2190 left_pr: state.pr.is_some(),
2191 quota_hit: !state.quota.is_empty(),
2192 parked: state.parked,
2193 no_viable_candidates: state.viable().is_empty(),
2194 };
2195 let detail = format!(
2196 "{}{}",
2197 phrases(&state.config.graph.language).recovered_running,
2198 describe(&state)
2199 );
2200 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2201 }
2202 None => {
2203 let why = phrases(language).no_run_to_recover;
2206 task.last_error = Some(why.to_owned());
2207 task.notice_cause("missing-run", None, None);
2210 task.hold_machine(Some(why.to_owned()));
2211 }
2212 }
2213}
2214
2215fn reclaim_orphaned_running(
2237 queue: &Queue,
2238 max_attempts: usize,
2239 live_workers: &std::collections::BTreeSet<String>,
2240) -> Vec<String> {
2241 let mut reclaimed = Vec::new();
2242 for listed in queue.list() {
2243 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2244 continue;
2245 }
2246 let Ok(_claim) = queue.claim(&listed.id) else {
2247 continue;
2248 };
2249 let Ok(mut task) = queue.get(&listed.id) else {
2253 continue;
2254 };
2255 if task.status != TaskStatus::Running {
2256 continue;
2257 }
2258 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2259 if let Some(state) = &last_run
2269 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2270 {
2271 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2272 }
2273 let language = if last_run.is_none() {
2274 language_of(&task, Path::new("."))
2275 } else {
2276 String::new()
2277 };
2278 reclaim(&mut task, last_run, max_attempts, &language);
2279 if task.status == TaskStatus::Done {
2280 supersede_prior_runs(&task, &crate::run::home());
2281 }
2282 record(queue, &mut task);
2283 reclaimed.push(task.id.clone());
2284 }
2285 reclaimed
2286}
2287
2288fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2320 reclaim_abandoned_runs_with(
2321 home,
2322 now,
2323 crate::proc::pid_status,
2324 crate::proc::process_started_at,
2325 )
2326}
2327
2328fn reclaim_abandoned_runs_with<F, G>(
2334 home: &Path,
2335 now: Timestamp,
2336 query: F,
2337 identity: G,
2338) -> Vec<String>
2339where
2340 F: Fn(u32) -> Option<bool>,
2341 G: Fn(u32) -> Option<String>,
2342{
2343 let mut abandoned = Vec::new();
2344 for entry in std::fs::read_dir(home.join("runs"))
2345 .into_iter()
2346 .flatten()
2347 .flatten()
2348 {
2349 let id = entry.file_name().to_string_lossy().into_owned();
2350 if !crate::run::is_run_id(&id) {
2351 continue;
2352 }
2353 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2356 continue;
2357 };
2358 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2366 continue;
2367 };
2368 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2369 continue;
2370 };
2371 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2372 continue;
2373 }
2374 let daemon_claims = is_working_on(home, &id, now);
2386 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2387 != crate::run::Liveness::Dead
2388 {
2389 continue;
2390 }
2391 state.abandon("daemon");
2392 if let Err(e) = state.save_under(home) {
2393 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2394 continue;
2395 }
2396 if let Some(notice) = notices::run_ended(&state) {
2399 notices::raise_in(home, notice);
2400 }
2401 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2409 tracing::warn!("abandon questions for {id}: {e:#}");
2410 }
2411 abandoned.push(id);
2412 }
2413 abandoned
2414}
2415
2416#[derive(Debug, Clone, Serialize, Deserialize)]
2417struct TaskWorker {
2418 opts: Opts,
2419 queue: PathBuf,
2420 task: String,
2421 #[serde(default)]
2422 lane: PermitKind,
2423 #[serde(default)]
2424 standalone: bool,
2425}
2426
2427fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2428 Ok(crate::worker::active(home)?
2429 .into_iter()
2430 .filter(|job| job.kind == "task")
2431 .collect())
2432}
2433
2434fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2435 task_jobs(home)?
2436 .into_iter()
2437 .filter_map(
2438 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2439 Ok(request) if !request.standalone => Some(Ok(job)),
2440 Ok(_) => None,
2441 Err(error) => Some(Err(error.into())),
2442 },
2443 )
2444 .collect()
2445}
2446
2447#[must_use]
2449pub fn managed(home: &Path) -> bool {
2450 match crate::worker::active(home) {
2451 Err(error) => {
2452 tracing::error!("cannot determine scheduler ownership: {error:#}");
2453 true
2454 }
2455 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2456 Ok(_) => {
2457 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2458 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2459 }
2460 }
2461}
2462
2463pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2465 anyhow::ensure!(
2466 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2467 "the scheduler is stopping; wait for retirement before starting it again"
2468 );
2469 crate::worker::spawn(
2470 home,
2471 "scheduler",
2472 "scheduler",
2473 serde_json::to_value(opts.absolute()?)?,
2474 )
2475 .await
2476}
2477
2478fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2479 home.join("worker-parks").join(format!("{}.json", job.id))
2480}
2481
2482fn request_task_park(
2483 home: &Path,
2484 job: &crate::worker::Job,
2485 cause: crate::run::ParkCause,
2486) -> Result<()> {
2487 let path = park_request_path(home, job);
2488 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2489 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2490 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2491}
2492
2493pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2495 if control == crate::worker::Control::Park {
2496 for job in owned_task_jobs(home)? {
2497 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2498 }
2499 }
2500 crate::worker::request(home, "scheduler", control)
2501}
2502
2503pub async fn restart_detached(home: &Path) -> Result<()> {
2505 let active = crate::worker::active(home)?;
2506 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2507 if prior.is_none() && !managed(home) {
2508 return Ok(());
2509 }
2510 let opts: Opts = match prior {
2511 Some(job) => serde_json::from_value(job.payload)?,
2512 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2513 };
2514 request_control(home, crate::worker::Control::Stop)?;
2515 loop {
2516 let reserved = crate::worker::active(home)?
2517 .iter()
2518 .any(|job| job.kind == "scheduler");
2519 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2520 break;
2521 }
2522 tokio::time::sleep(Duration::from_millis(100)).await;
2523 }
2524 spawn_detached(home, &opts).await?;
2525 Ok(())
2526}
2527
2528pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2530 let opts: Opts = serde_json::from_value(payload)?;
2531 let _ = home;
2532 serve_until(opts, Stop::new()).await
2533}
2534
2535pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2537 let request: TaskWorker = serde_json::from_value(payload)?;
2538 let queue = Queue::at(request.queue);
2539 let key = format!("task:{}", request.task);
2540 let job = task_jobs(home)?
2541 .into_iter()
2542 .find(|job| job.key == key)
2543 .context("task worker reservation is missing")?;
2544 let park_path = park_request_path(home, &job);
2545 let _claim = loop {
2548 match queue.claim(&request.task) {
2549 Ok(claim) => break claim,
2550 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2553 return Err(e.context("claim the task for its worker"));
2554 }
2555 Err(_) => {
2556 sweep_stale_claims(&queue, STALE_CLAIM);
2557 tokio::time::sleep(Duration::from_millis(50)).await;
2558 }
2559 }
2560 };
2561 let mut task = queue.get(&request.task)?;
2562 if !task.status.runnable()
2563 || (!request.standalone
2564 && request.lane
2565 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2566 {
2567 return Ok(());
2568 }
2569 let stop = Stop::new();
2570 let watcher = {
2571 let home = home.to_path_buf();
2572 let stop = stop.clone();
2573 let park_path = park_path.clone();
2574 tokio::spawn(async move {
2575 loop {
2576 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2577 let cause = std::fs::read(&park_path)
2578 .ok()
2579 .and_then(|body| serde_json::from_slice(&body).ok())
2580 .unwrap_or(crate::run::ParkCause::Operator);
2581 stop.park(cause);
2582 return;
2583 }
2584 tokio::time::sleep(Duration::from_millis(200)).await;
2585 }
2586 })
2587 };
2588 let quota = if request.standalone {
2589 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2590 } else {
2591 let status = Arc::new(Mutex::new(Status::new()));
2592 Box::pin(attempt(
2593 &request.opts,
2594 &queue,
2595 &status,
2596 &stop,
2597 crate::graph::Pause::new(),
2598 &mut task,
2599 ))
2600 .await
2601 };
2602 watcher.abort();
2603 let _ = std::fs::remove_file(park_path);
2604 persist_cooldown(home, "a).await?;
2605 Ok(())
2606}
2607
2608pub async fn spawn_claimed(
2611 home: &Path,
2612 opts: &Opts,
2613 queue: &Queue,
2614 task: &Task,
2615 claim: crate::queue::Claim,
2616) -> Result<crate::worker::Job> {
2617 let request = TaskWorker {
2618 opts: opts.absolute()?,
2619 queue: std::path::absolute(queue.root())?,
2620 task: task.id.clone(),
2621 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2622 standalone: true,
2623 };
2624 let job = crate::worker::spawn(
2625 home,
2626 "task",
2627 &format!("task:{}", task.id),
2628 serde_json::to_value(request)?,
2629 )
2630 .await?;
2631 drop(claim);
2632 Ok(job)
2633}
2634
2635pub async fn run_detached_claimed(
2638 opts: &Opts,
2639 queue: &Queue,
2640 task: &mut Task,
2641 claim: crate::queue::Claim,
2642) -> Result<()> {
2643 let home = crate::run::home();
2644 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2645 loop {
2646 crate::worker::active(&home)?;
2649 let current = crate::worker::get(&home, &job.id)?;
2650 if matches!(
2651 current.status,
2652 crate::worker::Status::Finished | crate::worker::Status::Failed
2653 ) {
2654 *task = queue.get(&task.id)?;
2655 anyhow::ensure!(
2656 current.status != crate::worker::Status::Failed,
2657 "task worker failed: {}",
2658 current.result.as_deref().unwrap_or("unknown failure")
2659 );
2660 return Ok(());
2661 }
2662 tokio::time::sleep(Duration::from_millis(200)).await;
2663 }
2664}
2665
2666async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2667 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2668 let _lease = loop {
2669 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2670 break lease;
2671 }
2672 tokio::time::sleep(Duration::from_millis(10)).await;
2673 };
2674 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2675 let path = home.join("task-quota-cooldown.json");
2676 let temporary = path.with_extension("json.tmp");
2677 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2678 std::fs::rename(temporary, path)?;
2679 }
2680 }
2681 Ok(())
2682}
2683
2684fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2685 match std::fs::read(home.join("task-quota-cooldown.json")) {
2686 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2687 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2688 Err(error) => Err(error.into()),
2689 }
2690}
2691
2692pub async fn serve(opts: Opts) -> Result<()> {
2698 tracing::info!(
2699 "automatic update and restart requires magi web; standalone serve does not restart"
2700 );
2701 serve_until(opts, Stop::new()).await
2702}
2703
2704pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2715 let opts = opts.absolute()?;
2716 let home = crate::run::home();
2717 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2718 .context("another scheduler already owns this home")?;
2719 if let Some(reading) = read_status(&home)
2720 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2721 {
2722 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2723 }
2724 if !crate::worker::active(&home)?
2725 .iter()
2726 .any(|job| job.kind == "scheduler")
2727 {
2728 crate::worker::clear_control(&home, "scheduler")?;
2729 }
2730 std::fs::write(
2731 home.join("scheduler-options.json"),
2732 serde_json::to_vec(&opts)?,
2733 )?;
2734 let control = {
2735 let home = home.clone();
2736 let stop = stop.clone();
2737 tokio::spawn(async move {
2738 loop {
2739 match crate::worker::control(&home, "scheduler").ok().flatten() {
2740 Some(crate::worker::Control::Park) => {
2741 let _ = request_control(&home, crate::worker::Control::Park);
2742 stop.park(crate::run::ParkCause::Operator);
2743 break;
2744 }
2745 Some(crate::worker::Control::Stop) => {
2746 stop.stop();
2747 break;
2748 }
2749 None => {}
2750 }
2751 tokio::time::sleep(Duration::from_millis(100)).await;
2752 }
2753 })
2754 };
2755 let signal = {
2756 let stop = stop.clone();
2757 tokio::spawn(async move {
2758 if tokio::signal::ctrl_c().await.is_ok() {
2759 stop.stop();
2760 tracing::info!("scheduler shutdown requested; independent workers continue");
2761 }
2762 })
2763 };
2764
2765 let worktrees_root = opts
2766 .worktrees_root
2767 .clone()
2768 .unwrap_or_else(crate::run::default_worktree_root);
2769 let outcome = drive(
2770 &opts,
2771 &Queue::open(),
2772 &status_path(),
2773 &crate::run::home(),
2774 &worktrees_root,
2775 &stop,
2776 )
2777 .await;
2778
2779 signal.abort();
2780 control.abort();
2781 outcome
2782}
2783
2784async fn drive(
2797 opts: &Opts,
2798 queue: &Queue,
2799 status_file: &Path,
2800 home: &Path,
2801 worktrees_root: &Path,
2802 stop: &Stop,
2803) -> Result<()> {
2804 let recovery_cfg = prepare(&opts.repo, opts).ok();
2812 let daemon_cfg = recovery_cfg
2813 .as_ref()
2814 .map(|c| c.daemon.clone())
2815 .unwrap_or_default();
2816 let reconciled = crate::driver::reconcile(home);
2817 if !reconciled.is_empty() {
2818 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2819 }
2820 let mut initial_status = Status::new();
2821 let owned = owned_task_jobs(home)?;
2822 initial_status.current = current_work(home, Timestamp::now())
2823 .into_iter()
2824 .filter(|current| {
2825 owned
2826 .iter()
2827 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2828 })
2829 .collect();
2830 initial_status.idle = initial_status.current.is_empty();
2831 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2832 recovery_cfg.as_ref(),
2833 daemon_cfg.max_deputies,
2834 ));
2835 let status = Arc::new(Mutex::new(initial_status));
2836 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2837 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2838
2839 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2845
2846 tracing::info!(
2847 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2848 queue.root().display(),
2849 opts.poll.as_secs(),
2850 opts.max_attempts,
2851 concurrency,
2852 if daemon_cfg.pause_for_interrupts {
2853 ", interrupts enabled"
2854 } else {
2855 ""
2856 }
2857 );
2858
2859 let outcome = tokio::select! {
2860 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2861 max_concurrent: concurrency,
2862 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2863 recovery_cfg,
2864 }) => outcome,
2865 () = async {
2866 while !stop.stopped() {
2867 tokio::time::sleep(Duration::from_millis(25)).await;
2868 }
2869 } => {
2870 if stop.parking() {
2871 request_control(home, crate::worker::Control::Park)?;
2872 }
2873 Ok(())
2874 }
2875 };
2876
2877 stop.busy.store(0, Ordering::SeqCst);
2878 beat.abort();
2879 clear_status_at(status_file);
2880 outcome
2881}
2882
2883pub(crate) fn release_watch_settings_from_config(
2886 home: &Path,
2887 repo: &Path,
2888 cfg: &Config,
2889) -> (Vec<PathBuf>, u64) {
2890 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2891 .into_iter()
2892 .map(|r| r.path)
2893 .collect();
2894 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2897 .list()
2898 .into_iter()
2899 .map(|t| t.repo)
2900 .collect();
2901 let runs = home.join("runs");
2902 for id in crate::run::list_ids_in(&runs) {
2903 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2904 .ok()
2905 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2906 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2907 known.extend(repo);
2908 }
2909 for k in known {
2910 let k = k.canonicalize().unwrap_or(k);
2911 if k.join(".git").exists() && !paths.contains(&k) {
2912 paths.push(k);
2913 }
2914 }
2915 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2916 if !paths.contains(&own) {
2917 paths.push(own);
2918 }
2919 (paths, cfg.daemon.release_stall_minutes)
2920}
2921
2922const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2924
2925async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2931 loop {
2932 tokio::time::sleep(HEARTBEAT).await;
2933 let snapshot = {
2934 let mut guard = lock(&status);
2935 guard.updated_at = Timestamp::now();
2936 guard.clone()
2937 };
2938 if let Err(e) = write_status_to(&path, &snapshot) {
2939 tracing::warn!("could not refresh the daemon status file: {e:#}");
2942 }
2943 }
2944}
2945
2946#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2949enum LandResume {
2950 NotLanding,
2953 StillWaiting,
2958 Ready,
2962}
2963
2964fn land_resume_state(task: &Task) -> LandResume {
2968 let Some(run_id) = task.runs.last() else {
2969 return LandResume::NotLanding;
2970 };
2971 let Ok(state) = RunState::load(run_id) else {
2972 return LandResume::NotLanding;
2973 };
2974 if !state.parked {
2975 return LandResume::NotLanding;
2976 }
2977 if state.status == RunStatus::Landing
2978 && state.land_approval.as_ref().is_some_and(|binding| {
2979 ask::Questions::open()
2980 .get(&binding.question)
2981 .is_ok_and(|q| {
2982 q.run == state.id
2983 && q.resolution()
2984 .as_deref()
2985 .is_some_and(crate::merge_decision::stops)
2986 })
2987 })
2988 {
2989 return LandResume::Ready;
2990 }
2991 if !implementers_match(task, &state) {
2992 return LandResume::NotLanding;
2993 }
2994 let node = if state.status == RunStatus::Landing {
2998 land::APPROVAL_NODE
2999 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
3000 crate::github_text::ASK_NODE
3001 } else {
3002 return LandResume::NotLanding;
3003 };
3004 let store = ask::Questions::open();
3005 let waiting = store
3006 .list()
3007 .into_iter()
3008 .filter(|q| &q.run == run_id && q.node == node)
3009 .max_by(|a, b| a.id.cmp(&b.id));
3010 let Some(mut q) = waiting else {
3011 return LandResume::Ready;
3012 };
3013 if !q.status.open() {
3014 if q.resolution().as_deref() == Some(land::REPAIR) {
3017 return if state.liveness(false) == Liveness::Live {
3018 LandResume::StillWaiting
3019 } else {
3020 LandResume::NotLanding
3021 };
3022 }
3023 return LandResume::Ready;
3024 }
3025 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
3032 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
3033 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3034 q.abandon(format!(
3035 "no answer within {}s of asking",
3036 timeout.as_secs().max(1)
3037 ));
3038 if store.put(&mut q).is_ok() {
3041 return LandResume::Ready;
3042 }
3043 }
3044 LandResume::StillWaiting
3045}
3046
3047const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3055
3056const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3068
3069#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3082enum Interrupt {
3083 Idle,
3085 Parking {
3097 parked: Vec<String>,
3098 interrupt_task: String,
3099 },
3100 Running {
3108 parked: Vec<String>,
3109 interrupt_task: String,
3110 },
3111 Resuming { parked: Vec<String> },
3118}
3119
3120fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3141 match state {
3142 Interrupt::Idle => {
3143 if in_flight.len() != 1 {
3155 return Interrupt::Idle;
3156 }
3157 match runnable.iter().find(|t| t.interrupt) {
3158 Some(t) => Interrupt::Parking {
3159 parked: in_flight.to_vec(),
3160 interrupt_task: t.id.clone(),
3161 },
3162 None => Interrupt::Idle,
3163 }
3164 }
3165 Interrupt::Parking {
3166 parked,
3167 interrupt_task,
3168 } => {
3169 if in_flight.iter().any(|id| parked.contains(id)) {
3170 Interrupt::Parking {
3172 parked,
3173 interrupt_task,
3174 }
3175 } else if in_flight.contains(&interrupt_task) {
3176 Interrupt::Running {
3177 parked,
3178 interrupt_task,
3179 }
3180 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3181 Interrupt::Parking {
3185 parked,
3186 interrupt_task,
3187 }
3188 } else {
3189 Interrupt::Resuming { parked }
3194 }
3195 }
3196 Interrupt::Running {
3197 parked,
3198 interrupt_task,
3199 } => {
3200 if in_flight.contains(&interrupt_task) {
3201 Interrupt::Running {
3202 parked,
3203 interrupt_task,
3204 }
3205 } else {
3206 Interrupt::Resuming { parked }
3212 }
3213 }
3214 Interrupt::Resuming { parked } => {
3215 if in_flight.iter().any(|id| parked.contains(id)) {
3216 Interrupt::Idle
3222 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3223 Interrupt::Resuming { parked }
3224 } else {
3225 Interrupt::Idle
3228 }
3229 }
3230 }
3231}
3232
3233fn advance_interrupt_tick(
3239 enabled: bool,
3240 state: Interrupt,
3241 in_flight: &[String],
3242 runnable: &[Task],
3243) -> Interrupt {
3244 if !enabled {
3245 return Interrupt::Idle;
3246 }
3247 advance_interrupt(state, in_flight, runnable)
3248}
3249
3250fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3255 match state {
3256 Interrupt::Idle => candidates,
3257 Interrupt::Parking {
3258 parked,
3259 interrupt_task,
3260 } => {
3261 if in_flight.iter().any(|id| parked.contains(id)) {
3262 Vec::new()
3263 } else {
3264 candidates
3265 .into_iter()
3266 .filter(|t| &t.id == interrupt_task)
3267 .collect()
3268 }
3269 }
3270 Interrupt::Running { .. } => Vec::new(),
3271 Interrupt::Resuming { parked } => candidates
3279 .into_iter()
3280 .find(|t| parked.contains(&t.id))
3281 .into_iter()
3282 .collect(),
3283 }
3284}
3285
3286struct DispatchLimits {
3290 max_concurrent: usize,
3293 pause_for_interrupts: bool,
3295 recovery_cfg: Option<Config>,
3296}
3297
3298#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3307enum PermitKind {
3308 None,
3313 Urgent,
3320 #[default]
3323 Ordinary,
3324}
3325
3326fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3329 if priority {
3330 PermitKind::None
3331 } else if urgent {
3332 PermitKind::Urgent
3333 } else {
3334 PermitKind::Ordinary
3335 }
3336}
3337
3338fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3339 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3340 (0usize, 0usize),
3341 |(ordinary, urgent), request| match request.lane {
3342 PermitKind::Ordinary => (ordinary + 1, urgent),
3343 PermitKind::Urgent => (ordinary, urgent + 1),
3344 PermitKind::None => (ordinary, urgent),
3345 },
3346 );
3347 (
3348 limit.saturating_sub(ordinary),
3349 1usize.saturating_sub(urgent),
3350 )
3351}
3352
3353struct DispatchClaim {
3354 task: Task,
3355 claim: crate::queue::Claim,
3356 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3357}
3358
3359fn claim_dispatch(
3362 queue: &Queue,
3363 candidate: &Task,
3364 cooling_down: bool,
3365 ordinary: &Arc<tokio::sync::Semaphore>,
3366 urgent: &Arc<tokio::sync::Semaphore>,
3367) -> Option<DispatchClaim> {
3368 let Ok(claim) = queue.claim(&candidate.id) else {
3369 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3370 return None;
3371 };
3372 let task = match queue.get(&candidate.id) {
3373 Ok(t) if t.status.runnable() => t,
3374 Ok(_) => return None,
3375 Err(e) => {
3376 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3377 return None;
3378 }
3379 };
3380 let resume = land_resume_state(&task);
3381 if resume == LandResume::StillWaiting {
3382 return None;
3383 }
3384 let priority = resume == LandResume::Ready;
3385 if !priority && cooling_down {
3386 return None;
3387 }
3388 let permit = match permit_kind(priority, task.urgent) {
3391 PermitKind::None => None,
3392 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3393 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3394 };
3395 let stamp = Timestamp::now();
3398 let mut task = task;
3399 task.dispatched_at = Some(stamp);
3400 if let Err(e) = queue.modify(&task.id, |t| {
3401 t.dispatched_at = Some(stamp);
3402 true
3403 }) {
3404 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3405 }
3406 Some(DispatchClaim {
3407 task,
3408 claim,
3409 permit,
3410 })
3411}
3412
3413#[derive(Serialize, Deserialize)]
3432struct ServiceWorker {
3433 opts: Opts,
3434 queue: PathBuf,
3435 worktrees_root: PathBuf,
3436 operation: String,
3437}
3438
3439fn load_conductor(home: &Path) -> Result<Conductor> {
3440 match std::fs::read(home.join("conductor-state.json")) {
3441 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3442 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3443 Err(error) => Err(error.into()),
3444 }
3445}
3446
3447fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3448 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3449 .into_iter()
3450 .map(|c| c.task)
3451 .collect();
3452 let stalled = stalled_tasks(queue, home, Timestamp::now());
3453 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3454 let finished = finished_tasks(queue)
3455 .into_iter()
3456 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3457 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3458 .collect();
3459 let queued = queued_tasks(queue)
3460 .into_iter()
3461 .filter(|t| !live.contains(&t.id))
3462 .collect();
3463 (queued, stalled, finished)
3464}
3465
3466async fn start_service(
3467 home: &Path,
3468 opts: &Opts,
3469 queue: &Queue,
3470 worktrees_root: &Path,
3471 operation: &str,
3472 cfg: Option<&Config>,
3473 busy: bool,
3474) -> Result<()> {
3475 let fetch_cfg = if operation == "fetch" {
3478 match prepare(&opts.repo, opts) {
3479 Ok(cfg) => Some(cfg),
3480 Err(error) => {
3481 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3482 return Ok(());
3483 }
3484 }
3485 } else {
3486 None
3487 };
3488 let cfg = fetch_cfg.as_ref().or(cfg);
3489 let ticks = home.join("service-ticks");
3490 std::fs::create_dir_all(&ticks)?;
3491 let receipt = ticks.join(format!("daemon-{operation}"));
3492 let last = std::fs::read(&receipt)
3493 .ok()
3494 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3495 let interval = match operation {
3496 "disk-pressure" => 30,
3497 "cache" => CACHE_CHECK_INTERVAL_SECS,
3498 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3499 _ => opts.poll.as_secs().max(1),
3500 };
3501 let now = Timestamp::now();
3502 if !cache_check_due(last, now, interval) {
3503 return Ok(());
3504 }
3505 let key = format!("service:daemon-{operation}");
3506 if crate::worker::active(home)?
3507 .iter()
3508 .any(|job| job.key == key)
3509 {
3510 return Ok(());
3511 }
3512 let eligible = match operation {
3513 "conductor" => {
3514 let (queued, stalled, finished) = conductor_work(queue, home);
3515 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3516 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3517 }
3518 "maintenance" => Questions::at(home.join("questions"))
3519 .list()
3520 .iter()
3521 .any(|q| {
3522 q.chosen_action().is_some_and(|action| action.maintenance())
3523 && !home
3524 .join("choice-actions")
3525 .join(format!("{}.json", q.id))
3526 .exists()
3527 }),
3528 "housekeeping" => !busy && runnable(queue).is_empty(),
3529 "cache" => {
3530 !busy
3531 && !runnable(queue).is_empty()
3532 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3533 }
3534 "disk-pressure" => busy,
3535 "fetch" => {
3536 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3537 }
3538 _ => false,
3539 };
3540 if !eligible {
3541 return Ok(());
3542 }
3543 let request = ServiceWorker {
3544 opts: opts.absolute()?,
3545 queue: std::path::absolute(queue.root())?,
3546 worktrees_root: std::path::absolute(worktrees_root)?,
3547 operation: operation.to_owned(),
3548 };
3549 if let Err(error) =
3550 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3551 {
3552 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3553 }
3554 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3555 Ok(())
3556}
3557
3558pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3561 let request: ServiceWorker = serde_json::from_value(payload)?;
3562 let opts = &request.opts;
3563 let queue = Queue::at(request.queue);
3564 let questions = Questions::at(home.join("questions"));
3565 match request.operation.as_str() {
3566 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3567 "conductor" => {
3568 let path = home.join("conductor-state.json");
3569 let mut conductor = load_conductor(home)?;
3570 let (queued, stalled, finished) = conductor_work(&queue, home);
3571 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3572 && conductor.worth_a_look(&queue, &stalled, &finished)
3573 {
3574 let cfg = prepare(&opts.repo, opts)?;
3575 conductor
3576 .maybe_run(
3577 &cfg,
3578 &opts.repo,
3579 &queue,
3580 &questions,
3581 home,
3582 &queued,
3583 &stalled,
3584 &finished,
3585 opts.max_attempts,
3586 )
3587 .await;
3588 let temporary = path.with_extension("json.tmp");
3589 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3590 std::fs::rename(temporary, path)?;
3591 }
3592 }
3593 "fetch" => {
3594 let cfg = prepare(&opts.repo, opts)?;
3595 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3596 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3597 tracing::debug!("fetch origins: {outcome:?}");
3598 }
3599 }
3600 "cache" => {
3601 if task_jobs(home)?.is_empty() {
3602 let mut checked = None;
3603 maybe_prune_cache_between_runs(
3604 &opts.repo,
3605 opts,
3606 home,
3607 &Stop::new(),
3608 &mut checked,
3609 Timestamp::now(),
3610 )
3611 .await;
3612 }
3613 }
3614 "disk-pressure" => {
3615 if let Ok(cfg) = prepare(&opts.repo, opts) {
3616 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3617 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3618 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3619 }
3620 for current in current_work(home, Timestamp::now()) {
3621 if let Ok(state) = RunState::load_under(¤t.run, home)
3622 && let Some(reason) = disk_gate(&state.repo, &state.config)
3623 {
3624 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3625 job.key.strip_prefix("task:") == Some(current.task.as_str())
3626 }) {
3627 request_task_park(
3628 home,
3629 &job,
3630 crate::run::ParkCause::Interrupt {
3631 reason: reason.clone(),
3632 },
3633 )?;
3634 }
3635 notices::raise_in(
3636 home,
3637 Notice::warn(
3638 "disk:pressure",
3639 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3640 ),
3641 );
3642 }
3643 }
3644 }
3645 }
3646 "housekeeping" => {
3647 if task_jobs(home)?.is_empty() {
3648 if let Ok(cfg) = prepare(&opts.repo, opts) {
3649 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3650 }
3651 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3652 resweep_superseded_attempts(&queue, home);
3653 triage_held(&queue, home, opts).await;
3654 }
3655 }
3656 operation => bail!("unknown daemon service {operation}"),
3657 }
3658 Ok(())
3659}
3660
3661fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3665 let key = format!("task:{}", request.task);
3666 let payload = serde_json::to_value(request)?;
3667 if crate::worker::active(home)?
3668 .iter()
3669 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3670 {
3671 tracing::warn!(
3672 "task {} acquired a competing worker reservation: {error:#}",
3673 crate::run::short_of(&request.task)
3674 );
3675 return Ok(());
3676 }
3677 tracing::error!("task worker startup stopped dispatch: {error:#}");
3678 notices::raise_in(
3679 home,
3680 Notice::error(
3681 "loop:worker-startup",
3682 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
3685 ).link(Link::Task { id: request.task.clone() }),
3686 );
3687 Err(error).with_context(|| {
3688 format!(
3689 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3690 crate::run::short_of(&request.task)
3691 )
3692 })
3693}
3694
3695async fn poll(
3696 opts: &Opts,
3697 queue: &Queue,
3698 status: &Arc<Mutex<Status>>,
3699 home: &Path,
3700 worktrees_root: &Path,
3701 stop: &Stop,
3702 limits: DispatchLimits,
3703) -> Result<()> {
3704 let mut attempted = Vec::new();
3705 let interrupt_path = home.join("scheduler-interrupt.json");
3706 let mut interrupt = match std::fs::read(&interrupt_path) {
3707 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3708 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3709 Err(error) => return Err(error.into()),
3710 };
3711 let mut previous = std::collections::BTreeSet::<String>::new();
3712 while !stop.stopped() {
3713 let jobs: Vec<_> = crate::worker::active(home)?
3714 .into_iter()
3715 .filter(|job| job.kind == "task")
3716 .collect();
3717 let requests: Vec<_> = jobs
3718 .iter()
3719 .map(|job| {
3720 serde_json::from_value::<TaskWorker>(job.payload.clone())
3721 .with_context(|| format!("read live task worker {}", job.id))
3722 })
3723 .collect::<Result<_>>()?;
3724 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3725 let owned: std::collections::BTreeSet<_> = requests
3726 .iter()
3727 .filter(|r| !r.standalone)
3728 .map(|r| r.task.clone())
3729 .collect();
3730 let current: Vec<_> = requests
3731 .iter()
3732 .zip(&jobs)
3733 .filter(|(request, _)| !request.standalone)
3734 .map(|(request, job)| Current {
3735 worker: Some(job.id.clone()),
3736 task: request.task.clone(),
3737 run: queue
3738 .get(&request.task)
3739 .ok()
3740 .and_then(|task| task.runs.last().cloned())
3741 .unwrap_or_default(),
3742 })
3743 .collect();
3744 {
3745 let mut snapshot = lock(status);
3746 snapshot.polls += 1;
3747 snapshot.completed += previous.difference(&owned).count();
3748 snapshot.current = current;
3749 snapshot.idle = owned.is_empty();
3750 }
3751 previous = owned.clone();
3752 stop.busy.store(owned.len(), Ordering::SeqCst);
3753
3754 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3757 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3758 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3759 let mut permits = Vec::new();
3760 sweep_stale_claims(queue, STALE_CLAIM);
3761 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3762 reclaim_abandoned_runs(home, Timestamp::now());
3763 let questions = Questions::at(home.join("questions"));
3764
3765 crate::task_chat::sweep(
3768 queue,
3769 &crate::talk::Talks::at(home.join("talks")),
3770 &questions,
3771 home,
3772 );
3773 repair_choice_offers(&questions);
3774 resolve_blockers(queue, &questions);
3775 apply_task_choice_actions(queue, &questions, home);
3776 triage::apply_answers(queue, &questions);
3777 reconcile_task_questions(queue, &questions);
3778 let talks = crate::talk::Talks::at(home.join("talks"));
3779 crate::chat_notify::sweep(queue, &talks, home);
3780 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
3781 crate::chat_watch::sweep(home, &talks);
3782 let mut turns = tokio::task::JoinSet::new();
3783 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3784 while turns.join_next().await.is_some() {}
3786
3787 crate::consult::start_services(
3788 home,
3789 &opts.repo,
3790 limits.recovery_cfg.as_ref(),
3791 opts.config.as_deref(),
3792 )
3793 .await?;
3794 for operation in [
3795 "conductor",
3796 "maintenance",
3797 "housekeeping",
3798 "disk-pressure",
3799 "cache",
3800 "fetch",
3801 ] {
3802 start_service(
3803 home,
3804 opts,
3805 queue,
3806 worktrees_root,
3807 operation,
3808 limits.recovery_cfg.as_ref(),
3809 !live.is_empty(),
3810 )
3811 .await?;
3812 }
3813 let latest = crate::worker::active(home)?;
3814 if latest.iter().any(|job| job.key == "service:daemon-cache")
3815 || latest
3816 .iter()
3817 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3818 {
3819 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3820 continue;
3821 }
3822 let candidates: Vec<Task> = runnable(queue)
3823 .into_iter()
3824 .filter(|task| !live.contains(&task.id))
3825 .filter(|task| !opts.once || !attempted.contains(&task.id))
3826 .collect();
3827 let in_flight: Vec<_> = owned.iter().cloned().collect();
3828 let prior_interrupt = std::mem::discriminant(&interrupt);
3829 interrupt = advance_interrupt_tick(
3830 limits.pause_for_interrupts,
3831 interrupt,
3832 &in_flight,
3833 &candidates,
3834 );
3835 if prior_interrupt != std::mem::discriminant(&interrupt) {
3838 let temporary = interrupt_path.with_extension("json.tmp");
3839 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3840 std::fs::rename(temporary, &interrupt_path)?;
3841 }
3842 if let Interrupt::Parking {
3843 parked,
3844 interrupt_task,
3845 } = &interrupt
3846 {
3847 for job in jobs.iter().filter(|job| {
3848 parked
3849 .iter()
3850 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3851 }) {
3852 if crate::worker::control(home, &job.key)?.is_none() {
3853 request_task_park(
3854 home,
3855 job,
3856 crate::run::ParkCause::Interrupt {
3857 reason: format!(
3858 "task {} asked to run first",
3859 crate::run::short_of(interrupt_task)
3860 ),
3861 },
3862 )?;
3863 }
3864 }
3865 }
3866 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3867 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3868 let mut started = false;
3869 let mut resync = false;
3870 for candidate in candidates {
3871 if stop.stopped() {
3872 break;
3873 }
3874 let Some(DispatchClaim {
3875 task,
3876 claim,
3877 permit,
3878 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3879 else {
3880 continue;
3881 };
3882 if crate::worker::active(home)?
3883 .iter()
3884 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3885 {
3886 drop(claim);
3887 resync = true;
3888 break;
3889 }
3890 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3891 let request = TaskWorker {
3892 opts: opts.absolute()?,
3893 queue: std::path::absolute(queue.root())?,
3894 task: task.id.clone(),
3895 lane,
3896 standalone: false,
3897 };
3898 if let Err(error) = crate::worker::spawn(
3901 home,
3902 "task",
3903 &format!("task:{}", task.id),
3904 serde_json::to_value(&request)?,
3905 )
3906 .await
3907 {
3908 task_worker_spawn_error(home, &request, error)?;
3909 drop(claim);
3910 resync = true;
3911 break;
3912 }
3913 drop(claim);
3914 permits.extend(permit);
3915 attempted.push(task.id);
3916 started = true;
3917 }
3918 if started || resync {
3919 continue;
3920 }
3921 if owned.is_empty() && opts.once {
3922 while crate::worker::active(home)?
3923 .iter()
3924 .any(|job| job.key == "service:daemon-housekeeping")
3925 {
3926 stop.idle(RECHECK_WHILE_BUSY).await;
3927 if stop.stopped() {
3928 return Ok(());
3929 }
3930 }
3931 break;
3932 }
3933 stop.idle(if live.is_empty() {
3934 opts.poll
3935 } else {
3936 RECHECK_WHILE_BUSY.min(opts.poll)
3937 })
3938 .await;
3939 }
3940 stop.busy.store(0, Ordering::SeqCst);
3943 Ok(())
3944}
3945
3946async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
3950 let busy = |error: &anyhow::Error| {
3951 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
3952 error.kind() == std::io::ErrorKind::WouldBlock
3953 || cfg!(windows) && error.raw_os_error() == Some(33)
3954 })
3955 };
3956 match crate::pr_head::owner_lock(home, id) {
3957 Ok(lock) => return Ok(lock),
3958 Err(error) if busy(&error) => {}
3959 Err(error) => return Err(error),
3960 }
3961 let wait = RunState::load_under(id, home)?
3962 .config
3963 .graph
3964 .timeout_resource_wait;
3965 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
3966}
3967
3968async fn public_resume_owner(
3972 home: &Path,
3973 runs: &[String],
3974 repo: &Path,
3975) -> Result<Option<RunState>> {
3976 let mut owner = None;
3977 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
3978 let _lock = resume_owner_lock(home, id).await?;
3979 let state = RunState::load_under(id, home)?;
3980 if state.released()
3981 || state.retired_to.is_some()
3982 || state.pr_taken_over_by.is_some()
3983 || !crate::handover::landing_checkpoint(&state)
3984 {
3985 continue;
3986 }
3987 anyhow::ensure!(
3988 crate::git::toplevel(&state.repo).await?.canonicalize()?
3989 == crate::git::toplevel(repo).await?.canonicalize()?,
3990 "public owner repository differs; inspect run lineage"
3991 );
3992 anyhow::ensure!(
3993 owner.is_none(),
3994 "multiple retained public owners; inspect run lineage"
3995 );
3996 anyhow::ensure!(
3997 crate::handover::can_resume(&state),
3998 "public owner cannot resume"
3999 );
4000 let claimed = is_working_on_other(home, id, Timestamp::now())?;
4001 anyhow::ensure!(
4002 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
4003 "public owner's driver is live or unknown; wait before retrying"
4004 );
4005 owner = Some(state);
4006 }
4007 Ok(owner)
4008}
4009
4010async fn attempt(
4016 opts: &Opts,
4017 queue: &Queue,
4018 status: &Arc<Mutex<Status>>,
4019 stop: &Stop,
4020 interrupt_pause: crate::graph::Pause,
4021 task: &mut Task,
4022) -> Vec<QuotaLoss> {
4023 let repo = repo_for(task, &opts.repo);
4024 tracing::info!(
4025 "task {} — {} (repo {})",
4026 task.short(),
4027 task.title,
4028 repo.display()
4029 );
4030
4031 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
4032 Ok(owner) => owner,
4033 Err(e) => {
4034 task.hold_machine(Some(format!(
4035 "Cannot verify existing public owner: {e:#}; no attempt started."
4036 )));
4037 record(queue, task);
4038 return Vec::new();
4039 }
4040 };
4041 let resume_runs = public_owner
4042 .as_ref()
4043 .map(|s| vec![s.id.clone()])
4044 .unwrap_or_else(|| task.runs.clone());
4045
4046 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4050 && previous
4051 .merge_decision
4052 .as_ref()
4053 .is_some_and(|decision| !decision.closed)
4054 && previous.status != RunStatus::Landing
4055 && !crate::merge_decision::resumable_hold(&previous)
4056 {
4057 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4058 record(queue, task);
4059 return Vec::new();
4060 }
4061 let stop_owner = resume_runs
4062 .last()
4063 .and_then(|id| RunState::load(id).ok())
4064 .filter(|state| {
4065 crate::merge_decision::resumable_hold(state)
4066 || state.status == RunStatus::Landing
4067 && state.land_approval.as_ref().is_some_and(|binding| {
4068 Questions::open().get(&binding.question).is_ok_and(|q| {
4069 q.run == state.id
4070 && q.resolution()
4071 .as_deref()
4072 .is_some_and(crate::merge_decision::stops)
4073 })
4074 })
4075 });
4076 if let Some(owner) = stop_owner {
4077 let mut runner = match Runner::resume(&owner.id) {
4078 Ok(runner) => runner,
4079 Err(e) => {
4080 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4081 record(queue, task);
4082 return Vec::new();
4083 }
4084 };
4085 runner.on_pause(stop.pause());
4086 runner.watch_interrupt(interrupt_pause);
4087 let quota_before = runner.state.quota.clone();
4088 task.start(owner.id.clone());
4089 record(queue, task);
4090 lock(status).current.push(Current {
4091 worker: None,
4092 task: task.id.clone(),
4093 run: owner.id,
4094 });
4095 runner.state.next_pass_task_entry = true;
4096 let result = runner.execute().await;
4097 return finish_attempt(
4098 opts.max_attempts,
4099 queue,
4100 task,
4101 &runner.state,
4102 "a_before,
4103 result,
4104 );
4105 }
4106
4107 if let Some(owner) = &public_owner
4108 && (task.fresh_start || !may_resume_run(owner, task))
4109 {
4110 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4111 record(queue, task);
4112 return Vec::new();
4113 }
4114
4115 let unfinished = (!task.fresh_start)
4116 .then(|| unfinished_run(&resume_runs, task.short()))
4117 .flatten()
4118 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4119 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4120 if !task.fresh_start
4121 && task.pr_recovery.is_none()
4122 && task.review_branch.is_none()
4123 && task.resume_override.is_none()
4124 && previous
4125 .as_ref()
4126 .is_some_and(|run| run.retired_to.is_some())
4127 && unfinished.is_none()
4128 {
4129 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4130 record(queue, task);
4131 return Vec::new();
4132 }
4133
4134 let saved_recovery = previous.as_ref().filter(|run| {
4135 task.replacement.is_none()
4136 && task.pr_recovery.as_ref().is_some_and(|selected| {
4137 selected.check_task(&task.id, &task.instruction).is_ok()
4138 && selected_starter(selected, Some(run), unfinished.as_deref())
4139 == Starter::Resume(run.id.clone())
4140 })
4141 });
4142
4143 let explicit = match &task.overrides {
4147 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4148 Some(_) => opts.config.as_deref(),
4149 None => opts.config.as_deref(),
4150 };
4151 let (mut config, layers) = if let Some(run) = saved_recovery {
4152 let mut saved = run.clone();
4155 if let Err(e) = saved.configure_recovery_resume() {
4156 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4157 record(queue, task);
4158 return Vec::new();
4159 }
4160 (saved.config, Vec::new())
4161 } else {
4162 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4163 Ok(loaded) => loaded,
4164 Err(e) => {
4165 let reason = format!("[config] {e:#}");
4166 record_config_failure(task, &e);
4167 task.last_error = Some(reason.clone());
4168 task.hold_machine(Some(reason));
4169 record(queue, task);
4170 return Vec::new();
4171 }
4172 };
4173 if let Err(e) = apply_options(&mut config, opts, task) {
4174 task.hold_manual(Some(format!(
4175 "[config] {e:#}; correct the merge override before releasing the task"
4176 )));
4177 record(queue, task);
4178 return Vec::new();
4179 }
4180 if let Some(selected) = &task.pr_recovery
4181 && let Err(e) = selected.configure(&mut config)
4182 {
4183 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4184 record(queue, task);
4185 return Vec::new();
4186 }
4187 (config, layers)
4188 };
4189 let config_evidence = crate::pr_recovery::ConfigEvidence {
4190 layers,
4191 overrides: task.overrides.clone(),
4192 daemon_merge: task
4193 .overrides
4194 .as_ref()
4195 .is_none_or(|o| o.inherit_daemon_options)
4196 .then(|| opts.merge.clone())
4197 .flatten(),
4198 };
4199 apply_task_implementers(&mut config, task);
4200 if let Some(replacement) = &task.replacement {
4201 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4202 old.liveness(false) == Liveness::Dead
4203 && old.merge_decision.as_ref().is_some_and(|d| {
4204 d.closed
4205 && d.question == replacement.question
4206 && d.replacement_run.as_ref().is_none_or(|run| {
4207 task.runs.last() == Some(run)
4208 && RunState::load(run).is_ok_and(|current| {
4209 current.replaces.as_ref().is_some_and(|pin| {
4210 pin.question == replacement.question
4211 && pin.agent == replacement.agent
4212 })
4213 })
4214 })
4215 })
4216 });
4217 if !safe {
4218 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4219 record(queue, task);
4220 return Vec::new();
4221 }
4222 match config.agent(&replacement.agent) {
4223 Ok(agent) if crate::agent::installed(agent) => {}
4224 _ => {
4225 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4226 record(queue, task);
4227 return Vec::new();
4228 }
4229 }
4230 config.graph.implementers = 1;
4231 config.graph.land_approval = true;
4232 config.merge.mode = MergeMode::Pr;
4233 config.graph.selected_implementer = Some(replacement.agent.clone());
4234 }
4235 let p = phrases(&config.graph.language);
4236 let start_failed = p.could_not_start;
4237 if let Ok(base) =
4242 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4243 .await
4244 && let Ok(head) =
4245 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4246 {
4247 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4248 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4249 let source = crate::git::rev_parse(&repo, branch)
4250 .await
4251 .unwrap_or_default();
4252 inputs.push_str(&format!(";review:{branch}:{source}"));
4253 }
4254 if let Some(reason) = task.recovery_exhausted(
4255 &head,
4256 &inputs,
4257 task.attempt_limit
4258 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4259 ) {
4260 task.hold_unrequested_retry(Some(reason));
4261 record(queue, task);
4262 return Vec::new();
4263 }
4264 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4265 }
4266
4267 if let Some(reason) = disk_gate(&repo, &config) {
4275 task.notice_cause("disk-space", None, None);
4276 task.last_error = Some(reason.clone());
4277 task.hold_machine(Some(reason.clone()));
4278 record(queue, task);
4279 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4280 notices::raise(
4283 Notice::warn(
4284 &format!("disk:{}", repo.display()),
4285 format!("Task {} held for want of free disk space: {reason}. Free some, then release it from the queue.", task.id),
4286 )
4287 .link(Link::Task {
4288 id: task.id.clone(),
4289 }),
4290 );
4291 return Vec::new();
4292 }
4293
4294 let review_branch = take_review_branch(task, previous.as_ref());
4319 let branch_exists = match &review_branch {
4320 Some(branch) => crate::git::branch_exists(&repo, branch)
4321 .await
4322 .unwrap_or(false),
4323 None => false,
4324 };
4325 let proposed = review_retry_starter(
4329 task,
4330 previous.as_ref(),
4331 review_branch.as_deref(),
4332 branch_exists,
4333 unfinished.as_deref(),
4334 );
4335 let retry_selected = task.replacement.is_some()
4336 && task.resumption.is_some()
4337 && !task.fresh_start
4338 && previous.as_ref().is_some_and(|run| {
4339 run.replaces.is_some()
4340 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4341 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4342 });
4343 let stop_pin = task
4344 .resume_override
4345 .as_ref()
4346 .filter(|pin| {
4347 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4348 q.node == land::APPROVAL_NODE
4349 && q.resolution()
4350 .as_deref()
4351 .is_some_and(crate::merge_decision::stops)
4352 })
4353 })
4354 .and_then(|pin| pin.pinned_run.clone());
4355 let starter = if let Some(owner) = &public_owner {
4356 Starter::Resume(owner.id.clone())
4357 } else if let Some(run) = stop_pin {
4358 Starter::Resume(run)
4361 } else if retry_selected {
4362 Starter::Resume(previous.as_ref().unwrap().id.clone())
4363 } else if let Some(selected) = &task.pr_recovery {
4364 if selected.check_task(&task.id, &task.instruction).is_ok() {
4365 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4366 } else {
4367 Starter::Review(selected.branch.clone())
4368 }
4369 } else {
4370 starter_with_implementers(task, previous.as_ref(), proposed)
4371 };
4372 if let Some(pin) = &task.resume_override
4375 && let Some(run) = &pin.pinned_run
4376 && starter != Starter::Resume(run.clone())
4377 {
4378 let run = run.clone();
4379 task.notice_cause("resume-owner", Some(&run), None);
4380 task.hold_machine(Some(format!(
4381 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4382 task.id
4383 )));
4384 record(queue, task);
4385 return Vec::new();
4386 }
4387 if task.replacement.is_some()
4388 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4389 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4390 {
4391 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4392 record(queue, task);
4393 return Vec::new();
4394 }
4395 if let Some(reason) = retained_attempt_hold(
4396 task,
4397 &starter,
4398 config.disk.retained_attempt_limit,
4399 RunState::load,
4400 ) {
4401 if let Starter::Review(branch) = &starter {
4404 task.review_branch = Some(branch.clone());
4405 }
4406 task.hold_unrequested_retry(Some(reason));
4407 record(queue, task);
4408 return Vec::new();
4409 }
4410 let attachments = match task_attachments(queue, task) {
4415 Ok(a) => a,
4416 Err(e) => {
4417 task.attempts += 1;
4418 task.fail_startup_target(
4419 "startup-attachments",
4420 e.name.clone(),
4421 format!("{start_failed}{e:#}"),
4422 opts.max_attempts,
4423 );
4424 record(queue, task);
4425 return Vec::new();
4426 }
4427 };
4428 let startup_pause = stop.pause();
4429 let startup_interrupt = interrupt_pause.clone();
4430 let wait = crate::repo_lock::WaitPolicy::new(
4431 Duration::from_secs(config.graph.timeout_resource_wait),
4432 move || startup_pause.parked() || startup_interrupt.parked(),
4433 );
4434 let started = wait.scope(Box::pin(async { match &starter {
4437 Starter::Review(_) if task.pr_recovery.is_some() => {
4438 let selected = task.pr_recovery.clone().expect("selected recovery");
4439 match selected.check_task(&task.id, &task.instruction) {
4440 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4441 .await
4442 .map(|mut r| {
4443 r.state.attachments = attachments.clone();
4444 r
4445 }),
4446 Err(e) => Err(e),
4447 }
4448 }
4449 Starter::Review(branch) => {
4450 tracing::info!(
4451 "task {} reopens `{branch}` as a review-only pass",
4452 task.short()
4453 );
4454 let takeover = crate::handover::Takeover {
4457 earlier: task.earlier_attempts().to_vec(),
4458 home: crate::run::home(),
4459 choice: take_divergence_answer(branch, &config.merge.remote, task),
4460 task_instruction: Some(instruction_for(task)),
4461 };
4462 Runner::review_taking_over(
4463 &repo,
4464 branch,
4465 config,
4466 Some(takeover),
4467 crate::run::Origin::queue(&task.id),
4468 )
4469 .await
4470 }
4471 Starter::Resume(id) => {
4472 tracing::info!("resuming run {id} rather than competing again");
4473 Runner::resume(id).and_then(|mut r| {
4474 if retry_selected {
4475 for candidate in &mut r.state.candidates {
4478 candidate.failed = None;
4479 candidate.empty = false;
4480 candidate.commits = 0;
4481 }
4482 r.state.status = RunStatus::Implementing;
4483 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4484 }
4485 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4486 r.state.sources = task.sources.clone();
4487 }
4488 if r.state.prep_failure.is_some() {
4489 let approval = task
4490 .resumption
4491 .as_ref()
4492 .context("preparation retry requires an explicit task release")?;
4493 r.retry_preparation(approval.at)?;
4494 }
4495 if let Some(instruction) =
4496 prepare_instruction(&starter, Some(&r.state.instruction), task)
4497 {
4498 r.state.instruction = instruction;
4499 }
4500 r.state.attachments = attachments.clone();
4501 if task.replacement.is_some() {
4502 r.state.config.merge.mode = MergeMode::Pr;
4503 r.state.config.graph.land_approval = true;
4504 }
4505 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4509 .overrides
4510 .as_ref()
4511 .and_then(|o| o.merge.as_deref())
4512 .and_then(|m| merge_mode(m).ok())
4513 {
4514 r.state.config.merge.mode = mode;
4515 }
4516 Ok(r)
4517 })
4518 }
4519 Starter::Start => {
4520 if let Some(branch) = &review_branch {
4521 tracing::warn!(
4522 "conductor chose review for task {} but branch `{branch}` no longer \
4523 exists; requeuing as a fresh competition instead",
4524 task.short()
4525 );
4526 }
4527 let instruction = prepare_instruction(&starter, None, task)
4528 .unwrap_or_else(|| task.instruction.clone());
4529 Runner::start_naming(
4530 &repo,
4531 instruction,
4532 &task.title,
4533 config,
4534 crate::run::Origin::queue(&task.id),
4535 )
4536 .await
4537 .map(|mut r| {
4538 r.state.attachments = attachments.clone();
4539 r.state.sources = task.sources.clone();
4540 r
4541 })
4542 }
4543 }})).await;
4544 let mut runner = match started {
4545 Ok(r) => r,
4546 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4547 task.last_error = Some(e.to_string());
4548 if stop.parking() || interrupt_pause.parked() {
4549 let attempts = task.attempts;
4550 task.park(e.to_string());
4551 task.attempts = attempts;
4553 } else {
4554 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4555 }
4556 record(queue, task);
4557 return Vec::new();
4558 }
4559 Err(e) if task.pr_recovery.is_some() => {
4563 task.startup_notice_cause("startup-public-head");
4564 task.attempts += 1;
4565 if let Some(partial) = e.downcast_ref::<crate::pr_recovery::StartupFailed>() {
4566 task.link_run(&partial.run);
4567 task.record_pass(&partial.run, 0, task.attempts, None);
4568 }
4569 task.last_error = Some(format!("{start_failed}{e:#}"));
4570 task.hold_unrequested_retry(Some(format!(
4571 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4572 )));
4573 record(queue, task);
4574 return Vec::new();
4575 }
4576 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4577 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4578 Some(r) => (p.handover_refused)(r),
4579 None => format!("{e:#}"),
4580 };
4581 task.startup_notice_cause("startup-handover");
4582 let reason = format!("{start_failed}{detail}");
4583 task.last_error = Some(reason.clone());
4584 let branch = match &starter {
4587 Starter::Review(branch) => Some(branch.clone()),
4588 _ => None,
4589 };
4590 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4594 record(queue, task);
4595 tracing::warn!(
4596 "holding {} for a branch it cannot take over: {e:#}",
4597 task.short()
4598 );
4599 return Vec::new();
4600 }
4601 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4606 let d = e
4607 .downcast_ref::<crate::reconcile::Diverged>()
4608 .expect("checked by the guard");
4609 let mut q = ask::Question::new(
4610 task.id.clone(),
4611 crate::reconcile::NODE.to_owned(),
4612 crate::reconcile::SEAT.to_owned(),
4613 d.summary(),
4614 d.detail(),
4615 d.choices(),
4616 );
4617 match Questions::open().put(&mut q) {
4618 Ok(()) => {
4619 task.last_error = Some(format!("{e:#}"));
4620 task.review_branch = Some(d.branch.clone());
4623 task.review_instruction = Some(task.instruction.clone());
4624 task.block(vec![q.id.clone()], Some(d.summary()));
4625 }
4626 Err(put) => {
4627 tracing::warn!("could not file the divergence question: {put:#}");
4628 task.attempts += 1;
4629 task.fail_startup(
4630 "startup-question-store",
4631 format!("{start_failed}{e:#}"),
4632 opts.max_attempts,
4633 );
4634 }
4635 }
4636 record(queue, task);
4637 return Vec::new();
4638 }
4639 Err(e) => {
4640 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4646 && let Starter::Review(branch) = &starter
4647 {
4648 task.review_branch = Some(branch.clone());
4649 task.review_instruction = Some(task.instruction.clone());
4650 task.last_error = Some(format!("{start_failed}{e:#}"));
4651 task.status = crate::queue::TaskStatus::Failed;
4652 record(queue, task);
4653 return Vec::new();
4654 }
4655 task.attempts += 1;
4656 task.last_error = Some(format!("{start_failed}{e:#}"));
4657 task.startup_notice_cause("startup-source");
4658 task.hold_unrequested_retry(Some(format!(
4659 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4660 )));
4661 record(queue, task);
4662 return Vec::new();
4663 }
4664 };
4665 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4666 runner.state.recovery_config = Some(config_evidence);
4667 }
4668 if let Some(replacement) = &task.replacement {
4669 if matches!(starter, Starter::Start) {
4670 runner.state.instruction.push_str(&format!(
4671 "\n\n# Replacement review context\n{}",
4672 replacement.context
4673 ));
4674 }
4675 runner.state.replaces = Some(replacement.clone());
4676 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4677 if let Ok(mut old) = RunState::load(&replacement.run)
4678 && let Some(d) = &mut old.merge_decision
4679 {
4680 d.replacement_run = Some(runner.state.id.clone());
4681 old.event(
4682 "replacement",
4683 format!(
4684 "replacement run {} owns the next attempt; old PR remains closed",
4685 runner.state.id
4686 ),
4687 );
4688 if let Err(e) = old.save() {
4689 task.hold_manual(Some(format!(
4690 "Could not link replacement history: {e}; no agents dispatched"
4691 )));
4692 record(queue, task);
4693 return Vec::new();
4694 }
4695 }
4696 }
4697 runner.state.task_instruction = Some(task.instruction.clone());
4698 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4701 if runner.state.origin_chat.is_none() {
4702 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4703 }
4704 runner.on_pause(stop.pause());
4706 runner.watch_interrupt(interrupt_pause);
4710
4711 let run = runner.state.id.clone();
4714 task.start(run.clone());
4715 record(queue, task);
4716 lock(status).current.push(Current {
4717 worker: None,
4718 task: task.id.clone(),
4719 run,
4720 });
4721
4722 let quota_before = runner.state.quota.clone();
4725 runner.state.next_pass_task_entry = true;
4726 let result = runner.execute().await;
4727 finish_attempt(
4728 opts.max_attempts,
4729 queue,
4730 task,
4731 &runner.state,
4732 "a_before,
4733 result,
4734 )
4735}
4736
4737pub fn finish_attempt(
4742 max_attempts: usize,
4743 queue: &Queue,
4744 task: &mut Task,
4745 state: &RunState,
4746 quota_before: &[QuotaLoss],
4747 result: Result<()>,
4748) -> Vec<QuotaLoss> {
4749 let detail = match result {
4750 Ok(()) => describe(state),
4751 Err(e) => format!("{e:#}"),
4752 };
4753 let fresh = losses_this_attempt(quota_before, &state.quota);
4754 let verdict = Verdict {
4755 status: state.status,
4756 left_pr: state.pr.is_some(),
4759 quota_hit: !fresh.is_empty(),
4765 parked: state.parked,
4769 no_viable_candidates: state.viable().is_empty(),
4772 };
4773 if let Some(replacement) = &state.replaces
4774 && let Some(pr) = &state.pr
4775 && let Ok(mut old) = RunState::load(&replacement.run)
4776 && let Some(d) = &mut old.merge_decision
4777 {
4778 d.replacement_pr = Some(pr.url.clone());
4779 let _ = old.save();
4780 }
4781 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4782 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4783 task.hold_unrequested_retry(Some(format!(
4784 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4785 task.short()
4786 )));
4787 }
4788 if task.status == TaskStatus::Done {
4789 supersede_prior_runs(task, &crate::run::home());
4790 }
4791 record(queue, task);
4792 tracing::info!(
4793 "task {} is {} after run {} ({})",
4794 task.short(),
4795 task.status.as_str(),
4796 state.short(),
4797 label(state.status)
4798 );
4799 fresh
4800}
4801
4802pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4806 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4809 && !awaiting_resume_with(task, RunState::load)
4810 && withheld_text_wait(task).is_none()
4814 && !(task.fresh_start && task.replacement.is_some())
4815 {
4816 let why = task.last_error.clone().map_or_else(
4817 || "the run did not finish".to_owned(),
4818 |e| format!("the run did not finish: {e}"),
4819 );
4820 task.hold_unrequested_retry(Some(format!(
4821 "{why}. It was started by hand, so it is not retried \
4822 automatically; `magi task release` retries it."
4823 )));
4824 record(queue, task);
4825 }
4826}
4827
4828#[must_use]
4832pub fn loop_would_resume(run: &str) -> bool {
4833 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4834}
4835
4836#[must_use]
4846pub fn foreign_loop(
4847 reading: Option<&Reading>,
4848 now: Timestamp,
4849 own_pid: u32,
4850) -> Option<Option<u32>> {
4851 let reading = reading.filter(|r| r.running(now))?;
4852 match reading.pid {
4853 Some(pid) if pid == own_pid => None,
4854 pid => Some(pid),
4855 }
4856}
4857
4858fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4859 reading.worker_protocol != Some(1)
4860 && (foreign_loop(Some(reading), now, own_pid).is_some()
4861 || reading
4862 .pid
4863 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4864}
4865
4866pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4877 run_claimed_until(opts, queue, task, &Stop::new()).await;
4878}
4879
4880async fn run_claimed_until(
4881 opts: &Opts,
4882 queue: &Queue,
4883 task: &mut Task,
4884 stop: &Stop,
4885) -> Vec<QuotaLoss> {
4886 let status = Arc::new(Mutex::new(Status::new()));
4887 let mut quota = Box::pin(attempt(
4890 opts,
4891 queue,
4892 &status,
4893 stop,
4894 crate::graph::Pause::new(),
4895 task,
4896 ))
4897 .await;
4898 let mut resumes = 0;
4907 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
4908 let Some(timeout) = withheld_text_wait(task) else {
4909 break;
4910 };
4911 eprintln!(
4912 "waiting for the owner's answer on a withheld pull request text \
4913 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
4914 );
4915 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
4918 let ready = loop {
4919 match land_resume_state(task) {
4920 LandResume::StillWaiting
4921 if !stop.parking() && std::time::Instant::now() < limit =>
4922 {
4923 tokio::time::sleep(WITHHELD_POLL).await;
4924 }
4925 LandResume::Ready => break true,
4926 _ => break false,
4927 }
4928 };
4929 if !ready || stop.parking() {
4930 break;
4931 }
4932 resumes += 1;
4933 quota.extend(
4934 Box::pin(attempt(
4935 opts,
4936 queue,
4937 &status,
4938 stop,
4939 crate::graph::Pause::new(),
4940 task,
4941 ))
4942 .await,
4943 );
4944 }
4945 hold_if_runnable(queue, task);
4946 quota
4947}
4948
4949const MAX_WITHHELD_RESUMES: u32 = 3;
4952
4953const WITHHELD_POLL: Duration = Duration::from_secs(15);
4955
4956fn withheld_text_wait(task: &Task) -> Option<u64> {
4959 let id = task.runs.last()?;
4960 let s = RunState::load(id).ok()?;
4961 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
4962 .then_some(s.config.graph.answer_timeout)
4963}
4964
4965fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
4975 after
4976 .iter()
4977 .filter(|q| !before.contains(q))
4978 .cloned()
4979 .collect()
4980}
4981
4982fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
4985 if quota.is_empty() {
4986 return None;
4987 }
4988 let with_hint = quota.iter().find(|q| q.reset.is_some());
4989 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
4990 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
4991 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
4992 Some(
4993 now.checked_add(jiff::SignedDuration::from_secs(secs))
4994 .unwrap_or(Timestamp::MAX),
4995 )
4996}
4997
4998fn implementers_match(task: &Task, state: &RunState) -> bool {
5000 task.replacement.as_ref().map_or_else(
5001 || {
5002 task.implementers()
5003 .is_none_or(|n| n == state.config.graph.implementers)
5004 },
5005 |replacement| {
5006 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
5007 && state.config.graph.implementers == 1
5008 },
5009 )
5010}
5011
5012fn starter_with_implementers(
5014 task: &Task,
5015 previous: Option<&RunState>,
5016 proposed: Starter,
5017) -> Starter {
5018 if previous.is_some_and(|s| !implementers_match(task, s)) {
5019 task.review_of
5020 .as_ref()
5021 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
5022 } else {
5023 proposed
5024 }
5025}
5026
5027fn apply_task_implementers(config: &mut Config, task: &Task) {
5029 if let Some(n) = task.implementers() {
5030 config.graph.implementers = n;
5031 }
5032}
5033
5034#[cfg(test)]
5039fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5040 let explicit = match &task.overrides {
5041 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5042 _ => opts.config.as_deref(),
5043 };
5044 let (mut config, _) = Config::discover(repo, explicit)?;
5045 apply_options(&mut config, opts, task)?;
5046 Ok(config)
5047}
5048
5049fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5050 if task
5051 .overrides
5052 .as_ref()
5053 .is_none_or(|o| o.inherit_daemon_options)
5054 && let Some(mode) = opts.merge.as_deref()
5055 {
5056 config.merge.mode = merge_mode(mode)?;
5057 }
5058 if let Some(o) = &task.overrides {
5059 o.apply(config);
5060 }
5061 Ok(())
5062}
5063
5064fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5066 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5067 if let Some(mode) = &opts.merge {
5068 config.merge.mode = merge_mode(mode)?;
5069 }
5070 Ok(config)
5071}
5072
5073async fn maybe_prune_cache_between_runs(
5078 repo: &Path,
5079 opts: &Opts,
5080 home: &Path,
5081 stop: &Stop,
5082 last_checked: &mut Option<Timestamp>,
5083 now: Timestamp,
5084) {
5085 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5086 return;
5087 }
5088 *last_checked = Some(now);
5089 let cfg = match prepare(repo, opts) {
5090 Ok(cfg) => cfg,
5091 Err(e) => {
5092 tracing::warn!("cache check: no config: {e:#}");
5093 return;
5094 }
5095 };
5096 match clean::prune_cache_if_over_limit(&cfg, home) {
5097 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5098 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5099 pruned.files,
5100 pruned.freed
5101 ),
5102 Ok(_) => {}
5103 Err(e) => {
5104 tracing::warn!("housekeep: prune cache: {e:#}");
5105 notices::raise_in(
5106 home,
5107 Notice::warn(
5108 "housekeep:cache",
5109 format!(
5110 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5111 e.to_string()
5112 .lines()
5113 .next()
5114 .unwrap_or("unknown cleanup error")
5115 ),
5116 ),
5117 );
5118 }
5119 }
5120}
5121
5122fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5126 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5127}
5128
5129async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5148 let cfg = match prepare(repo, opts) {
5149 Ok(cfg) => cfg,
5150 Err(e) => {
5151 tracing::warn!("housekeep: no config: {e:#}");
5152 return;
5153 }
5154 };
5155 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5164 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5165 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5170 let mut extra = Vec::new();
5171 if out.unreadable > 0 {
5172 extra.push(format!("{} unreadable", out.unreadable));
5173 }
5174 if out.orphaned_worktrees > 0 {
5175 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5176 }
5177 let detail = if extra.is_empty() {
5178 String::new()
5179 } else {
5180 format!(" ({})", extra.join(", "))
5181 };
5182 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5183 }
5184 if out.external_merges_recorded > 0 {
5185 tracing::info!(
5186 "housekeep: recorded {} run(s) as merged externally",
5187 out.external_merges_recorded
5188 );
5189 }
5190 if out.stale_pr_states_repaired > 0 {
5191 tracing::info!(
5192 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5193 out.stale_pr_states_repaired
5194 );
5195 }
5196 if out.cache_files > 0 {
5197 tracing::info!(
5198 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5199 out.cache_files,
5200 out.cache_freed
5201 );
5202 }
5203 if out.questions_abandoned > 0 {
5204 tracing::info!(
5205 "housekeep: abandoned {} question(s) left open by a finished run",
5206 out.questions_abandoned
5207 );
5208 }
5209}
5210
5211async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5220 let questions = Questions::at(home.join("questions"));
5221 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5222 if report.is_empty() {
5223 return;
5224 }
5225 if !report.quarantined.is_empty() {
5226 tracing::info!(
5227 "triage: held {} blocked task(s) whose blocked-on task or \
5228 question no longer exists: {}",
5229 report.quarantined.len(),
5230 report.quarantined.join(", ")
5231 );
5232 }
5233 if !report.resumed.is_empty() {
5234 tracing::info!(
5235 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5236 report.resumed.len(),
5237 report.resumed.join(", ")
5238 );
5239 }
5240 if !report.asked.is_empty() {
5241 tracing::info!(
5242 "triage: asked about {} held task(s): {}",
5243 report.asked.len(),
5244 report.asked.join(", ")
5245 );
5246 }
5247 if !report.answered.is_empty() {
5248 tracing::info!(
5249 "triage: applied {} operator answer(s): {}",
5250 report.answered.len(),
5251 report.answered.join(", ")
5252 );
5253 }
5254}
5255
5256pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5263 disk_gate_with(repo, config, crate::disk::free_bytes)
5264}
5265
5266fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5270 repo: &Path,
5271 config: &Config,
5272 free_bytes: F,
5273) -> Option<String> {
5274 let min = config.disk.min_free_bytes;
5275 if min == 0 {
5276 return None;
5277 }
5278 let mut paths = vec![repo.to_owned()];
5279 if let Some(cache) = config.cache_dir() {
5280 paths.push(cache);
5281 }
5282 if let Some(root) = &config.graph.worktree_root {
5283 paths.push(root.clone());
5284 }
5285 for path in paths {
5286 let probe = if path == repo {
5287 path.as_path()
5288 } else {
5289 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5290 };
5291 let reason = match free_bytes(probe) {
5292 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5293 Err(e) => Some(crate::disk::unmeasured_in(
5294 probe,
5295 &e,
5296 &config.graph.language,
5297 )),
5298 };
5299 if reason.is_some() {
5300 return reason;
5301 }
5302 }
5303 None
5304}
5305
5306const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5313
5314const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5318
5319fn quota_wait(
5328 reset_at: Option<Timestamp>,
5329 now: Timestamp,
5330 fallback: Duration,
5331 cap: Duration,
5332) -> Duration {
5333 match reset_at {
5334 Some(at) if at > now => {
5335 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5336 Duration::from_secs(secs).min(cap)
5337 }
5338 _ => fallback,
5339 }
5340}
5341
5342fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5356 parse_reset_hint_zoned(text, now)
5357 .or_else(|| parse_reset_hint_dated(text))
5358 .or_else(|| parse_reset_hint_relative(text, recorded))
5359}
5360
5361fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5365 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5366 let mut rest = rest.trim();
5367 if rest.is_empty() {
5368 return None;
5369 }
5370 let mut total: i64 = 0;
5371 let mut matched = false;
5372 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5373 if let Some((digits, tail)) = rest.split_once(unit)
5374 && !digits.is_empty()
5375 && digits.bytes().all(|b| b.is_ascii_digit())
5376 {
5377 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5378 rest = tail;
5379 matched = true;
5380 }
5381 }
5382 if !rest.is_empty() || !matched {
5383 return None;
5384 }
5385 recorded
5386 .checked_add(jiff::SignedDuration::from_secs(total))
5387 .ok()
5388}
5389
5390fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5394 let clock = clock.trim().to_lowercase();
5395 let (digits, pm) = clock
5396 .strip_suffix("am")
5397 .map(|d| (d, false))
5398 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5399 let (h, m) = digits.trim().split_once(':')?;
5400 let mut hour: i8 = h.trim().parse().ok()?;
5401 let minute: i8 = m.trim().parse().ok()?;
5402 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5403 return None;
5404 }
5405 if pm && hour != 12 {
5406 hour += 12;
5407 } else if !pm && hour == 12 {
5408 hour = 0;
5409 }
5410 Some((hour, minute))
5411}
5412
5413fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5418 let open = text.find('(')?;
5419 let close = text.rfind(')')?;
5420 if close <= open {
5421 return None;
5422 }
5423 let zone = text[open + 1..close].trim();
5424 let (hour, minute) = parse_12h_clock(&text[..open])?;
5425 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5426 let candidate = now
5427 .to_zoned(tz)
5428 .with()
5429 .hour(hour)
5430 .minute(minute)
5431 .second(0)
5432 .millisecond(0)
5433 .microsecond(0)
5434 .nanosecond(0)
5435 .build()
5436 .ok()?;
5437 let mut at = candidate.timestamp();
5438 if at <= now {
5439 at += jiff::SignedDuration::from_hours(24);
5440 }
5441 Some(at)
5442}
5443
5444fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5453 let words: Vec<&str> = text.split_whitespace().collect();
5454 if words.len() < 5 {
5455 return None;
5456 }
5457 (0..=words.len() - 5)
5458 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5459}
5460
5461fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5467 if trailing.is_some_and(|next| next.starts_with('(')) {
5468 return None;
5469 }
5470 let month = month_number(window[0])?;
5471 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5472 let day_digits = ["st", "nd", "rd", "th"]
5473 .iter()
5474 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5475 let day: i8 = day_digits.parse().ok()?;
5476 let year_token = window[2];
5477 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5478 return None;
5479 }
5480 let year: i16 = year_token.parse().ok()?;
5481 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5485 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5486 let date = jiff::civil::Date::new(year, month, day).ok()?;
5487 let candidate = date
5488 .at(hour, minute, 0, 0)
5489 .to_zoned(jiff::tz::TimeZone::UTC)
5490 .ok()?;
5491 Some(candidate.timestamp())
5492}
5493
5494fn month_number(name: &str) -> Option<i8> {
5497 const NAMES: [&str; 12] = [
5498 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5499 ];
5500 let lower = name.to_lowercase();
5501 NAMES
5502 .iter()
5503 .position(|n| *n == lower.as_str())
5504 .map(|i| i as i8 + 1)
5505}
5506
5507#[cfg(test)]
5509fn exhausted_review_budget(state: &RunState) -> bool {
5510 !crate::handover::can_resume(state)
5513 && state.status == RunStatus::Blocked
5514 && !state.released()
5515 && state.retired_to.is_none()
5516}
5517
5518fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5555 unfinished_run_with(runs, short, RunState::load)
5556}
5557
5558pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5561where
5562 F: FnMut(&str) -> Result<RunState>,
5563{
5564 let mut id = runs.last()?.clone();
5565 let mut visited = std::collections::BTreeSet::new();
5566 let mut repo = None;
5567 loop {
5568 if !runs.contains(&id) || !visited.insert(id.clone()) {
5569 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5570 return None;
5571 }
5572 let state = match load(&id) {
5573 Ok(state) if state.id == id => state,
5574 Ok(_) => return None,
5575 Err(e) => {
5576 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5577 return None;
5578 }
5579 };
5580 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5581 return None;
5582 }
5583 repo = Some(state.repo.clone());
5584 if let Some(authority) = state.retired_to {
5585 id = authority;
5586 continue;
5587 }
5588 return (crate::handover::can_resume(&state)
5589 && state.liveness(false) != crate::run::Liveness::Live)
5590 .then_some(id);
5591 }
5592}
5593
5594fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5602where
5603 F: FnOnce(&str) -> Result<RunState>,
5604{
5605 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5607 || task.fresh_start
5608 || task.review_branch.is_some()
5609 {
5610 return false;
5611 }
5612 let Some(id) = task.runs.last() else {
5613 return false;
5614 };
5615 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5616}
5617
5618#[derive(Debug, Clone, PartialEq, Eq)]
5621enum Starter {
5622 Review(String),
5625 Resume(String),
5627 Start,
5629}
5630
5631fn retained_attempt_hold(
5635 task: &Task,
5636 starter: &Starter,
5637 limit: usize,
5638 mut load: impl FnMut(&str) -> Result<RunState>,
5639) -> Option<String> {
5640 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5641 return None;
5642 }
5643 let mut retained = 0;
5644 for id in task.runs.iter().collect::<std::collections::BTreeSet<_>>() {
5645 let state = match load(id) {
5646 Ok(state) if state.id == *id => state,
5647 _ => {
5648 return Some(format!(
5649 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5650 ));
5651 }
5652 };
5653 if state.retired_to.is_some() || state.released() {
5654 continue;
5655 }
5656 retained += 1;
5659 }
5660 (retained >= limit).then(|| format!(
5661 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5662 ))
5663}
5664
5665fn selected_starter(
5667 selected: &crate::pr_recovery::Selection,
5668 previous: Option<&RunState>,
5669 unfinished: Option<&str>,
5670) -> Starter {
5671 if let Some(id) = unfinished
5672 && previous.is_some_and(|r| {
5673 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5674 })
5675 {
5676 Starter::Resume(id.to_owned())
5677 } else {
5678 Starter::Review(selected.branch.clone())
5679 }
5680}
5681
5682fn take_divergence_answer(
5686 branch: &str,
5687 remote: &str,
5688 task: &mut Task,
5689) -> Option<crate::reconcile::Choice> {
5690 let summary = crate::reconcile::summary_for(branch, remote);
5691 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5692 (a.question == summary)
5693 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5694 .flatten()
5695 .map(|c| (i, c))
5696 })?;
5697 task.answers.remove(idx);
5698 Some(choice)
5699}
5700
5701fn choose_starter(
5713 review_branch: Option<&str>,
5714 branch_exists: bool,
5715 unfinished: Option<&str>,
5716) -> Starter {
5717 match review_branch {
5718 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5719 Some(_) => Starter::Start,
5720 None => match unfinished {
5721 Some(id) => Starter::Resume(id.to_owned()),
5722 None => Starter::Start,
5723 },
5724 }
5725}
5726
5727fn review_retry_starter(
5731 task: &Task,
5732 previous: Option<&RunState>,
5733 branch: Option<&str>,
5734 exists: bool,
5735 unfinished: Option<&str>,
5736) -> Starter {
5737 if let Some(run) = previous
5738 && unfinished == Some(run.id.as_str())
5739 && run.reviewed_commits.is_some()
5740 && branch.is_some_and(|branch| {
5741 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
5742 })
5743 && may_resume_run(run, task)
5744 {
5745 return Starter::Resume(run.id.clone());
5746 }
5747 match (branch, unfinished, &task.review_of) {
5748 (None, None, Some(branch)) => Starter::Review(branch.clone()),
5749 _ => choose_starter(branch, exists, unfinished),
5750 }
5751}
5752
5753fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5755 let branch = task.review_branch.take();
5756 let instruction = task.review_instruction.take();
5757 branch.filter(|branch| match &instruction {
5758 Some(authored) => *authored == task.instruction,
5759 None => previous.map_or_else(
5762 || task.review_of.as_ref() == Some(branch),
5763 |run| compatible_run_instruction(run, task),
5764 ),
5765 })
5766}
5767
5768fn may_resume_run(run: &RunState, task: &Task) -> bool {
5771 task.resume_override
5772 .as_ref()
5773 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5774 || compatible_run_instruction(run, task)
5775}
5776
5777pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5779 if let Some(authored) = &run.task_instruction {
5780 return compatible_instruction(authored, task);
5781 }
5782 if compatible_instruction(&run.instruction, task) {
5783 return true;
5784 }
5785 if run.reviewed_commits.is_some() {
5788 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5789 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5790 }
5791 if let Some(branch) = &task.review_of {
5792 let canonical = format!(
5793 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5794 );
5795 let generated = format!(
5796 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5797 );
5798 return task.instruction == canonical && run.instruction.starts_with(&generated);
5799 }
5800 }
5801 false
5802}
5803
5804pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5806 strip_answers_block(old, task) == task.instruction
5807}
5808
5809fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5812 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5813 return fallback.to_path_buf();
5814 }
5815 task.repo.clone()
5816}
5817
5818const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5822
5823fn answers_block(task: &Task, count: usize) -> String {
5825 let mut s = ANSWERS_HEADER.to_owned();
5826 for a in &task.answers[..count] {
5827 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5828 }
5829 s
5830}
5831
5832fn resumption_block(r: &crate::queue::Resumption) -> String {
5835 format!(
5836 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5837 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5838 r.generation, r.at, r.instruction, r.answers_through,
5839 )
5840}
5841
5842fn append_answers(base: &str, task: &Task) -> String {
5844 let mut s = base.to_owned();
5845 if !task.answers.is_empty() {
5846 s.push_str(&answers_block(task, task.answers.len()));
5847 }
5848 if let Some(r) = &task.resumption {
5849 s.push_str(&resumption_block(r));
5850 }
5851 s
5852}
5853
5854fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5858 let mut instruction = instruction;
5859 for approval in task
5860 .resumption
5861 .iter()
5862 .chain(task.resumption_history.iter().rev())
5863 {
5864 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5865 instruction = base;
5866 break;
5867 }
5868 }
5869 for count in (1..=task.answers.len()).rev() {
5870 let block = answers_block(task, count);
5871 if let Some(base) = instruction.strip_suffix(&block) {
5872 return base;
5873 }
5874 }
5875 instruction
5876}
5877
5878fn instruction_for(task: &Task) -> String {
5886 append_answers(&task.instruction, task)
5887}
5888
5889fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
5901 append_answers(strip_answers_block(old_instruction, task), task)
5902}
5903
5904fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
5906 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
5907 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
5908 if let Some(cause) = task.hold_notice_cause.as_mut() {
5909 cause.target = failure.map(|f| f.target.clone());
5910 }
5911}
5912
5913#[derive(Debug, thiserror::Error)]
5914#[error("attachment `{name}` is recorded on the task but {path} is missing")]
5915struct MissingAttachment {
5916 name: String,
5917 path: PathBuf,
5918}
5919
5920fn task_attachments(
5922 queue: &Queue,
5923 task: &Task,
5924) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
5925 let paths = queue.attachment_paths(task);
5926 for (name, path) in task.attachments.iter().zip(&paths) {
5927 if !path.is_file() {
5928 return Err(MissingAttachment {
5929 name: name.clone(),
5930 path: path.clone(),
5931 });
5932 }
5933 }
5934 Ok(paths)
5935}
5936
5937fn prepare_instruction(
5948 starter: &Starter,
5949 old_instruction: Option<&str>,
5950 task: &Task,
5951) -> Option<String> {
5952 match starter {
5953 Starter::Start => Some(instruction_for(task)),
5954 Starter::Resume(_) => Some(resumed_instruction(
5955 old_instruction.expect("a resumed run always has a prior instruction"),
5956 task,
5957 )),
5958 Starter::Review(_) => None,
5959 }
5960}
5961
5962fn record(queue: &Queue, task: &mut Task) {
5966 if let Err(e) = queue.put(task) {
5967 tracing::error!("could not record task {}: {e:#}", task.short());
5968 let notice = Notice::error(
5969 "loop:record",
5970 format!(
5971 "Task {} state could not be saved: {}. Check the disk.",
5972 task.id,
5973 e.to_string().lines().next().unwrap_or("unknown save error")
5974 ),
5975 )
5976 .link(Link::Task {
5977 id: task.id.clone(),
5978 });
5979 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
5981 Some(home) => notices::raise_in(home, notice),
5982 None => notices::raise(notice),
5983 }
5984 }
5985}
5986
5987fn runnable(queue: &Queue) -> Vec<Task> {
5993 let mut tasks: Vec<Task> = queue
5994 .list()
5995 .into_iter()
5996 .filter(|t| t.status.runnable())
5997 .collect();
5998 tasks.sort_unstable_by(crate::queue::dispatch_order);
5999 tasks
6000}
6001
6002fn describe(state: &RunState) -> String {
6016 let p = phrases(&state.config.graph.language);
6017 let mut detail = if state.status == RunStatus::Stalled {
6018 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
6019 seats.sort_unstable();
6020 seats.dedup();
6021 if seats.is_empty() {
6022 p.quorum_lost.to_owned()
6023 } else {
6024 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
6025 }
6026 } else {
6027 format!("{}{}", p.run_ended, state.status.display_label())
6028 };
6029 if let Some(last) = state.events.last() {
6030 let unanswered = state
6034 .reviews
6035 .last()
6036 .filter(|r| {
6037 state.status == RunStatus::Blocked
6038 && last.node == "review"
6039 && r.incomplete()
6040 && r.blocking == 0
6041 && r.round == state.config.graph.review_rounds
6042 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6043 })
6044 .map(|r| (r.expected - r.answered, r.round));
6045 match unanswered {
6046 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6047 detail.push_str(&format!(
6048 " ({}: {})",
6049 last.node,
6050 (p.reviewers_never_answered)(missing, rounds)
6051 ));
6052 }
6053 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6054 }
6055 }
6056 detail.push_str(&format!(" [run {}]", state.id));
6057 detail
6058}
6059
6060const DIAGNOSTIC_MAX: usize = 4_000;
6066
6067const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6072
6073fn diagnostic(state: &RunState) -> Option<String> {
6087 let mut parts: Vec<String> = Vec::new();
6088
6089 for o in state.gate.iter().filter(|o| !o.ok()) {
6091 parts.push(format!(
6092 "gate `{}` failed ({}):\n{}",
6093 o.command,
6094 notices::exit_status(o.code),
6095 crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6096 ));
6097 }
6098
6099 if let Some(last) = state
6102 .events
6103 .iter()
6104 .rev()
6105 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6106 {
6107 parts.push(last.message.clone());
6108 }
6109
6110 if state.viable().is_empty() {
6117 for c in &state.candidates {
6118 if let Some(evidence) = &c.verified_noop {
6119 parts.push(format!(
6120 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6121 c.label
6122 ));
6123 } else if !c.summary.trim().is_empty() {
6124 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6125 } else if let Some(why) = &c.failed {
6126 parts.push(format!("candidate {}: {why}", c.label));
6127 }
6128 }
6129 }
6130
6131 if parts.is_empty() {
6132 return None;
6133 }
6134 Some(crate::run::tail(
6139 &parts.join("\n\n"),
6140 DIAGNOSTIC_MAX.saturating_sub(100),
6141 ))
6142}
6143
6144fn label(status: RunStatus) -> &'static str {
6152 status.as_str()
6153}
6154
6155pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6157 match mode {
6158 "none" => Ok(MergeMode::None),
6159 "local" => Ok(MergeMode::Local),
6160 "pr" => Ok(MergeMode::Pr),
6161 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6162 }
6163}
6164
6165fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6170 mutex
6171 .lock()
6172 .unwrap_or_else(std::sync::PoisonError::into_inner)
6173}
6174
6175#[cfg(test)]
6176mod tests {
6177 use super::*;
6178 use crate::queue::{Source, TaskStatus};
6179 use crate::run::{Candidate, CommandOutcome};
6180 use pretty_assertions::assert_eq;
6181
6182 fn task() -> Task {
6183 Task::new(
6184 "add retries".to_owned(),
6185 "add retries".to_owned(),
6186 PathBuf::from("/repo"),
6187 Source::Human,
6188 )
6189 }
6190
6191 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6192 let job = crate::worker::Job {
6193 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6194 sequence: 1,
6195 kind: "task".into(),
6196 key: format!("task:{}", request.task),
6197 payload: serde_json::to_value(request).unwrap(),
6198 generation: "a".repeat(64),
6199 version: None,
6200 status: crate::worker::Status::Starting,
6201 result: None,
6202 pid: None,
6203 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6204 finished_at: None,
6205 };
6206 let dir = home.join("workers/jobs").join(&job.id);
6207 std::fs::create_dir_all(&dir).unwrap();
6208 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6209 job
6210 }
6211
6212 #[test]
6213 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6214 let home = tempfile::tempdir().unwrap();
6215 let request = TaskWorker {
6216 opts: Opts::default(),
6217 queue: home.path().join("queue"),
6218 task: "startup".into(),
6219 lane: PermitKind::Ordinary,
6220 standalone: false,
6221 };
6222 let job = starting_task_reservation(home.path(), &request);
6223 let error = task_worker_spawn_error(
6224 home.path(),
6225 &request,
6226 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6227 )
6228 .expect_err("our own reservation must not turn a failed spawn into a resync");
6229 let active = crate::worker::active(home.path()).unwrap();
6230 assert_eq!(active.len(), 1);
6231 assert_eq!(active[0].id, job.id);
6232 assert_eq!(active[0].status, crate::worker::Status::Starting);
6233 assert!(
6234 crate::worker::control(home.path(), &job.key)
6235 .unwrap()
6236 .is_none()
6237 );
6238 let notices = notices::Notices::at(home.path().join("notifications")).list();
6239 assert_eq!(notices.len(), 1);
6240 assert_eq!(notices[0].key, "loop:worker-startup");
6241 assert!(!notices[0].message.contains("private-error-detail"));
6242 assert!(notices[0].message.contains(&request.task));
6243 assert!(
6244 notices[0]
6245 .message
6246 .contains("could not be started or acknowledged")
6247 );
6248 assert!(notices[0].message.contains("worker.log"));
6249 assert_eq!(
6250 notices[0].link,
6251 Some(Link::Task {
6252 id: request.task.clone()
6253 })
6254 );
6255 assert!(format!("{error:#}").contains("private-error-detail"));
6256 }
6257
6258 #[test]
6259 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6260 let home = tempfile::tempdir().unwrap();
6261 let mut request = TaskWorker {
6262 opts: Opts::default(),
6263 queue: home.path().join("queue"),
6264 task: "handoff".into(),
6265 lane: PermitKind::Ordinary,
6266 standalone: true,
6267 };
6268 let job = starting_task_reservation(home.path(), &request);
6269 request.standalone = false;
6270 task_worker_spawn_error(
6271 home.path(),
6272 &request,
6273 anyhow::anyhow!("operation key is already reserved for a different request"),
6274 )
6275 .expect("a different standalone request is the handoff race, not our startup failure");
6276 let active = crate::worker::active(home.path()).unwrap();
6277 assert_eq!(active.len(), 1);
6278 assert_eq!(active[0].id, job.id);
6279 assert_eq!(active[0].payload, job.payload);
6280 assert!(
6281 notices::Notices::at(home.path().join("notifications"))
6282 .list()
6283 .is_empty()
6284 );
6285 }
6286
6287 #[test]
6288 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6289 let home = tempfile::tempdir().unwrap();
6290 let queue = Queue::at(home.path().join("queue"));
6291 let mut task = task();
6292 task.runs = vec!["previous".into()];
6293 queue.put(&mut task).unwrap();
6294 let request = TaskWorker {
6295 opts: Opts::default(),
6296 queue: home.path().join("queue"),
6297 task: task.id.clone(),
6298 lane: PermitKind::Ordinary,
6299 standalone: false,
6300 };
6301 let job = starting_task_reservation(home.path(), &request);
6302 let claims = current_work(home.path(), Timestamp::now());
6303 assert_eq!(claims.len(), 1);
6304 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6305 assert_eq!(claims[0].run, "previous");
6306 assert!(other_claims_run(&claims, "previous", None));
6307 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6310 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6311 }
6312
6313 #[test]
6314 fn admission_excludes_only_the_callers_exact_reservation() {
6315 let own = Current {
6316 worker: Some("own".into()),
6317 task: "same-task".into(),
6318 run: "previous".into(),
6319 };
6320 assert!(!other_claims_run(
6321 std::slice::from_ref(&own),
6322 "previous",
6323 Some("own")
6324 ));
6325 assert!(other_claims_run(
6326 std::slice::from_ref(&own),
6327 "previous",
6328 None
6329 ));
6330 for worker in [None, Some("other".into())] {
6331 let other = Current {
6332 worker,
6333 ..own.clone()
6334 };
6335 assert!(other_claims_run(
6336 &[own.clone(), other],
6337 "previous",
6338 Some("own")
6339 ));
6340 }
6341 let legacy: Current =
6342 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6343 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6344 }
6345
6346 #[test]
6347 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6348 let mut task = task();
6349 let mut run = run_state(RunStatus::Reviewing);
6350 run.task_instruction = Some(task.instruction.clone());
6351 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6352 let branch = "magi/test/B";
6353 run.candidates.push(crate::run::Candidate {
6354 branch: branch.into(),
6355 ..candidate('B', "preserved work", false, None)
6356 });
6357 task.runs = vec![run.id.clone()];
6358 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6359 assert_eq!(
6360 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6361 Starter::Resume(run.id.clone())
6362 );
6363 assert!(matches!(
6364 review_retry_starter(
6365 &task,
6366 Some(&run),
6367 Some("different"),
6368 true,
6369 unfinished.as_deref()
6370 ),
6371 Starter::Review(_)
6372 ));
6373 assert_eq!(
6374 review_retry_starter(
6375 &task,
6376 Some(&run),
6377 Some(branch),
6378 false,
6379 unfinished.as_deref()
6380 ),
6381 Starter::Start
6382 );
6383 task.instruction.push_str(" changed criteria");
6384 assert!(matches!(
6385 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6386 Starter::Review(_)
6387 ));
6388 assert!(matches!(
6389 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6390 Starter::Review(_)
6391 ));
6392 }
6393
6394 #[test]
6395 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6396 let dir = tempfile::tempdir().unwrap();
6397 let queue = Queue::at(dir.path().join("queue"));
6398 let mut task = task();
6399 task.status = TaskStatus::Running;
6400 task.attempts = 1;
6401 queue.put(&mut task).unwrap();
6402 let live = std::collections::BTreeSet::from([task.id.clone()]);
6403 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6404 drop(queue.claim(&task.id).unwrap());
6405 let saved = queue.get(&task.id).unwrap();
6406 assert_eq!(saved.status, TaskStatus::Running);
6407 assert_eq!(saved.attempts, 1);
6408 assert!(saved.last_error.is_none());
6409 }
6410
6411 #[test]
6412 fn standalone_workers_do_not_consume_scheduler_lanes() {
6413 let request = |task: &str, lane, standalone| TaskWorker {
6414 opts: Opts::default(),
6415 queue: PathBuf::from("/queue"),
6416 task: task.to_owned(),
6417 lane,
6418 standalone,
6419 };
6420 let mut jobs = vec![
6421 request("ordinary", PermitKind::Ordinary, false),
6422 request("standalone-urgent", PermitKind::Urgent, true),
6423 request("standalone-ordinary", PermitKind::Ordinary, true),
6424 request("decision", PermitKind::None, false),
6425 ];
6426 assert_eq!(available_slots(&jobs, 2), (1, 1));
6427 jobs.push(request("urgent", PermitKind::Urgent, false));
6428 assert_eq!(available_slots(&jobs, 2), (1, 0));
6429 assert_eq!(available_slots(&jobs, 0), (0, 0));
6430 }
6431
6432 #[test]
6433 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6434 let now = Timestamp::now();
6435 let mut reading = Reading {
6436 pid: Some(std::process::id()),
6437 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6438 ..Reading::default()
6439 };
6440 assert!(!reading.running(now));
6441 assert!(legacy_scheduler_running(&reading, now, 0));
6442 reading.worker_protocol = Some(1);
6443 assert!(!legacy_scheduler_running(&reading, now, 0));
6444 }
6445
6446 #[test]
6447 fn selected_public_head_overrides_historical_resume_and_pin() {
6448 let mut t = task();
6449 let mut old = run_state(RunStatus::Blocked);
6450 let selection = crate::pr_recovery::Selection {
6451 url: "url".into(),
6452 branch: format!("magi/task-{}", t.id),
6453 head: "a".repeat(40),
6454 source_head: None,
6455 source_run: None,
6456 source_proof: None,
6457 disposition: Some(crate::pr_recovery::Disposition::Review),
6458 instruction: t.instruction.clone(),
6459 approved_at: Timestamp::now(),
6460 };
6461 t.resume_override = Some(crate::queue::OperatorResume {
6462 pinned_run: Some(old.id.clone()),
6463 question_id: "choice".into(),
6464 at: Timestamp::now(),
6465 conductor_rehold: None,
6466 forced: false,
6467 });
6468 assert_eq!(
6469 selected_starter(&selection, Some(&old), Some(&old.id)),
6470 Starter::Review(selection.branch.clone())
6471 );
6472 old.pr_recovery = Some(selection.clone());
6473 assert_eq!(
6474 selected_starter(&selection, Some(&old), Some(&old.id)),
6475 Starter::Review(selection.branch.clone())
6476 );
6477 old.reviewed_commits = Some(vec!["implementation".into()]);
6478 assert_eq!(
6479 selected_starter(&selection, Some(&old), Some(&old.id)),
6480 Starter::Resume(old.id.clone())
6481 );
6482 let mut changed = selection.clone();
6483 changed.head = "b".repeat(40);
6484 assert_eq!(
6485 selected_starter(&changed, Some(&old), Some(&old.id)),
6486 Starter::Review(changed.branch)
6487 );
6488 }
6489
6490 #[test]
6491 fn candidate_transplant_failure_is_one_actionable_hold() {
6492 let dir = tempfile::tempdir().unwrap();
6493 let q = Queue::at(dir.path().join("queue"));
6494 let mut t = task();
6495 let mut state = run_state(RunStatus::Blocked);
6496 state.event("prep", "cannot apply referenced commit: conflict");
6497 state.prep_failure = Some("conflict".into());
6498 t.start(state.id.clone());
6499 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6500 assert_eq!(t.attempts, 1);
6501 assert_eq!(t.status, TaskStatus::Held);
6502 assert!(t.operator_held());
6503 assert!(!t.status.runnable());
6504 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6505 let first = t.chat_notice_key().unwrap();
6506 t.chat_notices.push(first.clone());
6507 t.release_operator("test", None);
6508 state.id = "another-unprepared-run".into();
6509 t.start(state.id.clone());
6510 finish_attempt(
6511 3,
6512 &q,
6513 &mut t,
6514 &state,
6515 &[],
6516 Err(anyhow::anyhow!("still conflicts")),
6517 );
6518 assert_eq!(
6519 t.chat_notice_key().as_ref(),
6520 Some(&first),
6521 "no work happened"
6522 );
6523 t.release_operator("test", None);
6524 state.prep_failure = None;
6525 state.status = RunStatus::Failed;
6526 state.begin_pass();
6527 t.start(state.id.clone());
6528 finish_attempt(
6529 1,
6530 &q,
6531 &mut t,
6532 &state,
6533 &[],
6534 Err(anyhow::anyhow!("new gate failure")),
6535 );
6536 let failure = t.chat_notice_key().unwrap();
6537 assert_ne!(failure, first, "approved recovery reached a real failure");
6538 t.release_operator("test", None);
6539 state.begin_pass();
6540 t.start(state.id.clone());
6541 finish_attempt(
6542 1,
6543 &q,
6544 &mut t,
6545 &state,
6546 &[],
6547 Err(anyhow::anyhow!("new failure on resume")),
6548 );
6549 assert_ne!(
6550 t.chat_notice_key().as_ref(),
6551 Some(&failure),
6552 "a new settled pass"
6553 );
6554 }
6555
6556 #[test]
6557 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6558 for source in ["cli", "web"] {
6559 let mut t = task();
6560 let mut run = run_state(RunStatus::Blocked);
6561 run.instruction = "Run A original criteria".into();
6562 run.task_instruction = Some(run.instruction.clone());
6563 t.start(run.id.clone());
6564 t.release_operator("choice", Some("resume-question"));
6565 t.resume_override = Some(crate::queue::OperatorResume {
6566 question_id: "resume-question".into(),
6567 at: Timestamp::now(),
6568 conductor_rehold: None,
6569 forced: true,
6570 pinned_run: Some(run.id.clone()),
6571 });
6572 t.hold_machine(Some("not enough free space to start a run".into()));
6573 let mut unchanged = t.clone();
6574 unchanged
6575 .edit("title changed".into(), unchanged.instruction.clone())
6576 .unwrap();
6577 unchanged.release_operator(source, None);
6578 assert_eq!(
6579 unchanged
6580 .resume_override
6581 .as_ref()
6582 .unwrap()
6583 .pinned_run
6584 .as_deref(),
6585 Some(run.id.as_str())
6586 );
6587 assert!(!compatible_run_instruction(&run, &unchanged));
6588 assert!(may_resume_run(&run, &unchanged));
6589
6590 t.edit(
6591 "approved recovery".into(),
6592 "Reuse measured PR head B; fix Clippy; review".into(),
6593 )
6594 .unwrap();
6595 assert!(t.resume_override.is_none());
6596 t.release_operator(source, None);
6597 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6598 assert!(reloaded.resume_override.is_none());
6599 assert!(!may_resume_run(&run, &reloaded));
6600 let unfinished =
6601 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6602 .filter(|_| may_resume_run(&run, &reloaded));
6603 assert_eq!(
6604 choose_starter(None, false, unfinished.as_deref()),
6605 Starter::Start
6606 );
6607 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6608 assert_eq!(
6609 reloaded.resumption_history[0].question_id.as_deref(),
6610 Some("resume-question")
6611 );
6612
6613 unchanged.instruction = "Reuse measured PR head B".into();
6616 unchanged.hold_machine(Some("disk hold".into()));
6617 unchanged.release_operator(source, None);
6618 assert!(unchanged.resume_override.is_none());
6619 assert!(!may_resume_run(&run, &unchanged));
6620 }
6621 }
6622
6623 #[test]
6624 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6625 let mut t = task();
6626 let mut run = run_state(RunStatus::Blocked);
6627 t.review_of = Some("approved-head".into());
6628 t.instruction =
6629 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6630 .into();
6631 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6632 run.reviewed_commits = Some(vec!["Commit log".into()]);
6633 t.start(run.id.clone());
6634 t.hold_machine(Some("interrupted review".into()));
6635 t.release_operator("cli", None);
6636 assert!(
6637 compatible_run_instruction(&run, &t),
6638 "legacy explicit review remains resumable"
6639 );
6640 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6641 .filter(|_| compatible_run_instruction(&run, &t));
6642 assert_eq!(
6643 choose_starter(None, false, unfinished.as_deref()),
6644 Starter::Resume(run.id.clone())
6645 );
6646 run.task_instruction = Some(t.instruction.clone());
6647 assert!(compatible_run_instruction(&run, &t));
6648 let reloaded: RunState =
6649 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6650 assert!(compatible_run_instruction(&reloaded, &t));
6651 t.edit(
6652 "new approved head".into(),
6653 "Reuse measured PR head B".into(),
6654 )
6655 .unwrap();
6656 assert!(!compatible_run_instruction(&run, &t));
6657
6658 t.review_of = None;
6660 t.instruction = "Fix Clippy on approved head".into();
6661 run.task_instruction = None;
6662 run.instruction = format!(
6663 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6664 t.instruction
6665 );
6666 assert!(compatible_run_instruction(&run, &t));
6667 t.instruction.push('\n');
6668 assert!(
6669 compatible_run_instruction(&run, &t),
6670 "legacy generated review trimmed the original criteria"
6671 );
6672 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6673 let mut unchanged = t.clone();
6674 assert_eq!(
6675 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6676 Some("approved-head")
6677 );
6678 t.edit("new plan".into(), "Reuse measured PR head B".into())
6679 .unwrap();
6680 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6681 assert!(!compatible_run_instruction(&run, &t));
6682 t.review_branch = Some("approved-head".into());
6683 t.review_instruction = None;
6684 assert!(
6685 take_review_branch(&mut t, Some(&run)).is_none(),
6686 "legacy retained candidate cannot replace a revised approved head"
6687 );
6688 }
6689
6690 #[test]
6691 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6692 crate::run::pin_test_home();
6693 let dir = tempfile::tempdir().unwrap();
6694 let queue = Queue::at(dir.path().join("queue"));
6695 for resume in [true, false] {
6696 let state = run_state(RunStatus::Blocked);
6697 let mut t = task();
6698 t.start(state.id.clone());
6699 let action = if resume {
6700 ask::ChoiceAction::Resume {
6701 run: state.id.clone(),
6702 }
6703 } else {
6704 ask::ChoiceAction::Requeue
6705 };
6706 let mut q = action_question(&state.id, action);
6707 q.status = ask::QuestionStatus::Open;
6708 q.answer = None;
6709 q.answered_at = None;
6710 t.fail("run stopped with a question", 3);
6711 hold_if_runnable(&queue, &mut t);
6712 assert!(t.operator_decided_at.is_none());
6713 assert!(t.operator_held());
6714 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6715 let expected = if resume {
6716 ActionDecision::Resume(state.id.clone())
6717 } else {
6718 ActionDecision::Requeue
6719 };
6720 assert_eq!(
6721 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6722 expected
6723 );
6724 t.hold_manual(Some("later explicit hold".into()));
6725 assert_eq!(
6726 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6727 ActionDecision::Stale
6728 );
6729 }
6730 }
6731
6732 #[test]
6733 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6734 let mut t = task();
6735 t.answers.push(crate::queue::AnsweredQuestion {
6736 question: "Inspect the remote?".into(),
6737 answer: "Inspect then wait".into(),
6738 });
6739 let old = instruction_for(&t);
6740 assert!(compatible_instruction(&old, &t));
6741 t.instruction = "Reuse approved PR head and review".into();
6742 t.release_operator("cli", None);
6743 assert!(!compatible_instruction(&old, &t));
6744 let approved = instruction_for(&t);
6745 assert!(approved.contains("waiting premises are superseded"));
6746 assert!(compatible_instruction(&approved, &t));
6747 assert_eq!(resumed_instruction(&approved, &t), approved);
6748 t.release_operator("web", None);
6749 assert!(compatible_instruction(&approved, &t));
6750 assert_eq!(t.resumption_history.len(), 1);
6751 t.start("run".into());
6752 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6753 }
6754
6755 #[test]
6759 fn unrelated_ci_refunds_once_per_pass_and_holds_for_the_owner() {
6760 let mut state = RunState::new(
6761 PathBuf::from("/repo"),
6762 "main".into(),
6763 "base".into(),
6764 "task".into(),
6765 Config::default(),
6766 );
6767 state.status = RunStatus::Landing;
6768 state.parked = true;
6769 state.pr_pushed_head = Some("head".into());
6770 state.land_unrelated = Some(
6771 serde_json::from_value(serde_json::json!({
6772 "head": "head", "question": "ci-question",
6773 "evidence": {"head": "head", "base": "base", "changed": [], "checks": []}
6774 }))
6775 .unwrap(),
6776 );
6777 let mut t = task();
6778 t.start(state.id.clone());
6779 let verdict = Verdict {
6780 status: RunStatus::Landing,
6781 left_pr: true,
6782 quota_hit: false,
6783 parked: true,
6784 no_viable_candidates: false,
6785 };
6786 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6787 assert_eq!(t.attempts, 0);
6788 assert_eq!(t.status, TaskStatus::Held);
6789 assert!(!t.operator_held());
6790 assert!(
6791 t.hold_reason
6792 .as_deref()
6793 .unwrap()
6794 .starts_with("[ci-unrelated]")
6795 );
6796 t.attempts = 1;
6798 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6799 assert_eq!(t.attempts, 1);
6800 assert_eq!(t.passes.len(), 1);
6801 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
6802 state.pr_pushed_head = Some("new-head".into());
6803 state.parked = false;
6804 state.status = RunStatus::Blocked;
6805 let stale = Verdict {
6806 status: RunStatus::Blocked,
6807 parked: false,
6808 ..verdict
6809 };
6810 settle_and_diagnose(&mut t, stale, "stale CI hold", 2, &state);
6811 assert_eq!(t.attempts, 1);
6812 state.begin_pass();
6814 state.begin_pass();
6815 state.status = RunStatus::Landing;
6816 state.parked = true;
6817 state.pr_pushed_head = Some("head".into());
6818 settle_and_diagnose(&mut t, verdict, "CI hold on resume", 2, &state);
6819 assert_eq!(t.attempts, 0);
6820 assert_eq!(t.passes.len(), 2);
6821 }
6822
6823 #[test]
6824 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
6825 let mut state = RunState::new(
6826 PathBuf::from("/repo"),
6827 "main".to_owned(),
6828 "abc1234".to_owned(),
6829 "task".to_owned(),
6830 Config::default(),
6831 );
6832 state.status = RunStatus::Blocked;
6833 state.resource_block = Some(crate::run::ResourceBlock {
6834 cause: crate::run::ResourceBlockCause::Other,
6835 node: "graft".to_owned(),
6836 kind: "shared build cache".to_owned(),
6837 detail: "timed out".to_owned(),
6838 at: jiff::Timestamp::now(),
6839 });
6840 let mut t = task();
6841 t.start("20260902-000000-aaaa".to_owned());
6842 assert_eq!(t.attempts, 1);
6843 let verdict = Verdict {
6844 status: RunStatus::Blocked,
6845 left_pr: false,
6846 quota_hit: false,
6847 parked: false,
6848 no_viable_candidates: false,
6849 };
6850 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
6851 assert_eq!(t.status, TaskStatus::Held);
6852 assert_eq!(t.attempts, 0, "the attempt is refunded");
6853 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
6854 assert!(
6855 !t.operator_held(),
6856 "a machine hold, released without a human"
6857 );
6858
6859 let mut selected = task();
6860 selected.start("20260902-000000-cccc".into());
6861 selected.replacement = Some(crate::merge_decision::Replacement {
6862 question: "fixture".into(),
6863 run: "earlier".into(),
6864 pr: "https://example.invalid/pr/1".into(),
6865 agent: "selected".into(),
6866 context: "fixture".into(),
6867 });
6868 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
6869 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
6870 assert_eq!(selected.attempts, 0);
6871 assert!(
6872 !selected.operator_held(),
6873 "a selected agent also retains a machine resource hold"
6874 );
6875 assert!(
6876 selected
6877 .hold_reason
6878 .as_deref()
6879 .unwrap()
6880 .starts_with("[resource] ")
6881 );
6882
6883 let mut p = task();
6885 p.start("20260902-000000-bbbb".to_owned());
6886 settle_and_diagnose(
6887 &mut p,
6888 Verdict {
6889 parked: true,
6890 ..verdict
6891 },
6892 "parked",
6893 2,
6894 &state,
6895 );
6896 assert_eq!(p.status, TaskStatus::Parked);
6897 assert_eq!(
6898 p.attempts, 0,
6899 "parking refunds once rather than also settling the resource hold"
6900 );
6901 }
6902
6903 #[test]
6904 fn record_failure_notice_links_the_task() {
6905 let dir = tempfile::tempdir().unwrap();
6906 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
6909 let queue = Queue::at(dir.path().join("queue"));
6910 let mut t = interrupt_task("rec-fail-1");
6911 record(&queue, &mut t);
6912
6913 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
6914 assert_eq!(found.len(), 1);
6915 assert_eq!(found[0].key, "loop:record");
6916 assert_eq!(
6917 found[0].link,
6918 Some(Link::Task {
6919 id: "rec-fail-1".to_owned()
6920 })
6921 );
6922 }
6923
6924 fn interrupt_task(id: &str) -> Task {
6927 let mut t = task();
6928 t.id = id.to_owned();
6929 t.interrupt = true;
6930 t
6931 }
6932
6933 fn task_with_id(id: &str) -> Task {
6935 let mut t = task();
6936 t.id = id.to_owned();
6937 t
6938 }
6939
6940 fn urgent_task(id: &str) -> Task {
6942 let mut t = task();
6943 t.id = id.to_owned();
6944 t.urgent = true;
6945 t
6946 }
6947
6948 #[test]
6949 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
6950 let dir = tempfile::tempdir().unwrap();
6951 let queue = Queue::at(dir.path().join("queue"));
6952 let mut parked = task_with_id("parked");
6953 parked.start("run".into());
6954 parked.park("upgrade");
6955 parked.record_pass_at(
6956 "run",
6957 0,
6958 Some(Timestamp::now()),
6959 1,
6960 Some(crate::run::ParkCause::Upgrade {
6961 from: "old".into(),
6962 to: None,
6963 }),
6964 );
6965 let mut fresh = task_with_id("fresh");
6966 fresh.priority = 100;
6967 let mut urgent_task = urgent_task("urgent");
6968 for t in [&mut parked, &mut fresh, &mut urgent_task] {
6969 queue.put(t).unwrap();
6970 }
6971 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6972 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6973 let candidates = runnable(&queue);
6974 assert_eq!(candidates[0].id, parked.id);
6975 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
6976 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
6977 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
6978 drop(first);
6979 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
6980 let next = runnable(&queue)
6981 .into_iter()
6982 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
6983 .unwrap();
6984 assert_eq!(next.task.id, fresh.id);
6985 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
6986 drop(claimed_elsewhere);
6987 drop(next);
6988 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
6989 }
6990
6991 #[test]
6992 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
6993 let dir = tempfile::tempdir().unwrap();
6994 let queue = Queue::at(dir.path().join("queue"));
6995 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
6996 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
6997 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
6998 let mut first = task();
6999 queue.put(&mut first).unwrap();
7000 let stale_plain = queue.get(&first.id).unwrap();
7001 first.set_urgent(true).unwrap();
7002 queue.put(&mut first).unwrap();
7003 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
7004 let mut second = urgent_task("second");
7005 queue.put(&mut second).unwrap();
7006 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
7007 drop(queue.claim(&second.id).unwrap());
7008 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
7009 drop(dispatched);
7010 let stale_urgent = queue.get(&first.id).unwrap();
7011 first.set_urgent(false).unwrap();
7012 queue.put(&mut first).unwrap();
7013 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
7014 assert_eq!(urgent.available_permits(), 1);
7015 drop(queue.claim(&first.id).unwrap());
7016 drop(occupied);
7017 }
7018
7019 #[test]
7020 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
7021 let dir = tempfile::tempdir().unwrap();
7022 let queue = Queue::at(dir.path().join("queue"));
7023 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7024 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7025 let mut t = task();
7026 queue.put(&mut t).unwrap();
7027 for enabled in [true, false] {
7028 let stale = queue.get(&t.id).unwrap();
7029 t.set_urgent(enabled).unwrap();
7030 queue.put(&mut t).unwrap();
7031 assert_ne!(stale.urgent, enabled);
7032 let DispatchClaim {
7033 task: fresh,
7034 claim,
7035 permit,
7036 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
7037 assert_eq!(fresh.urgent, enabled);
7038 assert_eq!(ordinary.available_permits(), usize::from(enabled));
7039 assert_eq!(urgent.available_permits(), usize::from(!enabled));
7040 assert!(queue.claim(&t.id).is_err());
7041 assert_eq!(fresh.attempts, stale.attempts);
7042 drop((claim, permit));
7043 assert_eq!(ordinary.available_permits(), 1);
7044 assert_eq!(urgent.available_permits(), 1);
7045 }
7046 t.set_urgent(true).unwrap();
7047 queue.put(&mut t).unwrap();
7048 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
7049 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7050 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
7051 drop(queue.claim(&t.id).unwrap());
7052 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
7053 drop(occupied);
7054 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
7055 drop(queue.claim(&t.id).unwrap());
7056 let held = queue.claim(&t.id).unwrap();
7057 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7058 drop(held);
7059 t.status = TaskStatus::Held;
7060 queue.put(&mut t).unwrap();
7061 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7062 drop(queue.claim(&t.id).unwrap());
7063 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
7064 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7065 drop(queue.claim(&t.id).unwrap());
7066 assert_eq!(urgent.available_permits(), 1);
7067 }
7068
7069 #[test]
7074 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
7075 assert_eq!(permit_kind(true, false), PermitKind::None);
7076 assert_eq!(permit_kind(true, true), PermitKind::None);
7077 }
7078
7079 #[test]
7084 fn permit_kind_separates_urgent_from_ordinary() {
7085 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
7086 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
7087 }
7088
7089 #[test]
7096 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
7097 let cfg = Config::default();
7098 let repo = Path::new("/any/repo/path");
7099
7100 let reason =
7101 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
7102 assert!(reason.contains("1024"), "{reason}");
7103 assert!(
7104 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7105 "{reason}"
7106 );
7107
7108 assert_eq!(
7109 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
7110 None,
7111 "exactly at the floor is open"
7112 );
7113 assert_eq!(
7114 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
7115 None,
7116 "comfortably above the floor is open"
7117 );
7118 }
7119
7120 #[test]
7121 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
7122 let mut cfg = Config::default();
7123 cfg.disk.min_free_bytes = 0;
7124 let repo = Path::new("/any/repo/path");
7125 assert_eq!(
7126 disk_gate_with(repo, &cfg, |_| Ok(0)),
7127 None,
7128 "a zero floor never measures at all"
7129 );
7130 }
7131
7132 #[test]
7133 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7134 let cfg = Config::default();
7135 let repo = Path::new("/any/repo/path");
7136 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7137 .expect("a measurement failure must close the gate, not open it");
7138 assert!(reason.contains("could not measure"), "{reason}");
7139 }
7140
7141 #[test]
7142 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7143 let ordinary = task();
7144 let next = advance_interrupt(
7145 Interrupt::Idle,
7146 std::slice::from_ref(&ordinary.id),
7147 std::slice::from_ref(&ordinary),
7148 );
7149 assert_eq!(next, Interrupt::Idle);
7150 }
7151
7152 #[test]
7153 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7154 let marked = interrupt_task("marked");
7157 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7158 assert_eq!(next, Interrupt::Idle);
7159 }
7160
7161 #[test]
7162 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7163 let marked = interrupt_task("marked");
7164 let next = advance_interrupt(
7165 Interrupt::Idle,
7166 &["running".to_owned()],
7167 std::slice::from_ref(&marked),
7168 );
7169 assert_eq!(
7170 next,
7171 Interrupt::Parking {
7172 parked: vec!["running".to_owned()],
7173 interrupt_task: "marked".to_owned(),
7174 }
7175 );
7176 }
7177
7178 #[test]
7187 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7188 let marked = interrupt_task("marked");
7189
7190 let two = advance_interrupt(
7191 Interrupt::Idle,
7192 &["a".to_owned(), "b".to_owned()],
7193 std::slice::from_ref(&marked),
7194 );
7195 assert_eq!(two, Interrupt::Idle);
7196
7197 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7198 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7199 }
7200
7201 #[test]
7202 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7203 let state = Interrupt::Parking {
7204 parked: vec!["running".to_owned()],
7205 interrupt_task: "marked".to_owned(),
7206 };
7207 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7209 assert_eq!(still_going, state);
7210
7211 let stopped_but_not_yet_dispatched =
7215 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7216 assert_eq!(stopped_but_not_yet_dispatched, state);
7217
7218 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7220 assert_eq!(
7221 dispatched,
7222 Interrupt::Running {
7223 parked: vec!["running".to_owned()],
7224 interrupt_task: "marked".to_owned(),
7225 }
7226 );
7227 }
7228
7229 #[test]
7230 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7231 let state = Interrupt::Running {
7232 parked: vec!["running".to_owned()],
7233 interrupt_task: "marked".to_owned(),
7234 };
7235 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7236 assert_eq!(still_running, state);
7237
7238 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7245 assert_eq!(
7246 ended,
7247 Interrupt::Resuming {
7248 parked: vec!["running".to_owned()]
7249 }
7250 );
7251 }
7252
7253 #[test]
7254 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7255 let state = Interrupt::Resuming {
7256 parked: vec!["running".to_owned()],
7257 };
7258 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7259 assert_eq!(still_waiting, state);
7260
7261 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7262 assert_eq!(dispatched, Interrupt::Idle);
7263 }
7264
7265 #[test]
7271 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7272 {
7273 let state = Interrupt::Parking {
7274 parked: vec!["running".to_owned()],
7275 interrupt_task: "marked".to_owned(),
7276 };
7277 let next = advance_interrupt(state, &[], &[]);
7280 assert_eq!(
7281 next,
7282 Interrupt::Resuming {
7283 parked: vec!["running".to_owned()]
7284 },
7285 "abandoning the interrupt must not abandon the resume it owes"
7286 );
7287 }
7288
7289 #[test]
7292 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7293 let state = Interrupt::Resuming {
7294 parked: vec!["running".to_owned()],
7295 };
7296 let next = advance_interrupt(state, &[], &[]);
7297 assert_eq!(
7298 next,
7299 Interrupt::Idle,
7300 "nothing is left to wait for; the loop must not stay wedged"
7301 );
7302 }
7303
7304 #[test]
7305 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7306 let marked = interrupt_task("marked");
7307 let next = advance_interrupt_tick(
7308 false,
7309 Interrupt::Idle,
7310 &["running".to_owned()],
7311 std::slice::from_ref(&marked),
7312 );
7313 assert_eq!(
7314 next,
7315 Interrupt::Idle,
7316 "an unmarked, unconfigured daemon must behave exactly as before"
7317 );
7318 }
7319
7320 #[test]
7321 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7322 let state = Interrupt::Parking {
7323 parked: vec!["running".to_owned()],
7324 interrupt_task: "marked".to_owned(),
7325 };
7326 let candidates = vec![interrupt_task("marked"), task()];
7327 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7328 assert!(
7329 allowed.is_empty(),
7330 "nothing may dispatch - not even the interrupt task itself - \
7331 until the parked run has actually stopped"
7332 );
7333 }
7334
7335 #[test]
7349 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7350 for state in [
7351 Interrupt::Parking {
7352 parked: vec!["running".to_owned()],
7353 interrupt_task: "marked".to_owned(),
7354 },
7355 Interrupt::Running {
7356 parked: vec!["running".to_owned()],
7357 interrupt_task: "marked".to_owned(),
7358 },
7359 Interrupt::Resuming {
7360 parked: vec!["running".to_owned()],
7361 },
7362 ] {
7363 let candidates = vec![
7364 interrupt_task("marked"),
7365 urgent_task("hot"),
7366 task_with_id("ordinary"),
7367 ];
7368 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7369 assert!(
7370 !allowed.iter().any(|t| t.id == "hot"),
7371 "an urgent candidate must wait out the same gate as anything \
7372 else while the run it would run alongside has not actually \
7373 left flight, for state {state:?}: {allowed:?}"
7374 );
7375 }
7376 }
7377
7378 #[test]
7384 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7385 let state = Interrupt::Resuming {
7386 parked: vec!["hot".to_owned()],
7387 };
7388 let candidates = vec![urgent_task("hot"), task()];
7389 let allowed = interrupt_gate(&state, &[], candidates);
7390 assert_eq!(
7391 allowed.iter().filter(|t| t.id == "hot").count(),
7392 1,
7393 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7394 );
7395 }
7396
7397 #[test]
7398 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7399 let state = Interrupt::Parking {
7400 parked: vec!["running".to_owned()],
7401 interrupt_task: "marked".to_owned(),
7402 };
7403 let other = task();
7404 let candidates = vec![interrupt_task("marked"), other.clone()];
7405 let allowed = interrupt_gate(&state, &[], candidates);
7406 assert_eq!(allowed.len(), 1);
7407 assert_eq!(allowed[0].id, "marked");
7408 }
7409
7410 #[test]
7411 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7412 let state = Interrupt::Running {
7413 parked: vec!["running".to_owned()],
7414 interrupt_task: "marked".to_owned(),
7415 };
7416 let candidates = vec![task(), task()];
7417 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7418 assert!(allowed.is_empty());
7419 }
7420
7421 #[test]
7428 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7429 let state = Interrupt::Resuming {
7430 parked: vec!["a".to_owned(), "c".to_owned()],
7431 };
7432 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7433 let allowed = interrupt_gate(&state, &[], candidates);
7434 assert_eq!(
7435 allowed.len(),
7436 1,
7437 "at most one candidate may be offered while resuming: {allowed:?}"
7438 );
7439 assert_eq!(allowed[0].id, "a");
7440 }
7441
7442 #[test]
7443 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7444 let state = Interrupt::Resuming {
7445 parked: vec!["a".to_owned()],
7446 };
7447 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7448 assert!(allowed.is_empty());
7449 }
7450
7451 #[test]
7457 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7458 let running = task(); let marked = interrupt_task("marked");
7460
7461 let mut state = Interrupt::Idle;
7462 let in_flight = vec![running.id.clone()];
7464 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7465 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7466 assert!(gated.is_empty(), "still waiting on `running` to park");
7467
7468 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7470 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7471 assert_eq!(
7472 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7473 vec!["marked"],
7474 "only the interrupt task may be offered to the dispatcher now"
7475 );
7476
7477 state = advance_interrupt_tick(
7479 true,
7480 state,
7481 &["marked".to_owned()],
7482 std::slice::from_ref(&running),
7483 );
7484 let gated = interrupt_gate(
7485 &state,
7486 &["marked".to_owned()],
7487 vec![marked.clone(), running.clone()],
7488 );
7489 assert!(
7490 gated.is_empty(),
7491 "the parked run must not be offered back while the interrupt \
7492 task is still running"
7493 );
7494
7495 let other = task_with_id("other");
7499 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7500 assert_eq!(
7501 state,
7502 Interrupt::Resuming {
7503 parked: vec![running.id.clone()]
7504 }
7505 );
7506 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7507 assert_eq!(
7508 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7509 vec![running.id.as_str()],
7510 "exactly the parked run resumes - not the unrelated task, even \
7511 though it was offered first"
7512 );
7513
7514 state = advance_interrupt_tick(
7518 true,
7519 state,
7520 std::slice::from_ref(&running.id),
7521 std::slice::from_ref(&other),
7522 );
7523 assert_eq!(state, Interrupt::Idle);
7524 let gated = interrupt_gate(
7525 &state,
7526 std::slice::from_ref(&running.id),
7527 vec![other.clone()],
7528 );
7529 assert_eq!(
7530 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7531 vec![other.id.as_str()],
7532 "ordinary dispatch is unrestricted again"
7533 );
7534 }
7535
7536 #[test]
7537 fn every_run_status_settles_the_task_it_came_from() {
7538 let table = [
7540 (RunStatus::Merged, TaskStatus::Done, 1),
7541 (RunStatus::Ready, TaskStatus::Done, 1),
7542 (RunStatus::Stalled, TaskStatus::Failed, 0),
7543 (RunStatus::Blocked, TaskStatus::Failed, 1),
7544 (RunStatus::Failed, TaskStatus::Failed, 1),
7545 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7546 (RunStatus::Prep, TaskStatus::Failed, 1),
7547 (RunStatus::Implementing, TaskStatus::Failed, 1),
7548 (RunStatus::Judging, TaskStatus::Failed, 1),
7549 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7550 (RunStatus::Voting, TaskStatus::Failed, 1),
7551 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7552 (RunStatus::Gating, TaskStatus::Failed, 1),
7553 ];
7554 for (run, want, attempts) in table {
7555 let mut t = task();
7556 t.start("20260902-000000-aaaa".to_owned());
7557 settle(
7558 &mut t,
7559 Verdict {
7560 status: run,
7561 left_pr: false,
7562 parked: false,
7563 quota_hit: matches!(run, RunStatus::Stalled),
7564 no_viable_candidates: false,
7565 },
7566 "why",
7567 2,
7568 );
7569 assert_eq!(t.status, want, "task status after {}", label(run));
7570 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7571 }
7572 }
7573
7574 #[test]
7575 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7576 let mut stalled = task();
7577 stalled.start("20260902-000000-aaaa".to_owned());
7578 settle(
7579 &mut stalled,
7580 Verdict {
7581 status: RunStatus::Stalled,
7582 left_pr: false,
7583 parked: false,
7584 quota_hit: true,
7585 no_viable_candidates: false,
7586 },
7587 "quota",
7588 1,
7589 );
7590 assert_eq!(stalled.attempts, 0);
7591 assert!(
7592 stalled.status.runnable(),
7593 "a machine problem must leave the task in line"
7594 );
7595
7596 let mut blocked = task();
7597 blocked.start("20260902-000000-aaaa".to_owned());
7598 settle(
7599 &mut blocked,
7600 Verdict {
7601 status: RunStatus::Blocked,
7602 left_pr: false,
7603 parked: false,
7604 quota_hit: false,
7605 no_viable_candidates: false,
7606 },
7607 "findings open",
7608 1,
7609 );
7610 assert_eq!(blocked.attempts, 1);
7611 assert_eq!(
7612 blocked.status,
7613 TaskStatus::Held,
7614 "the last attempt hands the task to a human"
7615 );
7616 }
7617
7618 #[test]
7619 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7620 let mut delivered = task();
7623 delivered.start("20260903-080619-01c2".to_owned());
7624 settle(
7625 &mut delivered,
7626 Verdict {
7627 status: RunStatus::Blocked,
7628 left_pr: true,
7629 parked: false,
7630 quota_hit: false,
7631 no_viable_candidates: false,
7632 },
7633 "no check status",
7634 4,
7635 );
7636 assert_eq!(
7637 delivered.status,
7638 TaskStatus::Held,
7639 "a pull request waiting on CI or a person is not a retryable failure"
7640 );
7641 assert!(
7642 !delivered.status.runnable(),
7643 "the loop must not pick this task up again"
7644 );
7645 assert_eq!(
7646 delivered.last_error.as_deref(),
7647 Some("no check status"),
7648 "the operator needs to be told what the gate was waiting for"
7649 );
7650
7651 let mut empty_handed = task();
7654 empty_handed.start("20260903-080619-01c2".to_owned());
7655 settle(
7656 &mut empty_handed,
7657 Verdict {
7658 status: RunStatus::Blocked,
7659 left_pr: false,
7660 parked: false,
7661 quota_hit: false,
7662 no_viable_candidates: false,
7663 },
7664 "findings open",
7665 4,
7666 );
7667 assert_eq!(empty_handed.status, TaskStatus::Failed);
7668 assert!(empty_handed.status.runnable());
7669 }
7670
7671 #[test]
7672 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7673 let mut noop = task();
7679 noop.start("20260912-131304-391f".to_owned());
7680 settle(
7681 &mut noop,
7682 Verdict {
7683 status: RunStatus::VerifiedNoop,
7684 left_pr: false,
7685 parked: false,
7686 quota_hit: false,
7687 no_viable_candidates: true,
7688 },
7689 "candidate A: already fixed by b32cfc4, on main",
7690 4,
7691 );
7692 assert_eq!(
7693 noop.status,
7694 TaskStatus::Held,
7695 "an unverified claim is a request for a human, not a failure"
7696 );
7697 assert!(
7698 !noop.status.runnable(),
7699 "the loop must not requeue this on the same unverified claim"
7700 );
7701 assert_eq!(noop.attempts, 1);
7706 }
7707
7708 #[test]
7709 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7710 let mut parked = task();
7715 parked.start("20260903-183634-2d98".to_owned());
7716 settle(
7717 &mut parked,
7718 Verdict {
7719 status: RunStatus::Implementing,
7720 left_pr: false,
7721 quota_hit: false,
7722 parked: true,
7723 no_viable_candidates: false,
7724 },
7725 "parked after `implementing`",
7726 2,
7727 );
7728 assert_eq!(parked.attempts, 0, "a park is refunded");
7729 assert!(
7730 parked.status.runnable(),
7731 "and the task stays in line so the next loop resumes its run"
7732 );
7733 assert_eq!(parked.status, TaskStatus::Parked);
7734 assert_eq!(
7735 parked.park_reason.as_deref(),
7736 Some("parked after `implementing`"),
7737 "the card says where it stopped"
7738 );
7739 assert_eq!(parked.last_error, None, "nothing failed");
7740
7741 let mut broken = task();
7745 broken.start("20260903-183634-2d98".to_owned());
7746 settle(
7747 &mut broken,
7748 Verdict {
7749 status: RunStatus::Implementing,
7750 left_pr: false,
7751 quota_hit: false,
7752 parked: false,
7753 no_viable_candidates: false,
7754 },
7755 "returned mid-flight",
7756 2,
7757 );
7758 assert_eq!(broken.attempts, 1);
7759 }
7760
7761 #[test]
7762 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7763 let mut flaky = task();
7768 flaky.start("20260903-123023-e633".to_owned());
7769 settle(
7770 &mut flaky,
7771 Verdict {
7772 status: RunStatus::Stalled,
7773 left_pr: false,
7774 parked: false,
7775 quota_hit: false,
7776 no_viable_candidates: false,
7777 },
7778 "verdict rests on 1 of 3 judges",
7779 2,
7780 );
7781 assert_eq!(
7782 flaky.attempts, 1,
7783 "flakiness spends an attempt, so `max_attempts` still bounds it"
7784 );
7785 assert!(flaky.status.runnable(), "and it is still worth retrying");
7786
7787 let mut limited = task();
7789 limited.start("20260903-123023-e633".to_owned());
7790 settle(
7791 &mut limited,
7792 Verdict {
7793 status: RunStatus::Stalled,
7794 left_pr: false,
7795 parked: false,
7796 quota_hit: true,
7797 no_viable_candidates: false,
7798 },
7799 "judge-2, judge-3 out of quota",
7800 2,
7801 );
7802 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7803 assert!(limited.status.runnable());
7804
7805 let mut worn = task();
7808 for _ in 0..2 {
7809 worn.release();
7810 }
7811 worn.start("20260903-123023-e633".to_owned());
7812 worn.attempts = 2;
7813 settle(
7814 &mut worn,
7815 Verdict {
7816 status: RunStatus::Stalled,
7817 left_pr: false,
7818 parked: false,
7819 quota_hit: false,
7820 no_viable_candidates: false,
7821 },
7822 "no quorum again",
7823 2,
7824 );
7825 assert_eq!(worn.status, TaskStatus::Held);
7826 assert!(!worn.status.runnable());
7827 }
7828
7829 #[test]
7830 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
7831 let mut wiped_out = task();
7838 wiped_out.start("20260907-025000-a1b2".to_owned());
7839 settle(
7840 &mut wiped_out,
7841 Verdict {
7842 status: RunStatus::Failed,
7843 left_pr: false,
7844 parked: false,
7845 quota_hit: true,
7846 no_viable_candidates: true,
7847 },
7848 "no candidate produced a change; nothing to judge",
7849 2,
7850 );
7851 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
7852 assert!(
7853 wiped_out.status.runnable(),
7854 "a machine problem must leave the task in line"
7855 );
7856
7857 let mut partial_progress = task();
7863 partial_progress.start("20260907-025500-c3d4".to_owned());
7864 settle(
7865 &mut partial_progress,
7866 Verdict {
7867 status: RunStatus::Failed,
7868 left_pr: false,
7869 parked: false,
7870 quota_hit: true,
7871 no_viable_candidates: false,
7872 },
7873 "gate failed on the winning candidate",
7874 2,
7875 );
7876 assert_eq!(
7877 partial_progress.attempts, 1,
7878 "a candidate that actually produced a change spends the attempt \
7879 even though some other seat hit its quota"
7880 );
7881 assert!(partial_progress.status.runnable());
7882 }
7883
7884 #[test]
7885 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
7886 let mut t = task();
7892 t.start("20260907-025000-a1b2".to_owned());
7893 let mut state = run_state(RunStatus::Failed);
7894 state.quota.push(QuotaLoss {
7895 seat: "cand-a".to_owned(),
7896 node: "implement".to_owned(),
7897 at: Timestamp::now(),
7898 reset: None,
7899 });
7900 assert!(
7901 state.viable().is_empty(),
7902 "no candidate was added, so nothing is viable"
7903 );
7904 reclaim(&mut t, Some(state), 2, "en");
7905 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
7906 assert!(t.status.runnable());
7907 }
7908
7909 #[test]
7910 fn a_held_task_is_never_offered_to_the_loop() {
7911 let dir = tempfile::tempdir().unwrap();
7912 let queue = Queue::at(dir.path().to_path_buf());
7913 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
7914 let mut t = task();
7915 t.id = format!("2026090{n}-000000-000{n}");
7916 t.priority = priority;
7917 queue.put(&mut t).unwrap();
7918 }
7919 let mut held = task();
7920 held.id = "20260909-000000-9999".to_owned();
7921 held.priority = 99;
7922 held.hold_machine(None);
7923 queue.put(&mut held).unwrap();
7924
7925 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
7926 assert_eq!(order.len(), 3);
7927 assert!(!order.contains(&held.id));
7928 assert_eq!(
7929 order.first().cloned(),
7930 queue.next_runnable().map(|t| t.id),
7931 "the loop's first candidate is exactly what the queue offers"
7932 );
7933 assert_eq!(
7934 order,
7935 vec![
7936 "20260902-000000-0002".to_owned(),
7937 "20260903-000000-0003".to_owned(),
7938 "20260901-000000-0001".to_owned(),
7939 ],
7940 "priority first, then oldest, so nothing starves"
7941 );
7942 }
7943
7944 #[test]
7945 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
7946 let dir = tempfile::tempdir().unwrap();
7947 let queue = Queue::at(dir.path().to_path_buf());
7948 let mut old = task();
7949 old.id = "20260101-000000-old0".to_owned();
7950 queue.put(&mut old).unwrap();
7951 let mut fresh = task();
7952 fresh.id = "20260101-000000-new0".to_owned();
7953 queue.put(&mut fresh).unwrap();
7954
7955 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
7959 std::thread::sleep(Duration::from_millis(60));
7960 let live = queue.claim(&fresh.id).unwrap();
7961
7962 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7963 assert_eq!(swept, vec![old.id.clone()]);
7964 assert!(
7965 queue.claim(&old.id).is_ok(),
7966 "an unparseable lock older than the threshold is swept"
7967 );
7968 assert!(
7969 queue.claim(&fresh.id).is_err(),
7970 "a live pid protects its lock regardless of age"
7971 );
7972 drop(live);
7973 }
7974
7975 #[test]
7976 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
7977 let dir = tempfile::tempdir().unwrap();
7980 let queue = Queue::at(dir.path().to_path_buf());
7981 let mut t = task();
7982 t.id = "20260101-000000-live".to_owned();
7983 queue.put(&mut t).unwrap();
7984
7985 let claim = queue.claim(&t.id).unwrap();
7986 std::thread::sleep(Duration::from_millis(60));
7987
7988 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
7989 assert!(
7990 swept.is_empty(),
7991 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
7992 );
7993 assert!(
7994 queue.claim(&t.id).is_err(),
7995 "the lock still protects its task"
7996 );
7997 drop(claim);
7998 }
7999
8000 fn injected_dead_pid() -> u32 {
8003 std::process::id().checked_add(1).unwrap_or(1)
8004 }
8005
8006 #[test]
8007 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
8008 let dir = tempfile::tempdir().unwrap();
8009 let queue = Queue::at(dir.path().to_path_buf());
8010 let mut t = task();
8011 t.id = "20260101-000000-dead".to_owned();
8012 queue.put(&mut t).unwrap();
8013 let dead_pid = injected_dead_pid();
8014
8015 std::fs::write(
8020 dir.path().join(format!("{}.lock", t.id)),
8021 dead_pid.to_string(),
8022 )
8023 .unwrap();
8024
8025 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8026 pid != dead_pid
8027 });
8028 assert_eq!(
8029 swept,
8030 vec![t.id.clone()],
8031 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
8032 );
8033 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
8034 }
8035
8036 #[test]
8037 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
8038 let dir = tempfile::tempdir().unwrap();
8039 let queue = Queue::at(dir.path().to_path_buf());
8040 let mut t = task();
8041 t.id = "20260101-000000-late".to_owned();
8042 queue.put(&mut t).unwrap();
8043 let dead_pid = injected_dead_pid();
8044
8045 assert!(
8048 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
8049 "nothing has claimed the task yet"
8050 );
8051
8052 std::fs::write(
8055 dir.path().join(format!("{}.lock", t.id)),
8056 dead_pid.to_string(),
8057 )
8058 .unwrap();
8059
8060 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8064 pid != dead_pid
8065 });
8066 assert_eq!(swept, vec![t.id.clone()]);
8067 }
8068
8069 #[test]
8070 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
8071 crate::run::pin_test_home();
8076 let dir = tempfile::tempdir().unwrap();
8077 let queue = Queue::at(dir.path().to_path_buf());
8078 let mut t = task();
8079 t.id = "20260101-000000-crsh".to_owned();
8080 t.status = TaskStatus::Running;
8081 t.attempts = 1;
8082 t.runs.push("20260904-000000-4043".to_owned());
8086 queue.put(&mut t).unwrap();
8087 let dead_pid = injected_dead_pid();
8088
8089 std::fs::write(
8092 dir.path().join(format!("{}.lock", t.id)),
8093 dead_pid.to_string(),
8094 )
8095 .unwrap();
8096
8097 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
8103 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8104
8105 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8106 pid != dead_pid
8107 });
8108 assert_eq!(swept, vec![t.id.clone()]);
8109
8110 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8111 assert_eq!(reclaimed, vec![t.id.clone()]);
8112 let after = queue.get(&t.id).unwrap();
8113 assert_eq!(
8114 after.status,
8115 TaskStatus::Held,
8116 "no run.json to recover from, so a human is asked"
8117 );
8118 assert_eq!(
8119 after.runs,
8120 vec!["20260904-000000-4043".to_owned()],
8121 "the crashed run's id is kept as evidence, not discarded"
8122 );
8123 }
8124
8125 #[test]
8126 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
8127 let dir = tempfile::tempdir().unwrap();
8128 let queue = Queue::at(dir.path().to_path_buf());
8129 let mut t = task();
8130 t.id = "20260101-000000-unknown".to_owned();
8131 queue.put(&mut t).unwrap();
8132 let dead_pid = injected_dead_pid();
8133 std::fs::write(
8134 dir.path().join(format!("{}.lock", t.id)),
8135 dead_pid.to_string(),
8136 )
8137 .unwrap();
8138
8139 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8140 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8141 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8142 }
8143
8144 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8145 let mut s = run_state(status);
8146 s.config.graph.language = language.to_owned();
8147 s
8148 }
8149
8150 fn unstarted_verdict(status: RunStatus) -> Verdict {
8151 Verdict {
8152 status,
8153 left_pr: false,
8154 quota_hit: false,
8155 parked: false,
8156 no_viable_candidates: false,
8157 }
8158 }
8159
8160 #[test]
8161 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8162 let mut t = task();
8163 t.attempts = 1;
8164 settle_in(
8165 &mut t,
8166 unstarted_verdict(RunStatus::AlreadyInBase),
8167 "already in main",
8168 1,
8169 phrases("en"),
8170 );
8171 assert_eq!(t.status, TaskStatus::Done);
8172 assert_eq!(t.attempts, 0);
8173 }
8174
8175 #[test]
8176 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8177 let reason = |language: &str| {
8178 let mut t = task();
8179 settle_in(
8180 &mut t,
8181 unstarted_verdict(RunStatus::Judging),
8182 "boom",
8183 1,
8184 phrases(language),
8185 );
8186 t.last_error.or(t.hold_reason).unwrap_or_default()
8187 };
8188 assert!(
8189 reason("en").starts_with("the graph stopped at `"),
8190 "{}",
8191 reason("en")
8192 );
8193 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8194 assert!(reason("日本語").contains("boom"));
8195 assert_eq!(reason("fr"), reason("en"));
8196 }
8197
8198 #[test]
8199 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8200 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8201 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8202 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8203 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8204 assert!(ja.contains("[run "), "{ja}");
8205 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8206 assert!(ended.starts_with("run 終了: "), "{ended}");
8207 let mut de = run_state_in(RunStatus::Failed, "de");
8208 let mut en = run_state_in(RunStatus::Failed, "en");
8209 de.id = "same".to_owned();
8210 en.id = "same".to_owned();
8211 assert_eq!(describe(&de), describe(&en));
8212 }
8213
8214 #[test]
8215 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8216 use crate::handover::Refused;
8217 let cases = [
8218 Refused::Foreign {
8219 branch: "b".into(),
8220 path: "/w/x".into(),
8221 why: "made by hand".into(),
8222 },
8223 Refused::Unsafe {
8224 branch: "b".into(),
8225 path: "/w/x".into(),
8226 why: "its worktree has uncommitted changes (a.rs)".into(),
8227 },
8228 Refused::ReleaseFailed {
8229 branch: "b".into(),
8230 path: "/w/x".into(),
8231 run: "ab12".into(),
8232 },
8233 ];
8234 for r in &cases {
8235 let en = (phrases("en").handover_refused)(r);
8236 assert_eq!(en, r.to_string());
8237 let ja = (phrases("ja").handover_refused)(r);
8238 assert!(
8239 !ja.contains("is checked out") && !ja.contains("try again"),
8240 "{ja}"
8241 );
8242 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8243 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8244 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8245 }
8246 }
8247 assert!(phrases("en").handover_hint.contains("release the task"));
8248 assert!(phrases("ja").handover_hint.contains("解放"));
8249 }
8250
8251 #[test]
8252 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8253 let build = |lang: &str| {
8254 let mut s = run_state_in(RunStatus::Blocked, lang);
8255 s.id = "same".to_owned();
8256 s.config.graph.review_rounds = 3;
8257 let mut r = review_round(3);
8258 r.expected = 3;
8259 r.answered = 1;
8260 s.reviews.push(r);
8261 s.event(
8262 "review",
8263 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8264 );
8265 s
8266 };
8267 let en = describe(&build("en"));
8268 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8269 let ja = describe(&build("ja"));
8270 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8271 assert!(!ja.contains("never answered"), "{ja}");
8272 assert!(ja.contains("[run "), "{ja}");
8273
8274 let mut failed = build("ja");
8277 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8278 command: "cargo test".to_owned(),
8279 code: Some(1),
8280 output_tail: String::new(),
8281 duration_ms: 0,
8282 resource_blocked: false,
8283 });
8284 failed.event("review", "stopped; e2e failed: cargo test");
8285 let ja = describe(&failed);
8286 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8287 assert!(!ja.contains("席のレビュアー"), "{ja}");
8288 }
8289
8290 #[test]
8291 fn refusals_and_recovery_prose_follow_the_language() {
8292 let t = held_task_with("r1");
8293 let q = action_question(
8294 "r1",
8295 ask::ChoiceAction::Resume {
8296 run: "r1".to_owned(),
8297 },
8298 );
8299 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
8300 ActionDecision::Refuse(s) => s,
8301 other => panic!("{other:?}"),
8302 };
8303 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8304 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8305 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8306
8307 let mut held = task();
8308 reclaim(&mut held, None, 2, "ja");
8309 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8310 let mut held = task();
8311 reclaim(&mut held, None, 2, "xx");
8312 assert!(held.hold_reason.unwrap().contains("held for a human"));
8313 }
8314
8315 fn run_state(status: RunStatus) -> RunState {
8316 let mut state = RunState::new(
8317 PathBuf::from("/repo"),
8318 "main".to_owned(),
8319 "abc1234def".to_owned(),
8320 "add retries".to_owned(),
8321 Config::default(),
8322 );
8323 state.status = status;
8324 state
8325 }
8326
8327 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8328 Candidate {
8329 index: 0,
8330 label,
8331 agent: "claude".to_owned(),
8332 branch: format!("magi/x/{label}"),
8333 worktree: PathBuf::from("/repo"),
8334 summary: summary.to_owned(),
8335 stat: String::new(),
8336 files: 0,
8337 commits: usize::from(!empty),
8338 empty,
8339 failed: failed.map(str::to_owned),
8340 verified_noop: None,
8341 duration_ms: 0,
8342 folded: false,
8343 }
8344 }
8345
8346 #[test]
8347 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8348 let mut state = run_state(RunStatus::Blocked);
8349 state.gate = vec![
8350 CommandOutcome {
8351 command: "cargo make check".to_owned(),
8352 code: Some(0),
8353 output_tail: "ok".to_owned(),
8354 duration_ms: 0,
8355 resource_blocked: false,
8356 },
8357 CommandOutcome {
8358 command: "cargo test".to_owned(),
8359 code: Some(101),
8360 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8361 duration_ms: 0,
8362 resource_blocked: false,
8363 },
8364 ];
8365 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8366 assert!(d.contains("cargo test"), "{d}");
8367 assert!(
8368 !d.contains("cargo make check"),
8369 "a passing check is not a diagnostic: {d}"
8370 );
8371 assert!(d.contains("assertion failed"), "{d}");
8372 }
8373
8374 #[test]
8375 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8376 let mut state = run_state(RunStatus::Blocked);
8377 state.event(
8378 "land",
8379 "stopped: the fixer produced no commit while 2 check(s) were failing \
8380 (build, lint); stopping instead of looping on an unchanged tree",
8381 );
8382 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8383 assert!(d.contains("build"), "{d}");
8384 assert!(d.contains("lint"), "{d}");
8385 assert!(d.contains("fixer produced no commit"), "{d}");
8386 }
8387
8388 #[test]
8389 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8390 let state = run_state(RunStatus::VerifiedNoop);
8395 let d = describe(&state);
8396 assert!(
8397 d.contains("agent-verified no-op"),
8398 "expected the display label, not the wire spelling: {d}"
8399 );
8400 assert!(!d.contains("verified_noop"), "{d}");
8401 }
8402
8403 #[test]
8404 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8405 let mut state = run_state(RunStatus::Failed);
8411 state.candidates = vec![candidate(
8412 'A',
8413 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8414 true,
8415 None,
8416 )];
8417 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8418 assert!(d.contains("candidate A"), "{d}");
8419 assert!(d.contains("tagged v1.2.3"), "{d}");
8420 }
8421
8422 #[test]
8423 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8424 let mut state = run_state(RunStatus::Failed);
8425 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8426 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8427 assert!(d.contains("candidate A"), "{d}");
8428 assert!(d.contains("agent timed out"), "{d}");
8429 }
8430
8431 #[test]
8432 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8433 let mut state = run_state(RunStatus::Failed);
8436 state.candidates = vec![candidate('A', "did the work", false, None)];
8437 assert!(diagnostic(&state).is_none());
8438 }
8439
8440 #[test]
8441 fn diagnostic_is_bounded_however_much_a_run_printed() {
8442 let mut state = run_state(RunStatus::Blocked);
8443 state.gate = vec![
8444 CommandOutcome {
8445 command: "cargo test".to_owned(),
8446 code: Some(101),
8447 output_tail: "x".repeat(50_000),
8448 duration_ms: 0,
8449 resource_blocked: false,
8450 },
8451 CommandOutcome {
8452 command: "cargo clippy".to_owned(),
8453 code: Some(1),
8454 output_tail: "y".repeat(50_000),
8455 duration_ms: 0,
8456 resource_blocked: false,
8457 },
8458 ];
8459 state.candidates = vec![
8460 candidate('A', &"z".repeat(50_000), true, None),
8461 candidate('B', &"w".repeat(50_000), true, None),
8462 ];
8463 let d = diagnostic(&state).expect("plenty here to diagnose");
8464 assert!(
8465 d.len() <= DIAGNOSTIC_MAX,
8466 "diagnostic grew to {} bytes, unbounded",
8467 d.len()
8468 );
8469 }
8470
8471 #[test]
8472 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8473 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8474 let mut t = task();
8475 let mut state = run_state(status);
8476 t.pr_recovery = Some(crate::pr_recovery::Selection {
8477 url: "url".into(),
8478 branch: format!("magi/task-{}", t.id),
8479 head: "a".repeat(40),
8480 source_head: None,
8481 source_run: None,
8482 source_proof: None,
8483 disposition: Some(crate::pr_recovery::Disposition::Land),
8484 instruction: t.instruction.clone(),
8485 approved_at: Timestamp::now(),
8486 });
8487 state.pr_recovery = t.pr_recovery.clone();
8488 t.start(state.id.clone());
8489 settle_and_diagnose(
8490 &mut t,
8491 Verdict {
8492 status,
8493 left_pr: false,
8494 quota_hit: false,
8495 parked: false,
8496 no_viable_candidates: false,
8497 },
8498 "delivery stopped",
8499 5,
8500 &state,
8501 );
8502 assert_eq!(t.status, TaskStatus::Held);
8503 let reason = t.hold_reason.as_deref().unwrap();
8504 if status == RunStatus::Ready {
8505 assert!(reason.contains("Selected PR delivery is incomplete"));
8506 assert!(t.recovery_budget.is_none());
8507 } else {
8508 assert!(reason.contains("Selected recovery stopped"));
8509 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8510 }
8511 assert!(t.pr_recovery.is_some());
8512 }
8513 }
8514
8515 #[test]
8516 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8517 let mut state = run_state(RunStatus::Blocked);
8518 state.gate = vec![CommandOutcome {
8519 command: "cargo test".to_owned(),
8520 code: Some(101),
8521 output_tail: "assertion failed".to_owned(),
8522 duration_ms: 0,
8523 resource_blocked: false,
8524 }];
8525 let verdict = Verdict {
8526 status: RunStatus::Blocked,
8527 left_pr: false,
8528 quota_hit: false,
8529 parked: false,
8530 no_viable_candidates: false,
8531 };
8532
8533 let mut t = task();
8536 t.start("run-1".to_owned());
8537 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8538 assert_eq!(t.status, TaskStatus::Failed);
8539 assert!(t.diagnostic.is_none());
8540
8541 t.start("run-2".to_owned());
8544 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8545 assert_eq!(t.status, TaskStatus::Held);
8546 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8547 assert!(d.contains("cargo test"), "{d}");
8548 }
8549
8550 #[test]
8551 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8552 crate::run::pin_test_home();
8557 let home = crate::run::home();
8558 let state = run_state(RunStatus::VerifiedNoop);
8559 let mut q = ask::Question::new(
8560 state.id.clone(),
8561 "implement".to_owned(),
8562 "impl-A".to_owned(),
8563 "is this really a no-op?".to_owned(),
8564 String::new(),
8565 Vec::new(),
8566 );
8567 Questions::at(home.join("questions")).put(&mut q).unwrap();
8568
8569 let verdict = Verdict {
8570 status: RunStatus::VerifiedNoop,
8571 left_pr: false,
8572 quota_hit: false,
8573 parked: false,
8574 no_viable_candidates: false,
8575 };
8576 let mut t = task();
8577 t.start(state.id.clone());
8578 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8579
8580 assert_eq!(t.status, TaskStatus::Held);
8581 let reason = t.hold_reason.expect("a held task must record why");
8582 assert!(
8583 reason.starts_with("run ended agent-verified no-op"),
8584 "the original settle reason must survive unchanged: {reason}"
8585 );
8586 assert!(
8587 reason.contains(q.short()),
8588 "the open question's id must be named so the notice is actionable: {reason}"
8589 );
8590 }
8591
8592 #[test]
8593 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8594 crate::run::pin_test_home();
8595 let state = run_state(RunStatus::VerifiedNoop);
8596
8597 let verdict = Verdict {
8598 status: RunStatus::VerifiedNoop,
8599 left_pr: false,
8600 quota_hit: false,
8601 parked: false,
8602 no_viable_candidates: false,
8603 };
8604 let mut t = task();
8605 t.start(state.id.clone());
8606 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8607
8608 assert_eq!(t.status, TaskStatus::Held);
8609 assert_eq!(
8610 t.hold_reason.as_deref(),
8611 Some("run ended agent-verified no-op"),
8612 "nothing to append when the question was already answered or never asked"
8613 );
8614 }
8615
8616 #[test]
8617 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8618 crate::run::pin_test_home();
8619 let mut first = run_state(RunStatus::Blocked);
8620 first.id = "20260101-000000-sup1".to_owned();
8621 first.save().unwrap();
8622 let mut second = run_state(RunStatus::Merged);
8623 second.id = "20260101-000000-sup2".to_owned();
8624 second.save().unwrap();
8625
8626 let mut t = task();
8627 t.runs = vec![first.id.clone(), second.id.clone()];
8628 t.status = TaskStatus::Done;
8629
8630 supersede_prior_runs(&t, &crate::run::home());
8631
8632 assert_eq!(
8633 RunState::load(&first.id).unwrap().status,
8634 RunStatus::Superseded,
8635 "the first attempt's Blocked no longer needs anyone's attention"
8636 );
8637 assert_eq!(
8638 RunState::load(&second.id).unwrap().status,
8639 RunStatus::Merged,
8640 "the run that actually succeeded is left exactly as it was"
8641 );
8642 }
8643
8644 #[test]
8645 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8646 crate::run::pin_test_home();
8652 let mut first = run_state(RunStatus::Blocked);
8653 first.id = "20260101-000000-sup9".to_owned();
8654 first.driver_pid = Some(std::process::id());
8657 first.driver_started_at = Some(
8658 crate::proc::process_started_at(std::process::id())
8659 .expect("this test process's own start time must be queryable"),
8660 );
8661 first.save().unwrap();
8662 let mut second = run_state(RunStatus::Merged);
8663 second.id = "20260101-000000-supa".to_owned();
8664 second.save().unwrap();
8665
8666 let mut t = task();
8667 t.runs = vec![first.id.clone(), second.id.clone()];
8668 t.status = TaskStatus::Done;
8669
8670 supersede_prior_runs(&t, &crate::run::home());
8671
8672 assert_eq!(
8673 RunState::load(&first.id).unwrap().status,
8674 RunStatus::Blocked,
8675 "a live driver_pid means something is still actually working this run, \
8676 even though no daemon claims it - rewriting under it would just be \
8677 undone the next time that process saves"
8678 );
8679 }
8680
8681 #[test]
8682 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8683 let dir = tempfile::tempdir().unwrap();
8689 let home = dir.path().to_path_buf();
8690 let queue = Queue::at(dir.path().join("queue"));
8691
8692 let mut first = run_state(RunStatus::Blocked);
8693 first.id = "20260101-000000-supd".to_owned();
8694 first.driver_pid = Some(std::process::id());
8695 first.driver_started_at = Some(
8696 crate::proc::process_started_at(std::process::id())
8697 .expect("this test process's own start time must be queryable"),
8698 );
8699 first.save_under(&home).unwrap();
8700 let mut second = run_state(RunStatus::Merged);
8701 second.id = "20260101-000000-supe".to_owned();
8702 second.save_under(&home).unwrap();
8703
8704 let mut t = task();
8705 t.runs = vec![first.id.clone(), second.id.clone()];
8706 t.status = TaskStatus::Done;
8707 queue.put(&mut t).unwrap();
8708
8709 resweep_superseded_attempts(&queue, &home);
8710 assert_eq!(
8711 RunState::load_under(&first.id, &home).unwrap().status,
8712 RunStatus::Blocked,
8713 "still live on the first pass, so still untouched"
8714 );
8715
8716 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8722 stale.driver_started_at = Some("1".to_owned());
8723 stale.save_under(&home).unwrap();
8724
8725 resweep_superseded_attempts(&queue, &home);
8726 assert_eq!(
8727 RunState::load_under(&first.id, &home).unwrap().status,
8728 RunStatus::Superseded,
8729 "the second pass catches up what the first one correctly skipped"
8730 );
8731 }
8732
8733 #[test]
8734 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8735 crate::run::pin_test_home();
8736 let mut first = run_state(RunStatus::Blocked);
8737 first.id = "20260101-000000-sup3".to_owned();
8738 first.save().unwrap();
8739 let mut second = run_state(RunStatus::Blocked);
8740 second.id = "20260101-000000-sup4".to_owned();
8741 second.save().unwrap();
8742
8743 let mut t = task();
8744 t.runs = vec![first.id.clone(), second.id.clone()];
8745 t.status = TaskStatus::Failed;
8749
8750 supersede_prior_runs(&t, &crate::run::home());
8751
8752 assert_eq!(
8753 RunState::load(&first.id).unwrap().status,
8754 RunStatus::Blocked
8755 );
8756 assert_eq!(
8757 RunState::load(&second.id).unwrap().status,
8758 RunStatus::Blocked
8759 );
8760 }
8761
8762 #[test]
8763 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
8764 crate::run::pin_test_home();
8768 let mut first = run_state(RunStatus::Blocked);
8769 first.id = "20260101-000000-sup5".to_owned();
8770 first.save().unwrap();
8771
8772 let mut t = task();
8773 t.runs = vec![first.id.clone()];
8774 t.status = TaskStatus::Done;
8775
8776 supersede_prior_runs(&t, &crate::run::home());
8777
8778 assert_eq!(
8779 RunState::load(&first.id).unwrap().status,
8780 RunStatus::Blocked,
8781 "a single-attempt task has no earlier run to supersede"
8782 );
8783 }
8784
8785 #[test]
8786 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
8787 crate::run::pin_test_home();
8794 let mut first = run_state(RunStatus::Blocked);
8795 first.id = "20260101-000000-supb".to_owned();
8796 first.save().unwrap();
8797 let mut second = run_state(RunStatus::Failed);
8798 second.id = "20260101-000000-supc".to_owned();
8799 second.save().unwrap();
8800
8801 let mut t = task();
8802 t.runs = vec![first.id.clone(), second.id.clone()];
8803 t.status = TaskStatus::Done;
8804
8805 supersede_prior_runs(&t, &crate::run::home());
8806
8807 assert_eq!(
8808 RunState::load(&first.id).unwrap().status,
8809 RunStatus::Blocked,
8810 "the task's last attempt never landed, so there is nothing here \
8811 actually superseding it"
8812 );
8813 }
8814
8815 #[test]
8816 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
8817 crate::run::pin_test_home();
8821 let mut failed = run_state(RunStatus::Failed);
8822 failed.id = "20260101-000000-sup6".to_owned();
8823 failed.save().unwrap();
8824 let mut noop = run_state(RunStatus::VerifiedNoop);
8825 noop.id = "20260101-000000-sup7".to_owned();
8826 noop.save().unwrap();
8827 let mut winner = run_state(RunStatus::Ready);
8828 winner.id = "20260101-000000-sup8".to_owned();
8829 winner.save().unwrap();
8830
8831 let mut t = task();
8832 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
8833 t.status = TaskStatus::Done;
8834
8835 supersede_prior_runs(&t, &crate::run::home());
8836
8837 assert_eq!(
8838 RunState::load(&failed.id).unwrap().status,
8839 RunStatus::Failed
8840 );
8841 assert_eq!(
8842 RunState::load(&noop.id).unwrap().status,
8843 RunStatus::VerifiedNoop
8844 );
8845 }
8846
8847 fn approval_question(run: &str) -> ask::Question {
8848 ask::Question::new(
8849 run.to_owned(),
8850 land::APPROVAL_NODE.to_owned(),
8851 "land".to_owned(),
8852 "merge?".to_owned(),
8853 String::new(),
8854 vec!["merge".to_owned(), "hold".to_owned()],
8855 )
8856 }
8857
8858 #[test]
8859 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
8860 crate::run::pin_test_home();
8861 let mut state = run_state(RunStatus::Landing);
8862 state.id = "20260101-000000-fre1".to_owned();
8863 state.parked = true;
8864 state.save().unwrap();
8865 ask::Questions::open()
8866 .put(&mut approval_question(&state.id))
8867 .unwrap();
8868
8869 let mut t = task();
8870 t.runs.push(state.id.clone());
8871 assert_eq!(
8872 land_resume_state(&t),
8873 LandResume::StillWaiting,
8874 "nobody has answered and the timeout has not passed"
8875 );
8876 }
8877
8878 #[test]
8879 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
8880 crate::run::pin_test_home();
8881 let mut state = run_state(RunStatus::Gating);
8882 state.id = "20260101-000000-gt01".to_owned();
8883 state.parked = true;
8884 state.config.graph.answer_timeout = 60;
8885 state.github_text = Some(crate::run::GithubTextAsk {
8886 rewrite: None,
8887 fingerprint: "f".into(),
8888 title: true,
8889 body: false,
8890 categories: vec!["title-language".into()],
8891 question: None,
8892 asks: 1,
8893 resolved: false,
8894 chosen_title: None,
8895 choice: None,
8896 });
8897 state.save().unwrap();
8898 let store = ask::Questions::open();
8899 let mut q = ask::Question::new(
8900 state.id.clone(),
8901 crate::github_text::ASK_NODE.to_owned(),
8902 crate::github_text::ASK_SEAT.to_owned(),
8903 "s".into(),
8904 String::new(),
8905 vec!["use fallback".into()],
8906 );
8907 store.put(&mut q).unwrap();
8908 let mut t = task();
8909 t.runs.push(state.id.clone());
8910 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
8911 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8912 store.put(&mut q).unwrap();
8913 assert_eq!(land_resume_state(&t), LandResume::Ready);
8914 assert!(!store.get(&q.id).unwrap().status.open());
8915 state.github_text.as_mut().unwrap().resolved = true;
8917 state.save().unwrap();
8918 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
8919 }
8920
8921 #[test]
8922 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
8923 crate::run::pin_test_home();
8928 let mut state = run_state(RunStatus::Landing);
8929 state.id = "20260101-000000-exp1".to_owned();
8930 state.parked = true;
8931 state.config.graph.answer_timeout = 60;
8932 state.save().unwrap();
8933
8934 let store = ask::Questions::open();
8935 let mut q = approval_question(&state.id);
8936 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
8937 store.put(&mut q).unwrap();
8938
8939 let mut t = task();
8940 t.runs.push(state.id.clone());
8941 assert_eq!(
8942 land_resume_state(&t),
8943 LandResume::Ready,
8944 "an expired question must not be waited on forever"
8945 );
8946
8947 let after = store.get(&q.id).unwrap();
8948 assert!(
8949 !after.status.open(),
8950 "the question is abandoned, not silently ignored"
8951 );
8952 assert!(
8953 after.resolution().is_none(),
8954 "an abandoned question is not read as a decision"
8955 );
8956 }
8957
8958 #[test]
8959 fn reclaim_settles_a_running_task_against_its_last_run() {
8960 let mut t = task();
8961 t.start("20260904-000000-4043".to_owned());
8962 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
8963 assert_eq!(
8964 t.status,
8965 TaskStatus::Done,
8966 "a run that actually finished must not stay `running` forever"
8967 );
8968 }
8969
8970 #[test]
8971 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
8972 let mut t = task();
8973 t.start("20260904-000000-4043".to_owned());
8974 t.last_error = Some("earlier trouble".to_owned());
8975 let mut state = run_state(RunStatus::Judging);
8976 state.parked = true;
8977 reclaim(&mut t, Some(state), 1, "en");
8978 assert_eq!(t.status, TaskStatus::Parked);
8979 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
8980 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
8981 assert!(t.park_reason.is_some());
8982 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
8983 assert!(!t.fresh_start);
8984 assert!(t.status.runnable());
8985 }
8986
8987 #[test]
8988 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
8989 let mut t = task();
8990 t.start("20260904-000000-4043".to_owned());
8991 t.last_error = Some("earlier trouble".to_owned());
8992 settle(
8993 &mut t,
8994 Verdict {
8995 status: RunStatus::Judging,
8996 left_pr: false,
8997 quota_hit: false,
8998 parked: true,
8999 no_viable_candidates: false,
9000 },
9001 "parked after `judging`",
9002 1,
9003 );
9004 assert_eq!(t.status, TaskStatus::Parked);
9005 assert_eq!(t.attempts, 0);
9006 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9007 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
9008 assert_eq!(t.schema, crate::queue::SCHEMA);
9009 assert!(
9010 crate::notices::task_held(&t).is_none(),
9011 "a park is not news"
9012 );
9013 t.start("20260904-000000-4043".to_owned());
9014 assert_eq!(t.park_reason, None);
9015 }
9016
9017 #[test]
9018 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
9019 let mut t = task();
9023 t.start("20260904-000000-4043".to_owned());
9024 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
9025 assert_eq!(t.status, TaskStatus::Failed);
9026 assert!(t.status.runnable());
9027 }
9028
9029 #[test]
9030 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
9031 let mut t = task();
9032 t.start("20260904-000000-4043".to_owned());
9033 assert_eq!(t.attempts, 1);
9034 let mut run = run_state(RunStatus::Implementing);
9035 run.id = "20260904-000000-4043".to_owned();
9036 run.begin_pass();
9037 let cause = crate::run::ParkCause::Upgrade {
9038 from: "0.121.0".to_owned(),
9039 to: Some("0.121.1".to_owned()),
9040 };
9041 run.end_pass_parked(cause.clone());
9042 run.parked = true;
9043 let verdict = Verdict {
9044 status: run.status,
9045 left_pr: false,
9046 quota_hit: false,
9047 parked: true,
9048 no_viable_candidates: false,
9049 };
9050 let mut orphan = t.clone();
9053 reclaim(&mut orphan, Some(run.clone()), 2, "en");
9054 assert_eq!(orphan.status, TaskStatus::Parked);
9055 assert_eq!(orphan.attempts, 0);
9056 assert_eq!(
9057 orphan.upgrade_resume_key(),
9058 Some((run.passes[0].started_at, orphan.id.as_str()))
9059 );
9060 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
9061 assert_eq!(t.attempts, 0);
9062 assert_eq!(t.passes.len(), 1);
9063 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9064 assert_eq!(t.passes[0].parked, Some(cause));
9065 assert_eq!(t.passes[0].pass_index, 0);
9066 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
9067 assert!(t.upgrade_resume_key().is_some());
9068
9069 t.start("20260904-000000-4043".to_owned());
9072 run.begin_pass();
9073 run.status = RunStatus::Failed;
9074 let verdict = Verdict {
9075 status: RunStatus::Failed,
9076 left_pr: false,
9077 quota_hit: false,
9078 parked: false,
9079 no_viable_candidates: false,
9080 };
9081 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
9082 assert_eq!(t.passes.len(), 2);
9083 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9084 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
9085 assert_eq!(t.passes[1].pass_index, 1);
9086 assert_eq!(t.attempts, 1);
9087 }
9088
9089 #[test]
9090 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
9091 let mut t = task();
9092 t.start("20260904-000000-4043".to_owned());
9093 reclaim(&mut t, None, 2, "en");
9094 assert_eq!(t.status, TaskStatus::Held);
9095 assert!(
9096 t.last_error
9097 .as_deref()
9098 .is_some_and(|e| e.contains("running")),
9099 "the operator needs to know why this task was held"
9100 );
9101 }
9102
9103 #[test]
9104 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
9105 let dir = tempfile::tempdir().unwrap();
9106 let queue = Queue::at(dir.path().to_path_buf());
9107
9108 let mut orphaned = task();
9110 orphaned.id = "20260904-000000-orph".to_owned();
9111 orphaned.status = TaskStatus::Running;
9112 orphaned.attempts = 1;
9113 queue.put(&mut orphaned).unwrap();
9114
9115 let mut alive = task();
9116 alive.id = "20260904-000000-live".to_owned();
9117 alive.status = TaskStatus::Running;
9118 alive.attempts = 1;
9119 queue.put(&mut alive).unwrap();
9120 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
9121
9122 let mut queued = task();
9123 queued.id = "20260904-000000-wait".to_owned();
9124 queue.put(&mut queued).unwrap();
9125
9126 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9127 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
9128
9129 assert_eq!(
9130 queue.get(&orphaned.id).unwrap().status,
9131 TaskStatus::Held,
9132 "nothing was driving it and there was no run to recover"
9133 );
9134 assert_eq!(
9135 queue.get(&alive.id).unwrap().status,
9136 TaskStatus::Running,
9137 "a live claim must protect the task it belongs to"
9138 );
9139 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9140 }
9141
9142 fn read_run_under(home: &Path, id: &str) -> RunState {
9148 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9149 serde_json::from_str(&body).unwrap()
9150 }
9151
9152 #[test]
9153 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9154 let dir = tempfile::tempdir().unwrap();
9155 let home = dir.path().to_path_buf();
9156 let now = Timestamp::now();
9157 let overrun_seat = || crate::run::ActiveSeat {
9158 node: "implement".to_owned(),
9159 started_at: now - jiff::SignedDuration::new(21_000, 0),
9160 timeout_secs: 3_600,
9161 attempt: 0,
9162 task: None,
9163 command: None,
9164 index: None,
9165 total: None,
9166 };
9167
9168 let mut dead = run_state(RunStatus::Implementing);
9169 dead.id = "20260101-000000-dead".to_owned();
9170 dead.active.insert("impl-A".to_owned(), overrun_seat());
9171 dead.driver_pid = Some(4242);
9174 dead.save_under(&home).unwrap();
9175
9176 let mut alive = run_state(RunStatus::Implementing);
9179 alive.id = "20260101-000000-aliv".to_owned();
9180 alive.active.insert("impl-A".to_owned(), overrun_seat());
9181 alive.save_under(&home).unwrap();
9182 let mut status = Status::new();
9183 status.current = vec![Current {
9184 worker: None,
9185 task: "20260101-000000-task".to_owned(),
9186 run: alive.id.clone(),
9187 }];
9188 write_status_to(&home.join("daemon.json"), &status).unwrap();
9189
9190 let questions = Questions::at(home.join("questions"));
9194 let mut q = ask::Question::new(
9195 dead.id.clone(),
9196 "implement".to_owned(),
9197 "impl-A".to_owned(),
9198 "Which storage backend?".to_owned(),
9199 String::new(),
9200 vec!["SQLite".to_owned(), "Redis".to_owned()],
9201 );
9202 questions.put(&mut q).unwrap();
9203
9204 let abandoned = reclaim_abandoned_runs_with(
9205 &home,
9206 now,
9207 |pid| if pid == 4242 { Some(false) } else { None },
9208 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9209 );
9210 assert_eq!(abandoned, vec![dead.id.clone()]);
9211
9212 let reloaded = read_run_under(&home, &dead.id);
9213 assert_eq!(reloaded.status, RunStatus::Failed);
9214 assert!(reloaded.active.is_empty());
9215 assert!(
9216 !questions.get(&q.id).unwrap().status.open(),
9217 "the failed run's own open question must be settled in the same pass"
9218 );
9219
9220 let still_alive = read_run_under(&home, &alive.id);
9221 assert_eq!(
9222 still_alive.status,
9223 RunStatus::Implementing,
9224 "a live daemon's claim protects it"
9225 );
9226 assert!(!still_alive.active.is_empty());
9227 }
9228
9229 #[test]
9239 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9240 let dir = tempfile::tempdir().unwrap();
9241 let home = dir.path().to_path_buf();
9242 let now = Timestamp::now();
9243
9244 let mut manual = run_state(RunStatus::Reviewing);
9245 manual.id = "20260101-000000-manl".to_owned();
9246 manual.active.insert(
9247 "review-1".to_owned(),
9248 crate::run::ActiveSeat {
9249 node: "review".to_owned(),
9250 started_at: now - jiff::SignedDuration::new(21_000, 0),
9251 timeout_secs: 3_600,
9252 attempt: 0,
9253 task: None,
9254 command: None,
9255 index: None,
9256 total: None,
9257 },
9258 );
9259 manual.driver_pid = Some(4242);
9263 manual.driver_started_at = Some("1790000000".to_owned());
9264 manual.save_under(&home).unwrap();
9265
9266 let abandoned = reclaim_abandoned_runs_with(
9267 &home,
9268 now,
9269 |pid| if pid == 4242 { Some(true) } else { None },
9270 |pid| {
9271 if pid == 4242 {
9272 Some("1790000000".to_owned())
9273 } else {
9274 None
9275 }
9276 },
9277 );
9278 assert!(
9279 abandoned.is_empty(),
9280 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9281 );
9282
9283 let reloaded = read_run_under(&home, &manual.id);
9284 assert_eq!(reloaded.status, RunStatus::Reviewing);
9285 assert!(!reloaded.active.is_empty());
9286 }
9287
9288 #[test]
9289 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9290 let dir = tempfile::tempdir().unwrap();
9291 let queue = Queue::at(dir.path().to_path_buf());
9292 let mut only = task();
9293 queue.put(&mut only).unwrap();
9294
9295 let _elsewhere = queue.claim(&only.id).unwrap();
9296 let candidates = runnable(&queue);
9297 assert_eq!(candidates.len(), 1, "the task is still runnable");
9298 assert!(
9299 queue.claim(&candidates[0].id).is_err(),
9300 "the loop cannot take a claim somebody else holds"
9301 );
9302
9303 let after = queue.get(&only.id).unwrap();
9304 assert_eq!(after.status, TaskStatus::Queued);
9305 assert_eq!(
9306 after.attempts, 0,
9307 "losing the race is not an attempt at the task"
9308 );
9309 assert_eq!(after.last_error, None);
9310 }
9311
9312 #[test]
9313 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9314 let dir = tempfile::tempdir().unwrap();
9315 let path = dir.path().join("daemon.json");
9316
9317 let mut status = Status::new();
9318 status.idle = false;
9319 status.completed = 7;
9320 status.current = vec![Current {
9321 worker: None,
9322 task: "20260902-000000-t111".to_owned(),
9323 run: "20260902-000001-r111".to_owned(),
9324 }];
9325 write_status_to(&path, &status).unwrap();
9326 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9327 assert_eq!(first.schema, SCHEMA);
9328 assert_eq!(first.pid, std::process::id());
9329 assert!(!first.idle);
9330 assert_eq!(first.completed, 7);
9331 assert_eq!(first.current, status.current);
9332 assert!(
9333 !path.with_extension("json.tmp").exists(),
9334 "the temp file is renamed, not left behind"
9335 );
9336
9337 std::thread::sleep(Duration::from_millis(5));
9338 status.updated_at = Timestamp::now();
9339 status.polls = 3;
9340 write_status_to(&path, &status).unwrap();
9341 let second: Status =
9342 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9343 assert!(
9344 second.updated_at > first.updated_at,
9345 "a reader can only detect staleness if the heartbeat moves"
9346 );
9347 assert_eq!(
9348 second.started_at, first.started_at,
9349 "the start time is not a heartbeat"
9350 );
9351 assert_eq!(second.polls, 3);
9352 }
9353
9354 #[test]
9355 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9356 let dir = tempfile::tempdir().unwrap();
9357
9358 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9359
9360 let mut status = Status::new();
9361 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9362 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9363 let stale = read_status(dir.path()).unwrap();
9364 assert!(
9365 !stale.running(Timestamp::now()),
9366 "a minute without a heartbeat is a dead daemon, not a busy one"
9367 );
9368 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9369
9370 status.updated_at = Timestamp::now();
9371 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9372 let fresh = read_status(dir.path()).unwrap();
9373 assert!(fresh.running(Timestamp::now()));
9374 }
9375
9376 #[test]
9377 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9378 let dir = tempfile::tempdir().unwrap();
9379 let now = Timestamp::now();
9380 let mine = "20260903-080619-01c2";
9381
9382 assert!(
9383 !is_working_on(dir.path(), mine, now),
9384 "no status file means nobody is working on anything"
9385 );
9386
9387 let mut status = Status::new();
9388 status.current = vec![Current {
9389 worker: None,
9390 task: "20260903-080340-0167".to_owned(),
9391 run: mine.to_owned(),
9392 }];
9393 status.updated_at = now;
9394 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9395 assert!(is_working_on(dir.path(), mine, now));
9396 assert!(
9397 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9398 "a daemon busy with one run is not working on another"
9399 );
9400
9401 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9404 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9405 assert!(
9406 !is_working_on(dir.path(), mine, now),
9407 "a stale heartbeat is a dead daemon, so its run is a leftover"
9408 );
9409 }
9410
9411 #[test]
9412 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9413 let dir = tempfile::tempdir().unwrap();
9414 let now = Timestamp::now();
9415
9416 assert!(
9417 !is_working_on_short(dir.path(), "01c2", now),
9418 "no status file means nobody is working on anything"
9419 );
9420
9421 let mut status = Status::new();
9422 status.current = vec![Current {
9423 worker: None,
9424 task: "20260903-080340-0167".to_owned(),
9425 run: "20260903-080619-01c2".to_owned(),
9426 }];
9427 status.updated_at = now;
9428 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9429 assert!(
9430 is_working_on_short(dir.path(), "01c2", now),
9431 "the run's short id is the last block of its full id"
9432 );
9433 assert!(
9434 !is_working_on_short(dir.path(), "3cbf", now),
9435 "a daemon busy with one worktree bay is not working on another"
9436 );
9437 }
9438
9439 #[test]
9440 fn a_newer_status_file_still_yields_a_reading() {
9441 let dir = tempfile::tempdir().unwrap();
9442 std::fs::write(
9445 dir.path().join("daemon.json"),
9446 serde_json::json!({
9447 "schema": 2,
9448 "updated_at": Timestamp::now().to_string(),
9449 "idle": true,
9450 "surprise": { "nested": [1, 2, 3] },
9451 })
9452 .to_string(),
9453 )
9454 .unwrap();
9455
9456 let reading = read_status(dir.path()).expect("a forward-compatible read");
9457 assert!(reading.running(Timestamp::now()));
9458 assert!(reading.idle);
9459 assert!(reading.current.is_empty());
9460 }
9461
9462 #[test]
9463 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9464 let dir = tempfile::tempdir().unwrap();
9470 std::fs::write(
9471 dir.path().join("daemon.json"),
9472 serde_json::json!({
9473 "schema": 1,
9474 "pid": 4242,
9475 "updated_at": Timestamp::now().to_string(),
9476 "idle": false,
9477 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9478 "completed": 3,
9479 "polls": 9,
9480 })
9481 .to_string(),
9482 )
9483 .unwrap();
9484
9485 let reading = read_status(dir.path()).expect("an older shape must still parse");
9486 assert!(reading.running(Timestamp::now()));
9487 assert_eq!(
9488 reading.current,
9489 vec![Current {
9490 worker: None,
9491 task: "20260902-140501-aaaa".to_owned(),
9492 run: "20260902-140502-bbbb".to_owned(),
9493 }]
9494 );
9495 }
9496
9497 #[test]
9498 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9499 let dir = tempfile::tempdir().unwrap();
9500 std::fs::write(
9501 dir.path().join("daemon.json"),
9502 serde_json::json!({
9503 "schema": 1,
9504 "updated_at": Timestamp::now().to_string(),
9505 "idle": true,
9506 "current": null,
9507 })
9508 .to_string(),
9509 )
9510 .unwrap();
9511 let with_null = read_status(dir.path()).expect("null must still parse");
9512 assert!(with_null.current.is_empty());
9513
9514 std::fs::write(
9515 dir.path().join("daemon.json"),
9516 serde_json::json!({
9517 "schema": 1,
9518 "updated_at": Timestamp::now().to_string(),
9519 "idle": true,
9520 })
9521 .to_string(),
9522 )
9523 .unwrap();
9524 let absent = read_status(dir.path()).expect("a missing field must still parse");
9525 assert!(absent.current.is_empty());
9526 }
9527
9528 #[test]
9529 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9530 let fallback = Path::new("/default");
9531 let mut blank = task();
9532 blank.repo = PathBuf::new();
9533 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9534 let mut dot = task();
9535 dot.repo = PathBuf::from(".");
9536 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9537 assert_eq!(
9538 repo_for(&task(), fallback),
9539 PathBuf::from("/repo"),
9540 "a task that names a repository keeps it"
9541 );
9542 }
9543
9544 #[test]
9545 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9546 let dir = tempfile::tempdir().unwrap();
9547 let path = dir.path().join("explicit.toml");
9548 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9549 let opts = Opts {
9550 config: Some(path),
9551 merge: Some("none".into()),
9552 ..Default::default()
9553 };
9554 let mut t = task();
9555 t.set_implementers(
9556 2,
9557 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9558 .unwrap(),
9559 )
9560 .unwrap();
9561 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9562 assert_eq!(cfg.graph.implementers, 2);
9563 assert_eq!(cfg.graph.judges, 1);
9564 assert_eq!(cfg.merge.mode, MergeMode::None);
9565 t.set_implementers(1, &cfg).unwrap();
9566 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9567 apply_task_implementers(&mut cfg, &t);
9568 assert_eq!(cfg.graph.implementers, 1);
9569 assert_eq!(cfg.graph.judges, 1);
9570 assert_eq!(cfg.merge.mode, MergeMode::None);
9571 }
9572
9573 #[test]
9574 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9575 let dir = tempfile::tempdir().unwrap();
9576 let repo = dir.path().join("repo");
9577 std::fs::create_dir(&repo).unwrap();
9578 std::fs::write(
9579 repo.join("magi.toml"),
9580 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9581 )
9582 .unwrap();
9583 let daemon_config = dir.path().join("daemon.toml");
9584 std::fs::write(
9585 &daemon_config,
9586 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9587 )
9588 .unwrap();
9589 let opts = Opts {
9590 config: Some(daemon_config),
9591 merge: Some("pr".into()),
9592 ..Default::default()
9593 };
9594 let q = Queue::at(dir.path().join("queue"));
9595 let validation: Config =
9596 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9597 .unwrap();
9598 for record in ["{}", r#"{"candidates":2}"#] {
9600 let overrides = serde_json::from_str(record).unwrap();
9601 let filing = crate::direct::Filing {
9602 instruction: "work".into(),
9603 title: "direct".into(),
9604 repo: repo.clone(),
9605 source: crate::queue::Source::Human,
9606 solo: false,
9607 overrides,
9608 review_of: None,
9609 sources: Default::default(),
9610 };
9611 let filed =
9612 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9613 let crate::direct::Filed::Standalone {
9614 mut task,
9615 claim: _claim,
9616 } = filed
9617 else {
9618 panic!("local filing")
9619 };
9620 let before = prepare_for(&repo, &opts, &task).unwrap();
9621 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9622 assert_eq!(
9623 before.merge.mode,
9624 MergeMode::None,
9625 "daemon --merge must not leak"
9626 );
9627 for n in [2, 1, 2] {
9628 task.set_implementers(n, &validation).unwrap();
9629 q.put(&mut task).unwrap();
9630 task = q.get(&task.id).unwrap();
9631 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9632 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9633 apply_task_implementers(&mut cfg, &task);
9634 assert_eq!(cfg.graph.implementers, usize::from(n));
9635 assert_eq!(cfg.graph.judges, 2);
9636 assert_eq!(cfg.merge.mode, MergeMode::None);
9637 }
9638 }
9639 }
9640
9641 #[test]
9642 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9643 let cfg: Config =
9644 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9645 .unwrap();
9646 let mut t = task();
9647 t.review_of = Some("review-target".into());
9648 t.hold_manual(Some("retry".into()));
9649 let mut previous = RunState::new(
9650 PathBuf::from("/repo"),
9651 "main".into(),
9652 "abc1234".into(),
9653 "review".into(),
9654 cfg.clone(),
9655 );
9656 previous.config.graph.implementers = 2;
9657 for count in [1, 3] {
9658 t.set_implementers(count, &cfg).unwrap();
9659 t.release();
9660 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9661 for proposed in [
9662 Starter::Resume(previous.id.clone()),
9663 Starter::Review("recovery-target".into()),
9664 ] {
9665 assert_eq!(
9666 starter_with_implementers(&t, Some(&previous), proposed),
9667 Starter::Review("review-target".into())
9668 );
9669 }
9670 let mut configured = cfg.clone();
9671 apply_task_implementers(&mut configured, &t);
9672 assert_eq!(configured.graph.implementers, usize::from(count));
9673 assert_eq!(previous.config.graph.implementers, 2);
9674 }
9675 }
9676
9677 #[test]
9678 fn task_implementers_dispatch_and_resume_agree() {
9679 let mut t = task();
9680 let cfg: crate::config::Config =
9681 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9682 .unwrap();
9683 let mut run = RunState::new(
9684 PathBuf::from("/repo"),
9685 "main".into(),
9686 "abc1234".into(),
9687 "work".into(),
9688 cfg.clone(),
9689 );
9690 run.config.graph.implementers = 2;
9691 for n in [2, 1, 3, 2] {
9692 t.set_implementers(n, &cfg).unwrap();
9693 let mut dispatched = cfg.clone();
9694 apply_task_implementers(&mut dispatched, &t);
9695 assert_eq!(dispatched.graph.implementers, usize::from(n));
9696 assert_eq!(implementers_match(&t, &run), n == 2);
9697 for proposed in [
9698 Starter::Resume(run.id.clone()),
9699 Starter::Review("old-branch".into()),
9700 ] {
9701 let expected = if n == 2 {
9702 proposed.clone()
9703 } else {
9704 Starter::Start
9705 };
9706 assert_eq!(
9707 starter_with_implementers(&t, Some(&run), proposed),
9708 expected
9709 );
9710 }
9711 assert_eq!(
9712 run.config.graph.implementers, 2,
9713 "history must not be rewritten"
9714 );
9715 }
9716 t.solo = false;
9717 t.overrides = None;
9718 assert!(implementers_match(&t, &run));
9719 }
9720
9721 #[test]
9722 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9723 let mut solo_cfg = Config::default();
9729 solo_cfg.graph.implementers = 3;
9730 let mut solo_task = task();
9731 solo_task.solo = true;
9732 apply_task_implementers(&mut solo_cfg, &solo_task);
9733 assert_eq!(solo_cfg.graph.implementers, 1);
9734
9735 let mut plain_cfg = Config::default();
9736 plain_cfg.graph.implementers = 3;
9737 let plain_task = task();
9738 assert!(!plain_task.solo);
9739 apply_task_implementers(&mut plain_cfg, &plain_task);
9740 assert_eq!(
9741 plain_cfg.graph.implementers, 3,
9742 "a task that did not ask to run alone keeps the config's candidates"
9743 );
9744 }
9745
9746 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9747 QuotaLoss {
9748 seat: seat.into(),
9749 node: "judge".into(),
9750 at: at.parse().unwrap(),
9751 reset: reset.map(str::to_string),
9752 }
9753 }
9754
9755 #[test]
9756 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
9757 let old: Vec<QuotaLoss> = (1..=4)
9758 .map(|i| {
9759 loss(
9760 &format!("judge-{i}"),
9761 "2026-09-23T05:23:00Z",
9762 Some("2:40pm (Asia/Tokyo)"),
9763 )
9764 })
9765 .collect();
9766 let fresh = losses_this_attempt(&old, &old);
9767 assert!(fresh.is_empty());
9768 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
9769 }
9771
9772 #[test]
9773 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
9774 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
9775 let now = Timestamp::now();
9776 let mut after = old.clone();
9777 after.push(loss("judge-2", &now.to_string(), None));
9778 let fresh = losses_this_attempt(&old, &after);
9779 assert_eq!(fresh, vec![after[1].clone()]);
9780 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
9781 assert_eq!(
9782 until,
9783 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
9784 );
9785 }
9786
9787 #[test]
9788 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
9789 let before = vec![
9792 loss("judge-1", "2026-09-23T05:23:00Z", None),
9793 loss("judge-2", "2026-09-23T05:24:00Z", None),
9794 ];
9795 let after = vec![
9796 loss("judge-2", "2026-09-23T05:24:00Z", None),
9797 loss("judge-1", "2026-09-24T01:00:00Z", None),
9798 ];
9799 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
9800 }
9801
9802 #[test]
9803 fn merge_overrides_are_parsed_or_refused() {
9804 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
9805 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
9806 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
9807 assert!(merge_mode("squash").is_err());
9808 }
9809
9810 #[test]
9811 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
9812 let now = Timestamp::now();
9813 let fallback = Duration::from_secs(300);
9814 let cap = Duration::from_secs(1800);
9815
9816 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
9818
9819 let soon = now + jiff::SignedDuration::from_secs(600);
9821 assert_eq!(
9822 quota_wait(Some(soon), now, fallback, cap),
9823 Duration::from_secs(600)
9824 );
9825
9826 let past = now - jiff::SignedDuration::from_secs(60);
9829 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
9830
9831 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
9834 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
9835 }
9836
9837 #[test]
9838 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
9839 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9840
9841 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
9842 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
9843
9844 let already_past =
9848 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
9849 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
9850
9851 assert!(
9852 parse_reset_hint("session limit reached", now, now).is_none(),
9853 "free text with no recognised shape is not guessed at"
9854 );
9855 assert!(
9856 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
9857 "an unresolvable zone name is not guessed at either"
9858 );
9859 }
9860
9861 #[test]
9862 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
9863 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
9864
9865 let at = parse_reset_hint(
9866 "You've hit your usage limit. Visit \
9867 https://chatgpt.com/codex/settings/usage to purchase more \
9868 credits or try again at Sep 19th, 2026 5:10 PM.",
9869 now,
9870 now,
9871 )
9872 .expect("the codex reset wording is a recognised shape");
9873 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
9874
9875 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
9880 .expect("an explicit year needs no rollover");
9881 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
9882
9883 assert!(
9884 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
9885 "a two-digit year is not the documented shape and is not guessed at"
9886 );
9887 assert!(
9888 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
9889 "a four-letter month name is not the documented three-letter abbreviation"
9890 );
9891 assert!(
9892 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
9893 "an explicit zone on the dated shape is a format nobody has \
9894 documented, and is refused rather than guessed at as UTC"
9895 );
9896 }
9897
9898 #[test]
9899 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
9900 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
9901 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
9902
9903 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
9904 assert_eq!(at.as_second() - recorded.as_second(), 3769);
9905
9906 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
9907 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
9908
9909 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
9910 assert!(
9911 parse_reset_hint(bad, now, recorded).is_none(),
9912 "{bad:?} must not be guessed at"
9913 );
9914 }
9915 }
9916
9917 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
9921 let config = dir.join("magi.toml");
9922 std::fs::write(
9923 &config,
9924 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
9925 )
9926 .unwrap();
9927 let opts = Opts {
9928 poll: Duration::from_secs(30),
9929 config: Some(config),
9930 repo: dir.join("repo"),
9934 ..Opts::default()
9935 };
9936 let home = dir.join("home");
9945 let stamps = home.join("service-ticks");
9948 std::fs::create_dir_all(&stamps).unwrap();
9949 for service in ["waiter", "deputies", "release-watch"] {
9950 std::fs::write(
9951 stamps.join(service),
9952 Timestamp::now().as_second().to_string(),
9953 )
9954 .unwrap();
9955 }
9956 for operation in [
9957 "conductor",
9958 "maintenance",
9959 "housekeeping",
9960 "disk-pressure",
9961 "cache",
9962 "fetch",
9963 ] {
9964 std::fs::write(
9965 stamps.join(format!("daemon-{operation}")),
9966 serde_json::to_vec(&Timestamp::now()).unwrap(),
9967 )
9968 .unwrap();
9969 }
9970 let worktrees = dir.join("wt");
9971 (
9972 opts,
9973 Queue::at(dir.join("queue")),
9974 home.join("daemon.json"),
9975 home,
9976 worktrees,
9977 )
9978 }
9979
9980 #[test]
9981 fn a_stop_is_idempotent_and_once_set_stays_set() {
9982 let stop = Stop::new();
9983 assert!(!stop.stopped());
9984
9985 stop.stop();
9986 assert!(stop.stopped());
9987 stop.stop();
9988 assert!(stop.stopped(), "a second stop is not a toggle");
9989
9990 let shared = stop.clone();
9991 assert!(
9992 shared.stopped(),
9993 "a clone is the same stop; that is how the loop and its caller share one"
9994 );
9995 }
9996
9997 #[test]
9998 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
9999 let stop = Stop::new();
10000 stop.busy.store(1, Ordering::SeqCst);
10001 assert!(
10002 !stop.finishing(),
10003 "a busy loop nobody has asked to stop is just running"
10004 );
10005
10006 stop.stop();
10007 assert!(
10008 stop.finishing(),
10009 "a stop asked for mid-run has not landed until the run is settled"
10010 );
10011
10012 stop.busy.store(0, Ordering::SeqCst);
10013 assert!(
10014 !stop.finishing(),
10015 "once the run is settled the stop has landed and there is nothing to finish"
10016 );
10017 }
10018
10019 #[test]
10020 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
10021 let stop = Stop::new();
10022 stop.busy.store(2, Ordering::SeqCst);
10023 stop.stop();
10024 assert!(stop.finishing(), "two runs still in flight");
10025
10026 stop.busy.store(1, Ordering::SeqCst);
10027 assert!(
10028 stop.finishing(),
10029 "one run finished, but a sibling is still working"
10030 );
10031
10032 stop.busy.store(0, Ordering::SeqCst);
10033 assert!(
10034 !stop.finishing(),
10035 "the last run out is what actually lands the stop"
10036 );
10037 }
10038
10039 #[tokio::test]
10040 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
10041 let dir = tempfile::tempdir().unwrap();
10042 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10043 let stop = Stop::new();
10044 stop.stop();
10045
10046 let began = std::time::Instant::now();
10047 tokio::time::timeout(
10048 Duration::from_secs(2),
10049 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10050 )
10051 .await
10052 .expect("a stopped loop must return, not sit out its poll interval")
10053 .expect("the loop's own setup and teardown must not fail");
10054 assert!(
10055 began.elapsed() < opts.poll,
10056 "returned only after {:?}, which is a poll interval, not a stop",
10057 began.elapsed()
10058 );
10059 }
10060
10061 #[tokio::test]
10062 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
10063 let dir = tempfile::tempdir().unwrap();
10064 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10065 let stop = Stop::new();
10066
10067 let asker = {
10070 let stop = stop.clone();
10071 tokio::spawn(async move {
10072 tokio::time::sleep(Duration::from_millis(20)).await;
10073 stop.stop();
10074 })
10075 };
10076
10077 let began = std::time::Instant::now();
10078 tokio::time::timeout(
10079 Duration::from_secs(2),
10080 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10081 )
10082 .await
10083 .expect("a stop asked for while idle must wake the wait")
10084 .expect("the loop's own setup and teardown must not fail");
10085 asker.await.unwrap();
10086 assert!(
10087 began.elapsed() < opts.poll,
10088 "returned only after {:?}, so the stop waited on the sleep",
10089 began.elapsed()
10090 );
10091 }
10092
10093 #[tokio::test]
10094 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
10095 let dir = tempfile::tempdir().unwrap();
10096 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10097 let stop = Stop::new();
10098 stop.stop();
10099
10100 tokio::time::timeout(
10101 Duration::from_secs(2),
10102 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10103 )
10104 .await
10105 .expect("a stopped loop must return")
10106 .expect("the loop's own setup and teardown must not fail");
10107
10108 assert!(
10109 home.is_dir(),
10110 "the loop did publish a status file, so its removal is the teardown and not an absence"
10111 );
10112 assert!(
10113 !status_file.exists(),
10114 "a stopped loop clears its status file"
10115 );
10116 assert!(
10117 read_status(&home).is_none(),
10118 "a reader must see no daemon at all, not a heartbeat that merely stopped"
10119 );
10120 }
10121
10122 #[tokio::test]
10123 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
10124 let dir = tempfile::tempdir().unwrap();
10125 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10126
10127 let mut settled = RunState::new(
10128 dir.path().join("repo"),
10129 "main".to_owned(),
10130 "abc1234".to_owned(),
10131 "fixture".to_owned(),
10132 Config::default(),
10133 );
10134 settled.status = RunStatus::Ready;
10135 let run_dir = home.join("runs").join(&settled.id);
10136 std::fs::create_dir_all(&run_dir).unwrap();
10137 std::fs::write(
10138 run_dir.join("run.json"),
10139 serde_json::to_string_pretty(&settled).unwrap(),
10140 )
10141 .unwrap();
10142 let questions = Questions::at(home.join("questions"));
10143 let mut question = ask::Question::new(
10144 settled.id.clone(),
10145 "review".to_owned(),
10146 "reviewer-1".to_owned(),
10147 "Continue?".to_owned(),
10148 String::new(),
10149 Vec::new(),
10150 );
10151 questions.put(&mut question).unwrap();
10152
10153 run_service_worker(
10154 &home,
10155 serde_json::to_value(ServiceWorker {
10156 opts,
10157 queue: queue.root().to_path_buf(),
10158 worktrees_root: worktrees,
10159 operation: "housekeeping".to_owned(),
10160 })
10161 .unwrap(),
10162 )
10163 .await
10164 .unwrap();
10165
10166 assert_eq!(
10167 questions.get(&question.id).unwrap().status,
10168 ask::QuestionStatus::Abandoned,
10169 "the independent housekeeping worker settles ended-run questions"
10170 );
10171 }
10172
10173 #[test]
10174 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10175 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10176
10177 assert!(
10178 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10179 "never checked before: due at once"
10180 );
10181
10182 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10183 assert!(
10184 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10185 "well inside the interval: not due yet"
10186 );
10187
10188 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10189 assert!(
10190 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10191 "exactly at the edge: not yet due, same convention as `clean::due`"
10192 );
10193
10194 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10195 assert!(
10196 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10197 "past the interval: due again"
10198 );
10199 }
10200
10201 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10206 let config = dir.join("magi.toml");
10207 std::fs::write(
10213 &config,
10214 format!(
10215 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10216 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10217 cache_dir.display()
10218 ),
10219 )
10220 .unwrap();
10221 Opts {
10222 config: Some(config),
10223 repo: dir.join("repo"),
10224 ..Opts::default()
10225 }
10226 }
10227
10228 #[tokio::test]
10229 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10230 let dir = tempfile::tempdir().unwrap();
10231 let home = dir.path().join("home");
10232 let cache_dir = dir.path().join("cache");
10233 std::fs::create_dir_all(&cache_dir).unwrap();
10234 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10235 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10236
10237 let running = Stop::new();
10240 let mut last_checked = None;
10241 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10242 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10243 .await;
10244 assert_eq!(
10245 crate::disk::dir_size(&cache_dir),
10246 0,
10247 "over the cap on the first check ever: pruned at once, no idle queue required"
10248 );
10249 assert_eq!(last_checked, Some(t0));
10250
10251 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10253 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10254 maybe_prune_cache_between_runs(
10255 &opts.repo,
10256 &opts,
10257 &home,
10258 &running,
10259 &mut last_checked,
10260 too_soon,
10261 )
10262 .await;
10263 assert_eq!(
10264 crate::disk::dir_size(&cache_dir),
10265 10,
10266 "too soon since the last check: left alone rather than rescanned every call"
10267 );
10268 assert_eq!(
10269 last_checked,
10270 Some(t0),
10271 "an idle check does not reset the clock"
10272 );
10273
10274 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10276 maybe_prune_cache_between_runs(
10277 &opts.repo,
10278 &opts,
10279 &home,
10280 &running,
10281 &mut last_checked,
10282 due_again,
10283 )
10284 .await;
10285 assert_eq!(
10286 crate::disk::dir_size(&cache_dir),
10287 0,
10288 "due again: pruned back under the cap"
10289 );
10290 }
10291
10292 #[tokio::test]
10300 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
10301 let dir = tempfile::tempdir().unwrap();
10302 let home = dir.path().join("home");
10303 let cache_dir = dir.path().join("cache");
10304 std::fs::create_dir_all(&cache_dir).unwrap();
10305 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10306 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10307
10308 let stop = Stop::new();
10309 stop.stop();
10310 assert!(
10311 !stop.finishing(),
10312 "no run is in flight at a between-runs boundary, so nothing else \
10313 would tell the operator this stop had not taken effect yet"
10314 );
10315
10316 let mut last_checked = None;
10317 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10318 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10319 .await;
10320 assert_eq!(
10321 crate::disk::dir_size(&cache_dir),
10322 10,
10323 "over its cap, and due for the first check ever, but a stop outranks \
10324 it: the cap is a standing policy the next start measures again"
10325 );
10326 assert_eq!(
10327 last_checked, None,
10328 "a check that never happened must not claim the interval"
10329 );
10330 }
10331
10332 #[tokio::test]
10335 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10336 let dir = tempfile::tempdir().unwrap();
10337 let cache_dir = dir.path().join("cache");
10338 std::fs::create_dir_all(&cache_dir).unwrap();
10339 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10340
10341 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10342 opts.poll = Duration::from_millis(20);
10343 opts.max_attempts = 1_000;
10344
10345 let queue = Queue::at(dir.path().join("queue"));
10346 for i in 0..128 {
10347 let mut task = Task::new(
10348 format!("queued work {i}"),
10349 format!("queued work {i}"),
10350 opts.repo.clone(),
10351 Source::Human,
10352 );
10353 queue.put(&mut task).unwrap();
10354 }
10355
10356 let home = dir.path().join("home");
10357 let worktrees = dir.path().join("wt");
10358 run_service_worker(
10359 &home,
10360 serde_json::to_value(ServiceWorker {
10361 opts,
10362 queue: queue.root().to_path_buf(),
10363 worktrees_root: worktrees,
10364 operation: "cache".to_owned(),
10365 })
10366 .unwrap(),
10367 )
10368 .await
10369 .unwrap();
10370 let after = queue.list();
10371 assert_eq!(after.len(), 128);
10372 assert!(
10373 after
10374 .iter()
10375 .all(|task| task.status.runnable() && task.attempts == 0)
10376 );
10377
10378 assert_eq!(
10379 crate::disk::dir_size(&cache_dir),
10380 0,
10381 "an oversized cache must not be left to grow unboundedly just because the \
10382 queue kept the loop busy the whole time"
10383 );
10384 }
10385
10386 #[test]
10387 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10388 let dir = tempfile::tempdir().unwrap();
10389 let queue = Queue::at(dir.path().join("queue"));
10390 let questions = Questions::at(dir.path().join("questions"));
10391 let mut task = task();
10392 queue.put(&mut task).unwrap();
10393
10394 let mut task_question = ask::Question::new(
10395 task.id.clone(),
10396 crate::conduct::NODE.to_owned(),
10397 "conduct".to_owned(),
10398 "Which backend?".to_owned(),
10399 String::new(),
10400 Vec::new(),
10401 );
10402 questions.put(&mut task_question).unwrap();
10403 task.block(vec![task_question.id.clone()], None);
10404 queue.put(&mut task).unwrap();
10405
10406 let mut run_question = ask::Question::new(
10407 "20260101-000000-run1".to_owned(),
10408 "review".to_owned(),
10409 "reviewer-1".to_owned(),
10410 "Run question".to_owned(),
10411 String::new(),
10412 Vec::new(),
10413 );
10414 questions.put(&mut run_question).unwrap();
10415
10416 let mut coincidental = ask::Question::new(
10421 task.id.clone(),
10422 "review".to_owned(),
10423 "reviewer-1".to_owned(),
10424 "Unrelated review question".to_owned(),
10425 String::new(),
10426 Vec::new(),
10427 );
10428 questions.put(&mut coincidental).unwrap();
10429
10430 reconcile_task_questions(&queue, &questions);
10431 assert!(questions.get(&task_question.id).unwrap().status.open());
10432 assert!(questions.get(&run_question.id).unwrap().status.open());
10433 assert!(questions.get(&coincidental.id).unwrap().status.open());
10434
10435 task.release();
10436 queue.put(&mut task).unwrap();
10437 reconcile_task_questions(&queue, &questions);
10438 assert_eq!(
10439 questions.get(&task_question.id).unwrap().status,
10440 ask::QuestionStatus::Abandoned
10441 );
10442 assert!(
10443 questions.get(&run_question.id).unwrap().status.open(),
10444 "run questions remain the run janitor's responsibility"
10445 );
10446 assert!(
10447 questions.get(&coincidental.id).unwrap().status.open(),
10448 "a non-conductor question must not be abandoned just because its \
10449 run id coincides with a task id"
10450 );
10451 }
10452
10453 #[test]
10454 fn a_freshly_started_running_task_is_never_stalled() {
10455 let dir = tempfile::tempdir().unwrap();
10456 let mut t = task();
10457 t.start("run-1".to_owned());
10458 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10461 }
10462
10463 #[test]
10464 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10465 let dir = tempfile::tempdir().unwrap();
10466 let mut t = task();
10467 t.start("run-1".to_owned());
10468 t.updated_at = Timestamp::now()
10469 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10470 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10471 assert_eq!(
10472 stalled_tasks(
10473 &Queue::at(dir.path().join("q")),
10474 dir.path(),
10475 Timestamp::now()
10476 )
10477 .len(),
10478 0,
10479 "the task was never written to this queue"
10480 );
10481 }
10482
10483 #[test]
10484 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10485 let dir = tempfile::tempdir().unwrap();
10486 let mut t = task();
10487 t.id = "20260903-080340-0167".to_owned();
10488 t.start("20260903-080619-01c2".to_owned());
10489 t.updated_at = Timestamp::now()
10490 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10491
10492 let mut status = Status::new();
10493 status.current = vec![Current {
10494 worker: None,
10495 task: t.id.clone(),
10496 run: "20260903-080619-01c2".to_owned(),
10497 }];
10498 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10499
10500 assert!(
10501 !is_stalled(&t, dir.path(), Timestamp::now()),
10502 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10503 );
10504 }
10505
10506 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10510 let path = queue.path_of(id);
10511 let body = std::fs::read_to_string(&path).unwrap();
10512 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10513 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10514 v["updated_at"] = serde_json::Value::String(old.to_string());
10515 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10516 }
10517
10518 #[test]
10519 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10520 let dir = tempfile::tempdir().unwrap();
10533 let queue = Queue::at(dir.path().join("queue"));
10534 let home = dir.path().join("home");
10535
10536 let mut t = task();
10537 t.id = "20260101-000001-lock".to_owned();
10538 t.start("run-1".to_owned());
10539 queue.put(&mut t).unwrap();
10540 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10541 std::fs::write(
10542 dir.path().join("queue").join(format!("{}.lock", t.id)),
10543 "not a pid",
10544 )
10545 .unwrap();
10546
10547 let now = Timestamp::now();
10548 assert!(
10549 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10550 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10551 and reclaim must leave the task alone"
10552 );
10553 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10554
10555 let stalled = stalled_tasks(&queue, &home, now);
10556 assert_eq!(
10557 stalled.len(),
10558 1,
10559 "reclaim's inability to claim it yet must not hide it from the conductor"
10560 );
10561 assert_eq!(stalled[0].id, t.id);
10562 }
10563
10564 #[test]
10565 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10566 let dir = tempfile::tempdir().unwrap();
10567 crate::run::set_home(dir.path().join("run-home"));
10568 let queue = Queue::at(dir.path().join("queue"));
10569 let home = dir.path().join("home");
10570 let questions = Questions::at(dir.path().join("questions"));
10571
10572 let mut t = task();
10573 t.id = "20260101-000003-dead".to_owned();
10574 t.start("missing-run".to_owned());
10575 queue.put(&mut t).unwrap();
10576 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10577
10578 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10581 assert_eq!(
10582 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10583 [&t.id]
10584 );
10585 assert_eq!(
10586 reclaim_orphaned_running(&queue, 2, &Default::default()),
10587 [t.id.clone()]
10588 );
10589 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10590
10591 crate::conduct::apply(
10594 &queue,
10595 &questions,
10596 &crate::conduct::Verdict {
10597 decisions: vec![crate::conduct::Decision {
10598 id: t.id.clone(),
10599 recovery: Some(crate::conduct::Recovery::Requeue),
10600 ..crate::conduct::Decision::default()
10601 }],
10602 },
10603 )
10604 .unwrap();
10605 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10606 }
10607
10608 #[test]
10609 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10610 let dir = tempfile::tempdir().unwrap();
10611 let queue = Queue::at(dir.path().join("queue"));
10612 let home = dir.path().join("home");
10613
10614 let mut fresh = task();
10615 fresh.id = "20260101-000001-aaaa".to_owned();
10616 fresh.start("run-1".to_owned());
10617 queue.put(&mut fresh).unwrap();
10618
10619 let mut old = task();
10620 old.id = "20260101-000002-bbbb".to_owned();
10621 old.start("run-2".to_owned());
10622 queue.put(&mut old).unwrap();
10623 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10624
10625 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10626 assert_eq!(stalled.len(), 1);
10627 assert_eq!(stalled[0].id, old.id);
10628 }
10629
10630 #[test]
10631 fn queued_and_finished_task_views_partition_by_status() {
10632 let dir = tempfile::tempdir().unwrap();
10633 let queue = Queue::at(dir.path().join("queue"));
10634
10635 let mut queued = task();
10636 queued.id = "20260101-000001-aaaa".to_owned();
10637 queue.put(&mut queued).unwrap();
10638
10639 let mut failed = task();
10640 failed.id = "20260101-000002-bbbb".to_owned();
10641 failed.start("run-1".to_owned());
10642 failed.fail("gate red", 5);
10643 queue.put(&mut failed).unwrap();
10644
10645 let mut held = task();
10646 held.id = "20260101-000003-cccc".to_owned();
10647 held.hold_machine(None);
10648 queue.put(&mut held).unwrap();
10649
10650 let mut running = task();
10651 running.id = "20260101-000004-dddd".to_owned();
10652 running.start("run-2".to_owned());
10653 queue.put(&mut running).unwrap();
10654
10655 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
10656 assert_eq!(queued_ids, [queued.id.clone()]);
10657
10658 let mut finished_ids: Vec<String> =
10659 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
10660 finished_ids.sort_unstable();
10661 let mut want = vec![failed.id.clone(), held.id.clone()];
10662 want.sort_unstable();
10663 assert_eq!(finished_ids, want);
10664 }
10665
10666 #[test]
10667 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
10668 for short in [false, true] {
10669 let dir = tempfile::tempdir().unwrap();
10670 let queue = Queue::at(dir.path().join("queue"));
10671 let questions = ask::Questions::at(dir.path().join("questions"));
10672 let mut running = task();
10673 running.id = "20261013-192213-feb2".into();
10674 running.start("existing-run".into());
10675 queue.put(&mut running).unwrap();
10676 let mut middle = task();
10677 middle.id = "20261013-193611-ca5b".into();
10678 middle.block(
10679 vec![if short {
10680 "feb2".into()
10681 } else {
10682 running.id.clone()
10683 }],
10684 None,
10685 );
10686 queue.put(&mut middle).unwrap();
10687 let mut waiting = task();
10688 waiting.id = "20261013-202849-ee37".into();
10689 waiting.block(
10690 vec![if short {
10691 "ca5b".into()
10692 } else {
10693 middle.id.clone()
10694 }],
10695 Some("wait for implementation".into()),
10696 );
10697 queue.put(&mut waiting).unwrap();
10698 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
10699 let inv = crate::blockers::Inventory::load(&queue, &questions);
10700 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
10701 assert!(inv.stuck().is_empty());
10702 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10703 assert!(report.quarantined.is_empty());
10704 assert!(report.asked.is_empty());
10705 resolve_blockers(&queue, &questions);
10706 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10707 assert!(questions.list().is_empty());
10708 running.succeed();
10711 queue.put(&mut running).unwrap();
10712 resolve_blockers(&queue, &questions);
10713 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
10714 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10715 middle = queue.get(&middle.id).unwrap();
10716 middle.succeed();
10717 queue.put(&mut middle).unwrap();
10718 resolve_blockers(&queue, &questions);
10719 let after = queue.get(&waiting.id).unwrap();
10720 assert_eq!(after.status, TaskStatus::Queued);
10721 assert!(after.blocked_by.is_empty());
10722 assert_eq!(after.instruction, waiting.instruction);
10723 }
10724 }
10725
10726 #[test]
10727 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
10728 for future_schema in [false, true] {
10729 let name = "20261013-193611-ca5b";
10730 let mut future = task();
10731 future.id = name.into();
10732 future.schema = crate::queue::SCHEMA + 1;
10733 let contents = if future_schema {
10734 serde_json::to_string(&future).unwrap()
10735 } else {
10736 "corrupt".into()
10737 };
10738 let dir = tempfile::tempdir().unwrap();
10739 let queue = Queue::at(dir.path().join("queue"));
10740 let questions = ask::Questions::at(dir.path().join("questions"));
10741 let mut waiting = task();
10742 waiting.id = "20261013-202849-ee37".into();
10743 waiting.block(vec!["ca5b".into()], None);
10744 queue.put(&mut waiting).unwrap();
10745 std::fs::write(queue.path_of(name), contents).unwrap();
10746 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
10747 assert!(
10748 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
10749 );
10750 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10751 assert!(report.asked.is_empty());
10752 resolve_blockers(&queue, &questions);
10753 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10754 assert!(questions.list().is_empty());
10755 }
10756 let dir = tempfile::tempdir().unwrap();
10757 let queue = Queue::at(dir.path().join("queue"));
10758 let questions = ask::Questions::at(dir.path().join("questions"));
10759 let mut waiting = task();
10760 waiting.id = "20261013-202849-ee37".into();
10761 waiting.block(vec!["ca5b".into()], None);
10762 queue.put(&mut waiting).unwrap();
10763 let mut done = task();
10764 done.id = "20261013-193611-ca5b".into();
10765 done.succeed();
10766 queue.put(&mut done).unwrap();
10767 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
10770 assert!(queue.get("ca5b").is_err());
10771 assert_eq!(
10772 crate::queue::dependency(&queue, &questions, "ca5b"),
10773 crate::queue::Dependency::Ambiguous
10774 );
10775 resolve_blockers(&queue, &questions);
10776 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
10777 assert!(report.asked.is_empty());
10778 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
10779 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
10780 }
10781
10782 #[test]
10783 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
10784 for held in [false, true] {
10785 for question in [false, true] {
10786 for reference in ["ca5b", "20261013-193611-ca5b"] {
10787 let dir = tempfile::tempdir().unwrap();
10788 let queue = Queue::at(dir.path().join("queue"));
10789 let questions = ask::Questions::at(dir.path().join("questions"));
10790 let mut prerequisite = task();
10791 prerequisite.id = "20261013-193611-ca5b".into();
10792 prerequisite.start("existing-run".into());
10793 queue.put(&mut prerequisite).unwrap();
10794 let mut waiting = task();
10795 waiting.id = "20261013-202849-ee37".into();
10796 if held {
10797 waiting.hold_machine(Some("waiting for separate consent".into()));
10798 }
10799 queue.put(&mut waiting).unwrap();
10800 let reason = "Deleting saved data requires separate owner consent";
10801 crate::conduct::apply(
10802 &queue,
10803 &questions,
10804 &crate::conduct::Verdict {
10805 decisions: vec![crate::conduct::Decision {
10806 id: waiting.id.clone(),
10807 blocked_by: vec![reference.into()],
10808 question: question.then(|| "May the saved data be deleted?".into()),
10809 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
10810 reason: Some(reason.into()),
10811 ..crate::conduct::Decision::default()
10812 }],
10813 },
10814 )
10815 .unwrap();
10816 let before = queue.get(&waiting.id).unwrap();
10817 if question {
10818 assert_eq!(before.status, TaskStatus::Blocked);
10819 let q = questions.get(&before.blocked_by[0]).unwrap();
10820 assert_eq!(q.summary, "May the saved data be deleted?");
10821 assert!(q.status.open());
10822 } else {
10823 assert_eq!(before.status, TaskStatus::Held);
10824 let recorded_reason = before.hold_reason.as_deref().unwrap();
10825 assert!(recorded_reason.starts_with(reason));
10826 if held {
10827 assert!(recorded_reason.contains("waiting for separate consent"));
10828 }
10829 }
10830 prerequisite.succeed();
10831 queue.put(&mut prerequisite).unwrap();
10832 resolve_blockers(&queue, &questions);
10833 let after = queue.get(&waiting.id).unwrap();
10834 assert_eq!(after.status, before.status);
10835 assert_eq!(after.blocked_by, before.blocked_by);
10836 assert_eq!(after.hold_reason, before.hold_reason);
10837 assert_eq!(after.instruction, waiting.instruction);
10838 if question {
10839 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
10840 }
10841 assert!(
10842 queue.next_runnable().is_none(),
10843 "dependency completion is not consent"
10844 );
10845 }
10846 }
10847 }
10848 }
10849
10850 #[test]
10851 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
10852 let dir = tempfile::tempdir().unwrap();
10853 let queue = Queue::at(dir.path().join("queue"));
10854 let questions = ask::Questions::at(dir.path().join("questions"));
10855
10856 let mut dep = task();
10857 dep.id = "20260101-000001-dep0".to_owned();
10858 dep.succeed();
10859 queue.put(&mut dep).unwrap();
10860
10861 let mut still_going = task();
10862 still_going.id = "20260101-000002-dep1".to_owned();
10863 queue.put(&mut still_going).unwrap();
10864
10865 let mut blocked = task();
10866 blocked.id = "20260101-000003-main".to_owned();
10867 blocked.block(
10868 vec![dep.id.clone(), still_going.id.clone()],
10869 Some("waits on both".to_owned()),
10870 );
10871 queue.put(&mut blocked).unwrap();
10872
10873 resolve_blockers(&queue, &questions);
10874
10875 let after = queue.get(&blocked.id).unwrap();
10876 assert_eq!(
10877 after.status,
10878 TaskStatus::Blocked,
10879 "one dependency is still outstanding"
10880 );
10881 assert_eq!(after.blocked_by, [still_going.id.clone()]);
10882 }
10883
10884 #[test]
10885 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
10886 let dir = tempfile::tempdir().unwrap();
10887 let queue = Queue::at(dir.path().join("queue"));
10888 let questions = ask::Questions::at(dir.path().join("questions"));
10889
10890 let mut q = crate::ask::Question::new(
10891 "20260101-000001-main".to_owned(),
10892 crate::conduct::NODE.to_owned(),
10893 "conduct".to_owned(),
10894 "Which backend?".to_owned(),
10895 String::new(),
10896 Vec::new(),
10897 );
10898 questions.put(&mut q).unwrap();
10899 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
10900 .unwrap();
10901 questions.put(&mut q).unwrap();
10902
10903 let mut blocked = task();
10904 blocked.id = "20260101-000001-main".to_owned();
10905 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
10906 queue.put(&mut blocked).unwrap();
10907
10908 resolve_blockers(&queue, &questions);
10909
10910 let after = queue.get(&blocked.id).unwrap();
10911 assert_eq!(
10912 after.status,
10913 TaskStatus::Queued,
10914 "the only blocker resolved"
10915 );
10916 assert_eq!(after.answers.len(), 1);
10917 assert_eq!(after.answers[0].question, "Which backend?");
10918 assert_eq!(after.answers[0].answer, "SQLite");
10919
10920 let instruction = instruction_for(&after);
10922 assert!(instruction.contains("Which backend?"));
10923 assert!(instruction.contains("SQLite"));
10924 }
10925
10926 #[test]
10927 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
10928 let dir = tempfile::tempdir().unwrap();
10929 let queue = Queue::at(dir.path().join("queue"));
10930 let questions = ask::Questions::at(dir.path().join("questions"));
10931
10932 let mut q = crate::ask::Question::new(
10933 "20260101-000001-main".to_owned(),
10934 crate::conduct::NODE.to_owned(),
10935 "conduct".to_owned(),
10936 "Is the setup done?".to_owned(),
10937 String::new(),
10938 Vec::new(),
10939 );
10940 q.abandon("no answer within 60s of asking");
10941 questions.put(&mut q).unwrap();
10942
10943 let mut blocked = task();
10944 blocked.id = "20260101-000001-main".to_owned();
10945 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
10946 queue.put(&mut blocked).unwrap();
10947
10948 resolve_blockers(&queue, &questions);
10949
10950 let after = queue.get(&blocked.id).unwrap();
10951 assert_eq!(
10952 after.status,
10953 TaskStatus::Held,
10954 "never left blocked on nothing"
10955 );
10956 assert!(!after.operator_held(), "a machine hold, for triage");
10957 assert!(
10958 after
10959 .hold_reason
10960 .as_deref()
10961 .unwrap_or_default()
10962 .contains("went unanswered")
10963 );
10964 }
10965
10966 #[test]
10967 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
10968 let dir = tempfile::tempdir().unwrap();
10974 let queue = Queue::at(dir.path().join("queue"));
10975 let questions = ask::Questions::at(dir.path().join("questions"));
10976
10977 let mut q = crate::ask::Question::new(
10978 "20260101-000001-main".to_owned(),
10979 crate::conduct::NODE.to_owned(),
10980 "conduct".to_owned(),
10981 "How should this be handled?".to_owned(),
10982 String::new(),
10983 Vec::new(),
10984 );
10985 questions.put(&mut q).unwrap();
10986 q.answer(crate::ask::Answer::Text(
10987 "leave it held, a human will look at it later".to_owned(),
10988 ))
10989 .unwrap();
10990 questions.put(&mut q).unwrap();
10991
10992 let mut held = task();
10993 held.id = "20260101-000001-main".to_owned();
10994 held.hold_machine(Some("out of attempts".to_owned()));
10995 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
10996 queue.put(&mut held).unwrap();
10997
10998 resolve_blockers(&queue, &questions);
10999
11000 let after = queue.get(&held.id).unwrap();
11001 assert_eq!(after.status, TaskStatus::Held);
11002 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
11003 assert_eq!(
11004 after.answers[0].answer,
11005 "leave it held, a human will look at it later"
11006 );
11007 }
11008
11009 #[test]
11010 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
11011 let dir = tempfile::tempdir().unwrap();
11012 let queue = Queue::at(dir.path().join("queue"));
11013 let questions = ask::Questions::at(dir.path().join("questions"));
11014 let mut dep = task();
11015 dep.id = "20260101-000001-gone".to_owned();
11016 queue.put(&mut dep).unwrap();
11017 let mut blocked = task();
11018 blocked.id = "20260101-000003-main".to_owned();
11019 blocked.block(vec![dep.id.clone()], None);
11020 queue.put(&mut blocked).unwrap();
11021
11022 let claim = queue.claim(&blocked.id).unwrap();
11024 queue.remove(&dep.id, false, &questions).unwrap();
11025 drop(claim);
11026 resolve_blockers(&queue, &questions);
11027
11028 let after = queue.get(&blocked.id).unwrap();
11029 assert_eq!(after.status, TaskStatus::Queued);
11030 assert!(after.blocked_by.is_empty());
11031 }
11032
11033 #[test]
11034 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
11035 let dir = tempfile::tempdir().unwrap();
11041 let queue = Queue::at(dir.path().join("queue"));
11042 let questions = ask::Questions::at(dir.path().join("questions"));
11043
11044 let mut still_going = task();
11045 still_going.id = "20260101-000002-dep1".to_owned();
11046 queue.put(&mut still_going).unwrap();
11047
11048 let mut blocked = task();
11049 blocked.id = "20260101-000003-main".to_owned();
11050 blocked.block(
11051 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
11052 Some("waits on both".to_owned()),
11053 );
11054 queue.put(&mut blocked).unwrap();
11055
11056 resolve_blockers(&queue, &questions);
11057
11058 let after = queue.get(&blocked.id).unwrap();
11059 assert_eq!(
11060 after.status,
11061 TaskStatus::Held,
11062 "a missing dependency must not leave the task blocked forever"
11063 );
11064 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
11065 assert!(after.blocked_by.is_empty());
11066 let reason = after.hold_reason.as_deref().unwrap_or_default();
11067 assert!(
11068 reason.contains("20260101-000001-gone"),
11069 "the missing id must be named so an operator can tell what happened: {reason}"
11070 );
11071 assert!(
11072 reason.contains(&still_going.id),
11073 "the still-valid dependency must not silently vanish from the record: {reason}"
11074 );
11075 }
11076
11077 #[test]
11078 fn instruction_for_is_unchanged_without_any_answers() {
11079 let t = task();
11080 assert_eq!(instruction_for(&t), t.instruction);
11081 }
11082
11083 #[test]
11084 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
11085 let dir = tempfile::tempdir().unwrap();
11086 let q = Queue::at(dir.path().join("queue"));
11087 let src = dir.path().join("shot.png");
11088 std::fs::write(&src, "x").unwrap();
11089 let mut t = task();
11090 q.attach(&mut t, &[src]).unwrap();
11091 let paths = task_attachments(&q, &t).unwrap();
11092 assert_eq!(paths.len(), 1);
11093 assert!(paths[0].is_absolute() && paths[0].is_file());
11094 std::fs::remove_file(&paths[0]).unwrap();
11095 let err = task_attachments(&q, &t).unwrap_err().to_string();
11096 assert!(err.contains("shot.png"), "{err}");
11097 }
11098
11099 #[tokio::test]
11100 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
11101 let dir = tempfile::tempdir().unwrap();
11102 let queue_path = dir.path().join("queue");
11103 let queue = Queue::at(queue_path.clone());
11104 let mut t = task();
11105 t.source = Source::Agent {
11106 run: "originating-chat".into(),
11107 node: crate::queue::CHAT_NODE.into(),
11108 };
11109 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
11110 stage: "config-fetch",
11111 target: "origin/main".into(),
11112 });
11113 record_config_failure(&mut t, &fetch);
11114 t.hold_machine(Some(format!("{fetch:#}")));
11115 let fetch_key = t.chat_notice_due().unwrap();
11116 t.chat_notices.push(fetch_key.clone());
11117 queue.put(&mut t).unwrap();
11118
11119 let queue = Queue::at(queue_path);
11120 let mut t = queue.get(&t.id).unwrap();
11121 let a = dir.path().join("a.toml");
11122 let b = dir.path().join("b.toml");
11123 for (path, text, new_cause) in [
11124 (&a, "[graph", true),
11125 (&a, "[verify", false),
11126 (&a, "[graph]\nimplementers = 'invalid-type'", true),
11127 (&b, "[graph]\nimplementers = 'invalid-type'", true),
11128 ] {
11129 t.release_operator("test", None);
11130 std::fs::write(path, text).unwrap();
11131 let error = Config::discover_fetched(dir.path(), Some(path))
11132 .await
11133 .unwrap_err();
11134 record_config_failure(&mut t, &error);
11135 t.hold_machine(Some(format!("{error:#}")));
11136 let due = t.chat_notice_due();
11137 assert_eq!(due.is_some(), new_cause, "{error:#}");
11138 if let Some(key) = due {
11139 assert_ne!(key, fetch_key);
11140 t.chat_notices.push(key);
11141 }
11142 }
11143 t.succeed();
11144 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11145 }
11146
11147 #[test]
11148 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11149 let dir = tempfile::tempdir().unwrap();
11150 let queue_path = dir.path().join("queue");
11151 let queue = Queue::at(queue_path.clone());
11152 let mut t = task();
11153 t.source = crate::queue::Source::Agent {
11154 run: "originating-chat".into(),
11155 node: crate::queue::CHAT_NODE.into(),
11156 };
11157 let a = dir.path().join("a.png");
11158 let b = dir.path().join("b.png");
11159 std::fs::write(&a, "a").unwrap();
11160 std::fs::write(&b, "b").unwrap();
11161 queue.attach(&mut t, &[a, b]).unwrap();
11162 let paths = task_attachments(&queue, &t).unwrap();
11163 for path in &paths {
11164 std::fs::remove_file(path).unwrap();
11165 }
11166 let missing = task_attachments(&queue, &t).unwrap_err();
11167 t.attempts += 1;
11168 let reason = missing.to_string();
11169 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11170 let first = t.chat_notice_due().expect("first attachment notice");
11171 t.chat_notices.push(first.clone());
11172 queue.put(&mut t).unwrap();
11173
11174 let queue = Queue::at(queue_path.clone());
11175 let mut t = queue.get(&t.id).unwrap();
11176 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11177 t.release_operator("test", None);
11178 t.attempts += 1;
11179 let missing = task_attachments(&queue, &t).unwrap_err();
11180 t.fail_startup_target(
11181 "startup-attachments",
11182 missing.name,
11183 "same missing attachment with regenerated wording",
11184 1,
11185 );
11186 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11187
11188 std::fs::write(&paths[0], "restored").unwrap();
11189 t.release_operator("test", None);
11190 t.attempts += 1;
11191 let missing = task_attachments(&queue, &t).unwrap_err();
11192 let reason = missing.to_string();
11193 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11194 let second = t
11195 .chat_notice_due()
11196 .expect("different attachment must notify");
11197 assert_ne!(first, second);
11198 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11199 }
11200
11201 #[test]
11202 fn resumed_instruction_is_unchanged_without_any_answers() {
11203 let t = task();
11204 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11205 }
11206
11207 #[test]
11208 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11209 let mut t = task();
11210 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11211 let old = t.instruction.clone();
11215
11216 let refreshed = resumed_instruction(&old, &t);
11217 assert!(refreshed.starts_with(&old), "the original text is kept");
11218 assert!(refreshed.contains("Which backend?"));
11219 assert!(refreshed.contains("SQLite"));
11220 }
11221
11222 #[test]
11223 fn resumed_instruction_keeps_an_original_answers_heading() {
11224 let mut t = task();
11225 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11226 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11227
11228 let refreshed = resumed_instruction(&t.instruction, &t);
11229
11230 assert!(
11231 refreshed.starts_with(&t.instruction),
11232 "an answers heading in the original instruction is not the appended block"
11233 );
11234 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11235 assert!(refreshed.contains("Which backend?"));
11236 assert!(refreshed.contains("SQLite"));
11237
11238 let repeated = resumed_instruction(&refreshed, &t);
11239 assert_eq!(
11240 repeated, refreshed,
11241 "only the final appended block is refreshed"
11242 );
11243 }
11244
11245 #[test]
11246 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11247 let mut t = task();
11248 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11249
11250 let once = resumed_instruction(&t.instruction, &t);
11254 let twice = resumed_instruction(&once, &t);
11255 assert_eq!(once, twice);
11256 assert_eq!(once.matches("Which backend?").count(), 1);
11257
11258 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
11260 let refreshed = resumed_instruction(&once, &t);
11261 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
11262 assert!(refreshed.contains("Which backend?"));
11263 assert!(refreshed.contains("Which cache?"));
11264 }
11265
11266 #[test]
11267 fn prepare_instruction_covers_all_three_starters() {
11268 let mut t = task();
11269 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11270
11271 assert_eq!(
11274 prepare_instruction(&Starter::Start, None, &t),
11275 Some(instruction_for(&t))
11276 );
11277
11278 let old = t.instruction.clone();
11281 assert_eq!(
11282 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
11283 Some(resumed_instruction(&old, &t))
11284 );
11285
11286 assert_eq!(
11290 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
11291 None
11292 );
11293 }
11294
11295 #[test]
11296 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
11297 assert_eq!(
11298 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
11299 Starter::Review("magi/eba2/A".to_owned())
11300 );
11301 }
11302
11303 #[test]
11304 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
11305 assert_eq!(
11306 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
11307 Starter::Start,
11308 "a vanished review branch must not fall back to resuming the old run either"
11309 );
11310 }
11311
11312 #[test]
11313 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11314 let mut t = task();
11315 t.start("old-run".to_owned());
11316 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11317 t.release();
11318 let branch = t.review_branch.take();
11319 assert_eq!(
11320 choose_starter(branch.as_deref(), true, Some("old-run")),
11321 Starter::Review("magi/eba2/A".to_owned()),
11322 "a review wins over resuming the old run"
11323 );
11324 }
11325
11326 #[test]
11327 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11328 assert_eq!(
11329 choose_starter(None, false, Some("some-run")),
11330 Starter::Resume("some-run".to_owned())
11331 );
11332 assert_eq!(choose_starter(None, false, None), Starter::Start);
11333 }
11334
11335 #[test]
11336 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11337 let mut released = task();
11338 released.start("stalled-run".to_owned());
11339 released.requeue();
11340 let unfinished = (!released.fresh_start)
11341 .then(|| Some("stalled-run".to_owned()))
11342 .flatten();
11343 assert_eq!(
11344 choose_starter(None, false, unfinished.as_deref()),
11345 Starter::Start,
11346 "release keeps run history but must not resume it"
11347 );
11348 assert_eq!(released.runs, ["stalled-run"]);
11349 }
11350
11351 fn stopped_public_run(status: RunStatus) -> RunState {
11352 let mut state = run_state(status);
11353 state.driver_exited = true;
11354 state.config.graph.land = true;
11355 state.config.graph.review_rounds = 1;
11356 state.reviews = vec![review_round(1)];
11357 state.merge = Some(crate::run::MergeOutcome {
11358 mode: MergeMode::Pr,
11359 ok: false,
11360 detail: "landing stopped".into(),
11361 empty: false,
11362 });
11363 state.pr = Some(crate::run::PrRecord {
11364 url: "https://example.test/pull/1".into(),
11365 number: 1,
11366 state: "open".into(),
11367 checks: "red".into(),
11368 round: 1,
11369 rounds: 1,
11370 red_at_merge: Vec::new(),
11371 });
11372 state
11373 }
11374
11375 #[test]
11376 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11377 for status in [RunStatus::Blocked, RunStatus::Gating] {
11378 let home = tempfile::tempdir().unwrap();
11379 let mut state = stopped_public_run(status);
11380 state.save_under(home.path()).unwrap();
11381 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11382 assert!(!exhausted_review_budget(&state));
11383 let mut task = held_task_with(&state.id);
11384 task.release();
11385 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11386 assert_eq!(
11387 choose_starter(None, false, unfinished.as_deref()),
11388 Starter::Resume(state.id.clone())
11389 );
11390 let question = action_question(&state.id, resume_action(&state.id));
11391 assert_eq!(
11392 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11393 ActionDecision::Resume(state.id.clone())
11394 );
11395 assert_eq!(
11397 crate::resume::validate(home.path(), &state.id).unwrap().id,
11398 state.id
11399 );
11400 let queue = Queue::at(home.path().join("queue"));
11401 let mut controlled = Task::new(
11402 "resume land".into(),
11403 state.instruction.clone(),
11404 state.repo.clone(),
11405 Source::Agent {
11406 run: "talk".into(),
11407 node: crate::queue::CHAT_NODE.into(),
11408 },
11409 );
11410 controlled.runs.push(state.id.clone());
11411 controlled.hold_machine(Some("land stopped".into()));
11412 queue.put(&mut controlled).unwrap();
11413 let resumed =
11414 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11415 assert_eq!(resumed.status, TaskStatus::Queued);
11416 assert_eq!(
11417 resumed.resume_override.unwrap().pinned_run,
11418 Some(state.id.clone())
11419 );
11420 assert_eq!(resumed.attempts, 0);
11421 }
11422 }
11423
11424 #[test]
11425 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11426 for review_rounds in [0, 1] {
11427 let home = tempfile::tempdir().unwrap();
11428 let mut state = run_state(RunStatus::Blocked);
11429 state.config.graph.review_rounds = review_rounds;
11430 if review_rounds > 0 {
11431 let mut review = review_round(1);
11432 review.clean = true;
11433 state.reviews = vec![review];
11434 }
11435 state.gate_ran = true;
11436 state.gate = vec![crate::run::CommandOutcome {
11437 command: "verify".into(),
11438 code: Some(1),
11439 output_tail: "gate failed after repair budget exhausted".into(),
11440 duration_ms: 0,
11441 resource_blocked: false,
11442 }];
11443 state.save_under(home.path()).unwrap();
11444 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11445 assert!(exhausted_review_budget(&state));
11446 let mut task = held_task_with(&state.id);
11447 task.release();
11448 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11449 let question = action_question(&state.id, resume_action(&state.id));
11450 assert_ne!(
11451 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11452 ActionDecision::Resume(state.id.clone())
11453 );
11454 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11455
11456 state.gate_ran = false;
11458 state.gate.clear();
11459 assert!(!exhausted_review_budget(&state));
11460 assert_eq!(
11461 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11462 Some(state.id)
11463 );
11464 }
11465 }
11466
11467 #[test]
11468 fn resource_blocked_verification_resumes_the_same_run() {
11469 for (kind, stage) in [
11470 ("cache", "clean"),
11471 ("coordination", "clean"),
11472 ("cache", "review-e2e"),
11473 ("coordination", "no-reviews"),
11474 ("coordination", "handoff"),
11475 ] {
11476 let home = tempfile::tempdir().unwrap();
11477 let mut state = run_state(RunStatus::Blocked);
11478 state.config.graph.review_rounds = 1;
11479 let mut review = review_round(1);
11480 review.clean = true;
11481 state.reviews = vec![review];
11482 if stage == "review-e2e" {
11483 let review = &mut state.reviews[0];
11484 review.clean = false;
11485 review.blocking = 1;
11486 review.e2e = vec![crate::run::CommandOutcome {
11487 command: "verify".into(),
11488 code: None,
11489 output_tail: "resource wait expired".into(),
11490 duration_ms: 0,
11491 resource_blocked: true,
11492 }];
11493 } else if stage == "no-reviews" {
11494 state.config.graph.review_rounds = 0;
11495 state.reviews.clear();
11496 } else if stage == "handoff" {
11497 let review = &mut state.reviews[0];
11500 review.clean = false;
11501 review.blocking = 1;
11502 review.e2e = vec![crate::run::CommandOutcome {
11503 command: "verify".into(),
11504 code: Some(0),
11505 output_tail: String::new(),
11506 duration_ms: 0,
11507 resource_blocked: false,
11508 }];
11509 }
11510 state.gate_ran = false;
11511 state.resource_block = Some(crate::run::ResourceBlock {
11512 cause: crate::run::ResourceBlockCause::Other,
11513 node: if stage == "review-e2e" {
11514 "review"
11515 } else {
11516 "gate"
11517 }
11518 .into(),
11519 kind: kind.into(),
11520 detail: "resource wait expired".into(),
11521 at: jiff::Timestamp::now(),
11522 });
11523 state.save_under(home.path()).unwrap();
11524 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11525 assert!(!exhausted_review_budget(&state));
11526 let mut task = held_task_with(&state.id);
11527 task.release();
11528 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11529 assert_eq!(
11530 choose_starter(None, false, unfinished.as_deref()),
11531 Starter::Resume(state.id.clone())
11532 );
11533 let question = action_question(&state.id, resume_action(&state.id));
11534 assert_eq!(
11535 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11536 ActionDecision::Resume(state.id.clone())
11537 );
11538 assert_eq!(
11539 crate::resume::validate(home.path(), &state.id).unwrap().id,
11540 state.id
11541 );
11542 let queue = Queue::at(home.path().join("queue"));
11543 let mut controlled = Task::new(
11544 "resume verification".into(),
11545 state.instruction.clone(),
11546 state.repo.clone(),
11547 Source::Agent {
11548 run: "talk".into(),
11549 node: crate::queue::CHAT_NODE.into(),
11550 },
11551 );
11552 controlled.runs.push(state.id.clone());
11553 controlled.hold_machine(Some("verification resource unavailable".into()));
11554 queue.put(&mut controlled).unwrap();
11555 let resumed =
11556 crate::chat_control::resume(&queue, home.path(), "talk", &state.id).unwrap();
11557 assert_eq!(resumed.status, TaskStatus::Queued);
11558 assert_eq!(
11559 resumed.resume_override.unwrap().pinned_run,
11560 Some(state.id.clone())
11561 );
11562 assert_eq!(resumed.attempts, 0);
11563
11564 state.config.graph.review_rounds = 1;
11566 let mut unresolved = review_round(1);
11567 unresolved.blocking = 1;
11568 state.reviews = vec![unresolved];
11569 state.resource_block = None;
11570 state.save_under(home.path()).unwrap();
11571 assert!(exhausted_review_budget(&state));
11572 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11573 }
11574 }
11575
11576 #[tokio::test]
11577 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11578 if !crate::standalone_test(
11579 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
11580 ) {
11581 return;
11582 }
11583 let home = tempfile::tempdir().unwrap();
11584 let mut owner = stopped_public_run(RunStatus::Blocked);
11585 owner.save_under(home.path()).unwrap();
11586 let history = vec![owner.id.clone()];
11587 let repo = owner.repo.clone();
11588 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11589 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11590 let initial =
11591 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11592 assert!(
11593 initial.is_pending(),
11594 "a busy record must not hold or resume the task"
11595 );
11596 owner.retired_to = Some("20260101-000001-bbbb".into());
11597 owner.save_under(home.path()).unwrap();
11598 drop(lock);
11599 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11600 .await
11601 .expect("ownership lock released")
11602 .expect("read settled owner");
11603 assert!(
11604 result.is_none(),
11605 "the retired owner must not be resumed from a stale read"
11606 );
11607 }
11608
11609 #[tokio::test]
11610 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11611 if !crate::standalone_test(
11612 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
11613 ) {
11614 return;
11615 }
11616 let home = tempfile::tempdir().unwrap();
11617 let mut owner = stopped_public_run(RunStatus::Blocked);
11618 let repo = tempfile::tempdir().unwrap();
11619 crate::git::git(repo.path(), &["init"]).await.unwrap();
11620 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11621 owner.id = "20260101-000000-aaaa".into();
11622 owner.save_under(home.path()).unwrap();
11623 let mut successor = run_state(RunStatus::Blocked);
11624 successor.id = "20260101-000001-bbbb".into();
11625 successor.save_under(home.path()).unwrap();
11626 let runs = vec![owner.id.clone(), successor.id.clone()];
11627 assert_eq!(
11628 public_resume_owner(home.path(), &runs, &owner.repo)
11629 .await
11630 .unwrap()
11631 .unwrap()
11632 .id,
11633 owner.id
11634 );
11635 let subdir = owner.repo.join("nested");
11636 std::fs::create_dir(&subdir).unwrap();
11637 assert_eq!(
11638 public_resume_owner(home.path(), &runs, &subdir)
11639 .await
11640 .unwrap()
11641 .unwrap()
11642 .id,
11643 owner.id
11644 );
11645 #[cfg(unix)]
11646 {
11647 let alias = home.path().join("repo-alias");
11648 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
11649 assert_eq!(
11650 public_resume_owner(home.path(), &runs, &alias.join("nested"))
11651 .await
11652 .unwrap()
11653 .unwrap()
11654 .id,
11655 owner.id
11656 );
11657 }
11658 let different = tempfile::tempdir().unwrap();
11659 crate::git::git(different.path(), &["init"]).await.unwrap();
11660 assert!(
11661 public_resume_owner(home.path(), &runs, different.path())
11662 .await
11663 .is_err()
11664 );
11665 std::fs::remove_file(
11666 home.path()
11667 .join("runs")
11668 .join(&successor.id)
11669 .join("run.json"),
11670 )
11671 .unwrap();
11672 assert!(
11673 public_resume_owner(home.path(), &runs, &owner.repo)
11674 .await
11675 .is_err()
11676 );
11677 }
11678
11679 #[test]
11680 fn an_ordinary_release_keeps_a_resumable_run_available() {
11681 let mut released = task();
11682 released.start("stalled-run".to_owned());
11683 released.release();
11684 let unfinished = (!released.fresh_start)
11685 .then(|| Some("stalled-run".to_owned()))
11686 .flatten();
11687 assert_eq!(
11688 choose_starter(None, false, unfinished.as_deref()),
11689 Starter::Resume("stalled-run".to_owned()),
11690 "manual release must preserve the normal resume path"
11691 );
11692 }
11693
11694 #[test]
11695 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
11696 let mut state = run_state(RunStatus::Blocked);
11697 state.config.graph.review_rounds = 3;
11698 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
11699 assert!(exhausted_review_budget(&state));
11700
11701 state.reviews.pop();
11703 assert!(!exhausted_review_budget(&state));
11704
11705 let mut stalled = run_state(RunStatus::Stalled);
11708 stalled.config.graph.review_rounds = 1;
11709 stalled.reviews = vec![review_round(1)];
11710 assert!(!exhausted_review_budget(&stalled));
11711 }
11712
11713 fn review_round(round: usize) -> crate::run::ReviewRound {
11714 crate::run::ReviewRound {
11715 round,
11716 head: "deadbeef".to_owned(),
11717 verified_head: None,
11718 verified_at: None,
11719 reviews: Vec::new(),
11720 e2e: Vec::new(),
11721 verify_retried: false,
11722 e2e_deferred: false,
11723 e2e_defer_reason: None,
11724 fix: None,
11725 blocking: 0,
11726 answered: 1,
11727 expected: 1,
11728 clean: false,
11729 progressed: true,
11730 vote_split: false,
11731 reconsideration: Vec::new(),
11732 verdict: None,
11733 }
11734 }
11735
11736 #[test]
11737 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
11738 let mut run = RunState::new(
11739 PathBuf::from("/repo"),
11740 "main".to_owned(),
11741 "abc1234def".to_owned(),
11742 "add retries".to_owned(),
11743 Config::default(),
11744 );
11745 run.status = RunStatus::Judging;
11746 run.parked = true;
11747 let mut task = Task::new(
11748 "add retries".to_owned(),
11749 "add retries".to_owned(),
11750 PathBuf::from("/repo"),
11751 crate::queue::Source::Human,
11752 );
11753 task.status = TaskStatus::Failed;
11754 task.runs = vec![run.id.clone()];
11755 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
11756 assert!(with(&task, &run), "parked after judging is the case");
11757 let mut parked_task = task.clone();
11758 parked_task.status = TaskStatus::Parked;
11759 assert!(with(&parked_task, &run), "the Parked status is the case");
11760
11761 let mut not_parked = run.clone();
11762 not_parked.parked = false;
11763 not_parked.status = RunStatus::Stalled;
11764 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
11765
11766 let mut fresh = task.clone();
11767 fresh.fresh_start = true;
11768 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
11769
11770 let mut review = task.clone();
11771 review.review_branch = Some("magi/x/A".to_owned());
11772 assert!(!with(&review, &run), "review is ranked before resume");
11773
11774 let mut held = task.clone();
11775 held.status = TaskStatus::Held;
11776 assert!(!with(&held, &run), "a hold stays visible to the conductor");
11777
11778 let mut released = run.clone();
11779 released.released_to = Some("20260901-000000-new1".to_owned());
11780 assert!(!with(&task, &released), "nothing left to resume into");
11781
11782 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
11783 "unreadable"
11784 )));
11785 }
11786
11787 #[test]
11788 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
11789 let mut released = RunState::new(
11790 PathBuf::from("/repo"),
11791 "main".to_owned(),
11792 "abc1234def".to_owned(),
11793 "add retries".to_owned(),
11794 Config::default(),
11795 );
11796 released.status = RunStatus::Blocked;
11797 assert_eq!(
11798 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11799 Some(released.id.clone())
11800 );
11801 released.released_to = Some("20260901-000000-new1".to_owned());
11802 assert_eq!(
11803 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
11804 None,
11805 "there is nothing left to resume it into"
11806 );
11807 }
11808
11809 #[test]
11810 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
11811 let mut exhausted = RunState::new(
11820 PathBuf::from("/repo"),
11821 "main".to_owned(),
11822 "abc1234def".to_owned(),
11823 "add retries".to_owned(),
11824 Config::default(),
11825 );
11826 exhausted.status = RunStatus::Blocked;
11827 exhausted.config.graph.review_rounds = 1;
11828 exhausted.reviews = vec![review_round(1)];
11829
11830 assert_eq!(
11831 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
11832 None,
11833 "an exhausted `Blocked` run must not be offered as resumable"
11834 );
11835
11836 let mut has_budget_left = RunState::new(
11839 PathBuf::from("/repo"),
11840 "main".to_owned(),
11841 "abc1234def".to_owned(),
11842 "add retries".to_owned(),
11843 Config::default(),
11844 );
11845 has_budget_left.status = RunStatus::Blocked;
11846 has_budget_left.config.graph.review_rounds = 3;
11847 has_budget_left.reviews = vec![review_round(1)];
11848
11849 assert_eq!(
11850 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
11851 Ok(has_budget_left.clone())
11852 }),
11853 Some(has_budget_left.id.clone())
11854 );
11855 }
11856
11857 #[test]
11858 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
11859 let mut run = RunState::new(
11860 PathBuf::from("/repo"),
11861 "main".to_owned(),
11862 "abc1234def".to_owned(),
11863 "add retries".to_owned(),
11864 Config::default(),
11865 );
11866 run.status = RunStatus::Blocked;
11867 run.config.graph.review_rounds = 1;
11868 run.reviews = vec![review_round(1)];
11869 run.pr_publication_pending = true;
11870 assert_eq!(
11871 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
11872 Some(run.id.clone()),
11873 "a spent review budget must not turn a failed publication into a re-competition"
11874 );
11875 }
11876
11877 #[test]
11878 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
11879 let mut task = Task::new(
11880 "recover work".into(),
11881 "recover work".into(),
11882 PathBuf::from("/repo"),
11883 crate::queue::Source::Human,
11884 );
11885 let mut history = std::collections::BTreeMap::new();
11886 for index in 0..8 {
11887 let mut run = run_state(RunStatus::Blocked);
11888 run.id = format!("20260101-000000-{index:04}");
11889 run.base_commit = format!("upstream-{index}");
11890 if index > 0 {
11892 run.candidates.clear();
11893 }
11894 task.runs.extend([run.id.clone(), run.id.clone()]);
11895 history.insert(run.id.clone(), run);
11896 task.requeue();
11897 }
11898 let check = |task: &Task, starter: &Starter, limit| {
11899 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
11900 };
11901 assert!(
11902 check(&task, &Starter::Start, 8)
11903 .unwrap()
11904 .contains("8 unretired")
11905 );
11906 assert!(check(&task, &Starter::Start, 9).is_none());
11907 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
11908 task.release_operator("owner", None);
11909 assert_eq!(task.attempts, 0);
11910 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
11911 assert!(check(&restored, &Starter::Start, 8).is_some());
11912 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
11913 assert!(check(&restored, &Starter::Start, 0).is_none());
11914 }
11915
11916 #[test]
11917 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
11918 let mut task = Task::new(
11919 "recover work".into(),
11920 "recover work".into(),
11921 PathBuf::from("/repo"),
11922 crate::queue::Source::Human,
11923 );
11924 let mut run = run_state(RunStatus::Superseded);
11925 run.candidates.clear();
11926 task.runs.push(run.id.clone());
11927 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
11928 run.retired_to = Some("authority".into());
11929 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11930 run.retired_to = None;
11931 run.released_to = Some("authority".into());
11932 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
11933 assert!(
11934 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
11935 .unwrap()
11936 .contains("Cannot verify")
11937 );
11938 assert!(
11940 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
11941 panic!("resume must not load historical attempts")
11942 })
11943 .is_none()
11944 );
11945 }
11946
11947 #[test]
11948 fn retired_resume_requires_a_valid_same_repository_authority() {
11949 let mut source = run_state(RunStatus::Blocked);
11950 let mut empty = run_state(RunStatus::Blocked);
11951 empty.retired_to = Some(source.id.clone());
11952 let runs = vec![source.id.clone(), empty.id.clone()];
11953 let resolve = |source: &RunState, empty: &RunState| {
11954 unfinished_run_with(&runs, "fixture", |id| {
11955 if id == source.id {
11956 Ok(source.clone())
11957 } else if id == empty.id {
11958 Ok(empty.clone())
11959 } else {
11960 bail!("unreadable authority")
11961 }
11962 })
11963 };
11964 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
11965 assert_eq!(
11966 choose_starter(None, false, resolve(&source, &empty).as_deref()),
11967 Starter::Resume(source.id.clone())
11968 );
11969 source.retired_to = Some(empty.id.clone());
11970 assert!(
11971 resolve(&source, &empty).is_none(),
11972 "cycles do not revive obsolete runs"
11973 );
11974 source.retired_to = None;
11975 source.repo = PathBuf::from("/another-repository");
11976 assert!(resolve(&source, &empty).is_none());
11977 source.repo = empty.repo.clone();
11978 source.released_to = Some("later".into());
11979 assert!(resolve(&source, &empty).is_none());
11980 empty.retired_to = Some("unlisted-run".into());
11981 assert!(resolve(&source, &empty).is_none());
11982 }
11983
11984 #[test]
11985 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
11986 let mut older_stalled = RunState::new(
11994 PathBuf::from("/repo"),
11995 "main".to_owned(),
11996 "abc1234def".to_owned(),
11997 "add retries".to_owned(),
11998 Config::default(),
11999 );
12000 older_stalled.status = RunStatus::Stalled;
12001
12002 let mut newest_exhausted = RunState::new(
12003 PathBuf::from("/repo"),
12004 "main".to_owned(),
12005 "abc1234def".to_owned(),
12006 "add retries".to_owned(),
12007 Config::default(),
12008 );
12009 newest_exhausted.status = RunStatus::Blocked;
12010 newest_exhausted.config.graph.review_rounds = 1;
12011 newest_exhausted.reviews = vec![review_round(1)];
12012
12013 assert_eq!(
12014 unfinished_run_with(
12015 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
12016 "t",
12017 |_| Ok(newest_exhausted.clone())
12018 ),
12019 None,
12020 "the newest run is exhausted, so nothing here is worth resuming - \
12021 least of all the older, already-superseded run"
12022 );
12023 }
12024
12025 #[test]
12026 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
12027 assert_eq!(
12028 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
12029 Err(anyhow::anyhow!("fixture is absent"))
12030 }),
12031 None
12032 );
12033 }
12034
12035 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
12036 let mut q = ask::Question::new(
12037 run.to_owned(),
12038 "implement".to_owned(),
12039 "impl-A".to_owned(),
12040 "continue?".to_owned(),
12041 String::new(),
12042 vec!["resume で続行する".to_owned(), "other".to_owned()],
12043 );
12044 q.actions.insert("resume で続行する".to_owned(), action);
12045 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
12046 .unwrap();
12047 q
12048 }
12049
12050 fn held_task_with(run: &str) -> Task {
12051 let mut t = task();
12052 t.runs = vec![run.to_owned()];
12053 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
12054 t
12055 }
12056
12057 fn resume_action(run: &str) -> ask::ChoiceAction {
12058 ask::ChoiceAction::Resume { run: run.into() }
12059 }
12060
12061 #[test]
12062 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
12063 for parked in [false, true] {
12064 let mut t = task();
12065 t.start("A".into());
12066 t.start("B".into());
12067 assert_eq!(t.runs, vec!["A", "B"]);
12068 t.start("A".into());
12069 let mut resumed = run_state(RunStatus::Blocked);
12070 resumed.id = "A".into();
12071 resumed.parked = parked;
12072 resumed.driver_exited = true;
12073 t.hold_machine(Some("resumed run stopped".into()));
12074 let q = action_question("A", resume_action("A"));
12075 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12076 assert_eq!(
12077 decide_action(&t, &q, &PHRASES_EN, |id| {
12078 assert_eq!(id, "A");
12079 Ok(resumed.clone())
12080 }),
12081 ActionDecision::Resume("A".into())
12082 );
12083 assert_eq!(
12084 action_standing(&t, &action_question("B", resume_action("B"))),
12085 ActionStanding::Stale
12086 );
12087 t.release_operator("cli", None);
12088 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
12089 assert_eq!(id, "A");
12090 Ok(resumed.clone())
12091 });
12092 assert_eq!(
12093 choose_starter(None, false, unfinished.as_deref()),
12094 Starter::Resume("A".into())
12095 );
12096 assert_eq!(t.runs, vec!["B", "A"]);
12097 }
12098 }
12099
12100 #[test]
12101 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
12102 let mut state = crate::land::tests::repair_state(true, true);
12103 state.status = RunStatus::Landing;
12104 state.parked = true;
12105 state.driver_exited = true;
12106 let mut q = ask::Question::new(
12107 state.id.clone(),
12108 land::APPROVAL_NODE.into(),
12109 "land".into(),
12110 "Merge?".into(),
12111 String::new(),
12112 vec![land::APPROVE.into(), land::HOLD.into()],
12113 );
12114 q.id = state.id.clone();
12115 state.land_approval = Some(crate::run::LandApproval {
12116 question: q.id.clone(),
12117 head: "current-head".into(),
12118 });
12119 assert!(land::upgrade_repair_choice(&mut q, &state));
12120 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12121 let mut task = held_task_with(&state.id);
12122 task.status = TaskStatus::Parked;
12123 assert_eq!(
12124 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12125 ActionDecision::Resume(state.id.clone())
12126 );
12127 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12128 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
12129 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
12130 assert_eq!(
12131 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12132 ActionDecision::Resume(state.id.clone()),
12133 "a repair answer newer than the manual hold resumes the existing owner"
12134 );
12135 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12136 for answered_at in [q.answered_at, None] {
12137 let mut older = q.clone();
12138 older.answered_at = answered_at;
12139 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
12140 panic!("a later manual hold must prevent dispatch")
12141 });
12142 let ActionDecision::Refuse(reason) = refused else {
12143 panic!("expected an actionable refusal, got {refused:?}");
12144 };
12145 assert!(reason.contains("later operator decision"));
12146 assert!(reason.contains(&format!("magi task release {}", task.id)));
12147 }
12148 task.operator_decided_at = None;
12149 let mut superseded = state.clone();
12150 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
12151 assert_eq!(
12152 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
12153 ActionDecision::Stale
12154 );
12155 for case in 1..4 {
12156 let mut changed = state.clone();
12157 match case {
12158 1 => changed.status = RunStatus::Blocked,
12159 2 => changed.released_to = Some("new-owner".into()),
12160 _ => changed.land_approval.as_mut().unwrap().head.clear(),
12161 }
12162 assert!(matches!(
12163 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
12164 ActionDecision::Refuse(_)
12165 ));
12166 }
12167 assert!(matches!(
12168 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
12169 ActionDecision::Refuse(_)
12170 ));
12171 task.runs.push("replacement".into());
12172 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12173 assert_eq!(
12174 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
12175 "old owner must not load"
12176 )),
12177 ActionDecision::Stale
12178 );
12179 }
12180
12181 #[test]
12182 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
12183 let mut state = crate::land::tests::repair_state(true, true);
12184 state.status = RunStatus::Landing;
12185 state.parked = true;
12186 state.driver_exited = true;
12187 let mut q = ask::Question::new(
12188 state.id.clone(),
12189 land::APPROVAL_NODE.into(),
12190 "land".into(),
12191 "Merge?".into(),
12192 String::new(),
12193 vec![land::APPROVE.into(), land::HOLD.into()],
12194 );
12195 q.id = state.id.clone();
12196 state.land_approval = Some(crate::run::LandApproval {
12197 question: q.id.clone(),
12198 head: "current-head".into(),
12199 });
12200 state.save().unwrap();
12201 land::upgrade_repair_choice(&mut q, &state);
12202 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12203 Questions::open().put(&mut q).unwrap();
12204 let tmp = tempfile::tempdir().unwrap();
12205 let queue = Queue::at(tmp.path().join("queue"));
12206 let mut task = held_task_with(&state.id);
12207 task.status = TaskStatus::Parked;
12208 queue.put(&mut task).unwrap();
12209 let empty = Arc::new(tokio::sync::Semaphore::new(0));
12210 let free = Arc::new(tokio::sync::Semaphore::new(1));
12211 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
12212 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
12213 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
12214 assert!(claim.permit.is_some());
12215 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
12216 drop(claim);
12217 Questions::open()
12218 .update(&q.id, |q| {
12219 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
12220 Ok(())
12221 })
12222 .unwrap();
12223 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
12224 assert!(claim.permit.is_none());
12225 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
12226 }
12227
12228 #[test]
12229 fn decide_action_resumes_only_the_latest_resumable_run() {
12230 let t = held_task_with("r1");
12231 let q = action_question("r1", resume_action("r1"));
12232 let load = |s: RunState| move |_: &str| Ok(s);
12233 assert_eq!(
12234 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
12235 ActionDecision::Resume("r1".into())
12236 );
12237 let q_other = action_question("r1", resume_action("r0"));
12239 assert!(matches!(
12240 decide_action(
12241 &t,
12242 &q_other,
12243 &PHRASES_EN,
12244 load(run_state(RunStatus::Blocked))
12245 ),
12246 ActionDecision::Refuse(_)
12247 ));
12248 assert!(matches!(
12250 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
12251 ActionDecision::Refuse(_)
12252 ));
12253 let mut released = run_state(RunStatus::Blocked);
12255 released.released_to = Some("elsewhere".into());
12256 assert!(matches!(
12257 decide_action(&t, &q, &PHRASES_EN, load(released)),
12258 ActionDecision::Refuse(_)
12259 ));
12260 assert!(matches!(
12262 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
12263 ActionDecision::Refuse(_)
12264 ));
12265 }
12266
12267 #[test]
12268 fn decide_action_ignores_a_question_about_an_earlier_run() {
12269 let mut t = held_task_with("r1");
12270 t.runs.push("r2".to_owned());
12271 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12272 assert_eq!(
12273 decide_action(
12274 &t,
12275 &action_question("r1", ask::ChoiceAction::Done),
12276 &PHRASES_EN,
12277 never
12278 ),
12279 ActionDecision::Stale
12280 );
12281 }
12282
12283 #[test]
12284 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
12285 let mut t = held_task_with("r1");
12286 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12287 assert_eq!(
12288 decide_action(
12289 &t,
12290 &action_question("r1", ask::ChoiceAction::Requeue),
12291 &PHRASES_EN,
12292 never
12293 ),
12294 ActionDecision::Requeue
12295 );
12296 let done_q = action_question("r1", ask::ChoiceAction::Done);
12297 assert_eq!(
12298 decide_action(&t, &done_q, &PHRASES_EN, never),
12299 ActionDecision::Done
12300 );
12301 t.mark_action_applied(&done_q.id);
12302 assert_eq!(
12303 decide_action(&t, &done_q, &PHRASES_EN, never),
12304 ActionDecision::Skip
12305 );
12306
12307 let mut plain = action_question("r1", ask::ChoiceAction::Done);
12309 plain.actions.clear();
12310 assert_eq!(
12311 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
12312 ActionDecision::Skip
12313 );
12314 let mut running = held_task_with("r1");
12316 running.status = TaskStatus::Running;
12317 assert_eq!(
12318 decide_action(
12319 &running,
12320 &action_question("r1", ask::ChoiceAction::Done),
12321 &PHRASES_EN,
12322 never
12323 ),
12324 ActionDecision::Skip
12325 );
12326 }
12327
12328 #[test]
12329 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12330 crate::run::pin_test_home();
12331 for legacy in [false, true] {
12332 let dir = tempfile::tempdir().unwrap();
12333 let queue = Queue::at(dir.path().join("queue"));
12334 let questions = Questions::at(dir.path().join("questions"));
12335 let run = "20260101-000000-hold";
12336 let mut task = held_task_with(run);
12337 task.status = TaskStatus::Parked;
12338 let mut q = ask::Question::new(
12339 run.into(),
12340 land::APPROVAL_NODE.into(),
12341 "land".into(),
12342 "Repair?".into(),
12343 String::new(),
12344 vec![land::REPAIR.into()],
12345 );
12346 q.actions.insert(land::REPAIR.into(), resume_action(run));
12347 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12348 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12350 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12351 questions.put(&mut q).unwrap();
12352 task.hold_manual(Some("Keep this PR paused".into()));
12353 if legacy {
12354 task.hold_source = None;
12355 }
12356 queue.put(&mut task).unwrap();
12357
12358 apply_task_choice_actions(&queue, &questions, dir.path());
12359 let after = queue.get(&task.id).unwrap();
12360 assert!(after.operator_held());
12361 assert_eq!(after.hold_source, task.hold_source);
12362 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12363 assert_eq!(after.runs, task.runs);
12364 assert_eq!(after.instruction, task.instruction);
12365 assert!(after.resume_override.is_none());
12366 assert!(after.action_applied(&q.id));
12367 let reason = after.hold_reason.as_deref().unwrap();
12368 assert!(reason.starts_with("Keep this PR paused"));
12369 assert!(reason.contains("later operator decision"));
12370 assert!(reason.contains(&format!("magi task release {}", task.id)));
12371
12372 for recovery in [
12373 crate::conduct::Recovery::Review,
12374 crate::conduct::Recovery::Requeue,
12375 ] {
12376 crate::conduct::apply(
12377 &queue,
12378 &questions,
12379 &crate::conduct::Verdict {
12380 decisions: vec![crate::conduct::Decision {
12381 id: task.id.clone(),
12382 recovery: Some(recovery),
12383 ..Default::default()
12384 }],
12385 },
12386 )
12387 .unwrap();
12388 }
12389 apply_task_choice_actions(&queue, &questions, dir.path());
12392 let still_held = queue.get(&task.id).unwrap();
12393 assert!(still_held.operator_held());
12394 assert_eq!(still_held.hold_reason, after.hold_reason);
12395 assert_eq!(still_held.runs, task.runs);
12396 assert!(still_held.review_branch.is_none());
12397 assert!(!still_held.fresh_start);
12398 assert!(queue.next_runnable().is_none());
12399 }
12400 }
12401
12402 #[test]
12403 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
12404 for held in [false, true] {
12405 for retry in [false, true] {
12406 let dir = tempfile::tempdir().unwrap();
12407 let queue = Queue::at(dir.path().join("queue"));
12408 let questions = Questions::at(dir.path().join("questions"));
12409 let home = dir.path().join("home");
12410 let mut t = task();
12411 if held {
12412 t.hold_machine(Some("driver state unknown".into()));
12413 }
12414 queue.put(&mut t).unwrap();
12415 let verdict = serde_json::from_value(serde_json::json!({
12416 "decisions": [{
12417 "id": t.id,
12418 "question": "Driver state unknown; retry the checks or hold?",
12419 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
12420 "actions": {
12421 "daemon: recheck and retry": {"do": "requeue"},
12422 "daemon: hold the task": {"do": "hold"}
12423 }
12424 }]
12425 }))
12426 .unwrap();
12427 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
12428 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12429 let mut q = questions.list().pop().unwrap();
12430 let choice = if retry {
12431 "daemon: recheck and retry"
12432 } else {
12433 "daemon: hold the task"
12434 };
12435 assert_eq!(q.actions.len(), 2);
12436 q.answer(ask::Answer::Choice(choice.into())).unwrap();
12437 questions.put(&mut q).unwrap();
12438
12439 questions.beat(&q.id, ask::WaiterKind::Deputy);
12441 for _ in 0..2 {
12442 resolve_blockers(&queue, &questions);
12443 apply_task_choice_actions(&queue, &questions, &home);
12444 let waiting = queue.get(&t.id).unwrap();
12445 assert_eq!(waiting.status, TaskStatus::Blocked);
12446 assert_eq!(waiting.blocked_by, [q.id.clone()]);
12447 assert!(waiting.answers.is_empty());
12448 assert!(!waiting.action_applied(&q.id));
12449 assert!(runnable(&queue).is_empty());
12450 let free = Arc::new(tokio::sync::Semaphore::new(1));
12451 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
12452 }
12453 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12454
12455 resolve_blockers(&queue, &questions);
12457 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
12458 apply_task_choice_actions(&queue, &questions, &home);
12459 let after = queue.get(&t.id).unwrap();
12460 assert_eq!(
12461 after.status,
12462 if retry {
12463 TaskStatus::Queued
12464 } else {
12465 TaskStatus::Held
12466 }
12467 );
12468 assert!(after.action_applied(&q.id));
12469 assert_eq!(after.operator_held(), !retry);
12470 assert_eq!(after.answers.len(), 1);
12471
12472 let mut changed = after;
12474 if retry {
12475 changed.hold_manual(Some("later hold".into()));
12476 } else {
12477 changed.release_operator("test", None);
12478 }
12479 let expected = changed.status;
12480 queue.put(&mut changed).unwrap();
12481 apply_task_choice_actions(&queue, &questions, &home);
12482 assert_eq!(queue.get(&t.id).unwrap().status, expected);
12483 }
12484 }
12485 }
12486
12487 #[test]
12488 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
12489 let dir = tempfile::tempdir().unwrap();
12490 let queue = Queue::at(dir.path().join("queue"));
12491 let questions = Questions::at(dir.path().join("questions"));
12492 let home = dir.path().join("home");
12493 let mut state = run_state(RunStatus::Blocked);
12494 state.id = "20260101-000000-act1".to_owned();
12495 state.save_under(&home).unwrap();
12496
12497 let mut t = held_task_with(&state.id);
12498 queue.put(&mut t).unwrap();
12499 let mut q = action_question(&state.id, resume_action(&state.id));
12500 questions.put(&mut q).unwrap();
12501
12502 apply_task_choice_actions(&queue, &questions, &home);
12503 let after = queue.get(&t.id).unwrap();
12504 assert_eq!(after.status, TaskStatus::Queued);
12505 assert!(!after.fresh_start);
12506 assert!(after.action_applied(&q.id));
12507 let pin = after.resume_override.clone().unwrap();
12508 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
12509 assert!(pin.forced);
12510
12511 let mut again = queue.get(&t.id).unwrap();
12513 again.hold_machine(Some("later".into()));
12514 queue.put(&mut again).unwrap();
12515 apply_task_choice_actions(&queue, &questions, &home);
12516 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12517 }
12518
12519 #[test]
12520 fn a_delivered_answer_is_still_acted_on_exactly_once() {
12521 let dir = tempfile::tempdir().unwrap();
12522 let queue = Queue::at(dir.path().join("queue"));
12523 let questions = Questions::at(dir.path().join("questions"));
12524 let home = dir.path().join("home");
12525 let mut state = run_state(RunStatus::Blocked);
12526 state.id = "20260101-000000-act2".to_owned();
12527 state.save_under(&home).unwrap();
12528
12529 let mut t = held_task_with(&state.id);
12530 queue.put(&mut t).unwrap();
12531 let mut q = action_question(&state.id, resume_action(&state.id));
12532 q.answer_delivered = true;
12533 questions.put(&mut q).unwrap();
12534
12535 apply_task_choice_actions(&queue, &questions, &home);
12538 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12539 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
12540
12541 let mut again = queue.get(&t.id).unwrap();
12543 again.hold_machine(Some("later".into()));
12544 queue.put(&mut again).unwrap();
12545 apply_task_choice_actions(&queue, &questions, &home);
12546 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12547 }
12548
12549 #[test]
12550 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
12551 let dir = tempfile::tempdir().unwrap();
12552 let queue = Queue::at(dir.path().join("queue"));
12553 let questions = Questions::at(dir.path().join("questions"));
12554 let home = dir.path().join("home");
12555 let mut state = run_state(RunStatus::Blocked);
12556 state.id = "20260101-000000-act3".to_owned();
12557 state.save_under(&home).unwrap();
12558 let mut t = held_task_with(&state.id);
12559 queue.put(&mut t).unwrap();
12560 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
12561 questions.put(&mut q).unwrap();
12562
12563 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
12564 apply_task_choice_actions(&queue, &questions, &home);
12565 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12566
12567 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
12568 apply_task_choice_actions(&queue, &questions, &home);
12569 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
12570 }
12571
12572 #[test]
12573 fn action_standing_tells_the_reasons_apart() {
12574 let mut t = held_task_with("r1");
12575 let q = action_question("r1", ask::ChoiceAction::Requeue);
12576 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12577 let mut plain = q.clone();
12578 plain.actions.clear();
12579 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
12580
12581 t.mark_action_applied(&q.id);
12583 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
12584
12585 let mut t = held_task_with("r1");
12587 t.start("r2".to_owned());
12588 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
12589 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
12591 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12592 t.status = TaskStatus::Blocked;
12593 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
12594
12595 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
12597 c.node = crate::conduct::NODE.to_owned();
12598 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
12599 assert!(!ActionStanding::Busy.daemon_owns());
12600 }
12601
12602 #[test]
12603 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
12604 let dir = tempfile::tempdir().unwrap();
12605 let queue = Queue::at(dir.path().join("queue"));
12606 let questions = Questions::at(dir.path().join("questions"));
12607 let home = dir.path().join("home");
12608 let mut state = run_state(RunStatus::Blocked);
12609 state.id = "20260101-000000-act4".to_owned();
12610 state.save_under(&home).unwrap();
12611 let mut t = held_task_with(&state.id);
12612 t.status = TaskStatus::Running;
12613 queue.put(&mut t).unwrap();
12614 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
12615 q.cwd = Some(dir.path().display().to_string());
12616 questions.put(&mut q).unwrap();
12617
12618 let running = queue.get(&t.id).unwrap();
12621 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
12622 assert!(matches!(
12623 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
12624 crate::waiter::Action::Deliver(_)
12625 ));
12626 apply_task_choice_actions(&queue, &questions, &home);
12628 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
12629 let mut back = queue.get(&t.id).unwrap();
12630 back.hold_machine(Some("later".into()));
12631 queue.put(&mut back).unwrap();
12632 apply_task_choice_actions(&queue, &questions, &home);
12633 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
12634 let mut again = queue.get(&t.id).unwrap();
12635 again.hold_machine(Some("again".into()));
12636 queue.put(&mut again).unwrap();
12637 apply_task_choice_actions(&queue, &questions, &home);
12638 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
12639 }
12640}