1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.runs.last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id,
519 run: task.runs.last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if q.chosen_action().is_none() {
887 ActionStanding::NoAction
888 } else if task.action_applied(&q.id) {
889 ActionStanding::Applied
890 } else if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
891 ActionStanding::Stale
892 } else if matches!(
893 task.status,
894 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
895 ) {
896 ActionStanding::Busy
897 } else {
898 ActionStanding::Pending
899 }
900}
901
902#[derive(Debug, Clone, PartialEq, Eq)]
904enum ActionDecision {
905 Skip,
907 Resume(String),
909 Requeue,
911 Hold,
913 Done,
915 Stale,
918 Refuse(String),
921}
922
923fn decide_action<F>(task: &Task, q: &ask::Question, p: &Phrases, load: F) -> ActionDecision
931where
932 F: FnOnce(&str) -> Result<RunState>,
933{
934 let Some(action) = q.chosen_action() else {
935 return ActionDecision::Skip;
936 };
937 if task.action_applied(&q.id)
938 || matches!(
939 task.status,
940 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
941 )
942 {
943 return ActionDecision::Skip;
944 }
945 if q.node != crate::conduct::NODE && task.runs.last() != Some(&q.run) {
948 return ActionDecision::Stale;
949 }
950 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
951 let stopping = q.node == land::APPROVAL_NODE
952 && q.resolution()
953 .as_deref()
954 .is_some_and(crate::merge_decision::stops);
955 let decision_at = q.answered_at.unwrap_or(q.asked_at);
959 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
960 return if repair || stopping {
961 ActionDecision::Refuse(format!(
962 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
963 if repair { "repair" } else { "stop" },
964 q.id,
965 q.run,
966 task.id
967 ))
968 } else {
969 ActionDecision::Stale
970 };
971 }
972 match action {
973 a if a.maintenance() => ActionDecision::Skip,
974 ask::ChoiceAction::ResumeRun { .. }
975 | ask::ChoiceAction::ReleaseTask { .. }
976 | ask::ChoiceAction::RerunChecks { .. }
977 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
978 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
979 ask::ChoiceAction::Hold => ActionDecision::Hold,
980 ask::ChoiceAction::Done => ActionDecision::Done,
981 ask::ChoiceAction::Resume { run } => {
982 if task.runs.last() != Some(run) {
983 return ActionDecision::Refuse((p.resume_not_latest)(
984 q.short(),
985 ask::short_id(run),
986 ));
987 }
988 match load(run) {
989 Ok(s) if stopping => {
990 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
991 || !s
992 .land_approval
993 .as_ref()
994 .is_some_and(|binding| binding.question == q.id)
995 {
996 ActionDecision::Stale
997 } else if s.liveness(false) == Liveness::Live {
998 ActionDecision::Skip
999 } else if s.status == RunStatus::Landing
1000 && !s.released()
1001 && s.retired_to.is_none()
1002 {
1003 ActionDecision::Resume(run.clone())
1004 } else {
1005 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1006 }
1007 }
1008 Ok(s)
1009 if q.node == land::APPROVAL_NODE
1010 && q.resolution().as_deref() == Some(land::REPAIR) =>
1011 {
1012 if s.merge_repairs.iter().any(|r| r.question == q.id)
1013 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1014 {
1015 ActionDecision::Stale
1018 } else if s.liveness(false) == Liveness::Live {
1019 ActionDecision::Skip
1020 } else if s.status != RunStatus::Landing
1021 || s.released()
1022 || s.reviews.is_empty()
1023 || !implementers_match(task, &s)
1024 || !s
1025 .land_approval
1026 .as_ref()
1027 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1028 {
1029 ActionDecision::Refuse(format!(
1030 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1031 q.id, run, run
1032 ))
1033 } else {
1034 ActionDecision::Resume(run.clone())
1035 }
1036 }
1037 Ok(s) if s.liveness(false) == Liveness::Live => {
1038 ActionDecision::Refuse("run already has a live driver".into())
1039 }
1040 Ok(s) => match crate::handover::resume_refusal(&s) {
1041 None => ActionDecision::Resume(run.clone()),
1042 Some(reason) => ActionDecision::Refuse((p.resume_cannot_progress)(
1043 q.short(),
1044 ask::short_id(run),
1045 &reason.diagnostic(&s),
1046 )),
1047 },
1048 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1049 q.short(),
1050 ask::short_id(run),
1051 &format!("{e:#}"),
1052 )),
1053 }
1054 }
1055 }
1056}
1057
1058struct Phrases {
1072 graph_stopped: fn(&str, &str) -> String,
1074 quorum_lost: &'static str,
1075 quota_took_out: &'static str,
1077 run_ended: &'static str,
1079 waiting_for_answer: &'static str,
1081 recovered_running: &'static str,
1082 no_run_to_recover: &'static str,
1083 could_not_start: &'static str,
1084 resume_not_latest: fn(&str, &str) -> String,
1085 resume_cannot_progress: fn(&str, &str, &str) -> String,
1086 resume_unreadable: fn(&str, &str, &str) -> String,
1087 handover_refused: fn(&crate::handover::Refused) -> String,
1090 handover_hint: &'static str,
1092 reviewers_never_answered: fn(usize, usize) -> String,
1095}
1096
1097const PHRASES_EN: Phrases = Phrases {
1098 graph_stopped: |status, detail| {
1099 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1100 },
1101 quorum_lost: "the judging panel lost its quorum",
1102 quota_took_out: "; quota took out ",
1103 run_ended: "run ended ",
1104 waiting_for_answer: " - waiting for operator answer to question ",
1105 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1106 no_run_to_recover: "task was `running` with no live daemon and no readable \
1107 run to recover; held for a human to check what happened",
1108 could_not_start: "could not start the run: ",
1109 resume_not_latest: |q, run| {
1110 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1111 },
1112 resume_cannot_progress: |q, run, reason| {
1113 format!("question {q} asked to resume run {run}, which cannot make progress: {reason}")
1114 },
1115 resume_unreadable: |q, run, e| {
1116 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1117 },
1118 handover_refused: |r| r.to_string(),
1119 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1120 reviewers_never_answered: |missing, rounds| {
1121 format!(
1122 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1123 refusing to call it clean"
1124 )
1125 },
1126};
1127
1128const PHRASES_JA: Phrases = Phrases {
1129 graph_stopped: |status, detail| {
1130 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1131 },
1132 quorum_lost: "審査パネルが定足数を失いました",
1133 quota_took_out: "。クォータで脱落: ",
1134 run_ended: "run 終了: ",
1135 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1136 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1137 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1138 何が起きたか人が確認するため保留にしました",
1139 could_not_start: "run を開始できませんでした: ",
1140 resume_not_latest: |q, run| {
1141 format!(
1142 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1143 )
1144 },
1145 resume_cannot_progress: |q, run, reason| {
1146 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません: {reason}")
1147 },
1148 resume_unreadable: |q, run, e| {
1149 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1150 },
1151 handover_refused: |r| {
1152 use crate::handover::Refused;
1153 match r {
1154 Refused::Foreign { branch, path, why } => format!(
1155 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1156 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1157 から、やり直してください(詳細: {why})"
1158 ),
1159 Refused::Unsafe { branch, path, why } => format!(
1160 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1161 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1162 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1163 ),
1164 Refused::ReleaseFailed { branch, path, run } => format!(
1165 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1166 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1167 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1168 ),
1169 }
1170 },
1171 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1172 reviewers_never_answered: |missing, rounds| {
1173 format!(
1174 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1175 クリーンとは認めません"
1176 )
1177 },
1178};
1179
1180fn phrases(language: &str) -> &'static Phrases {
1181 if crate::lang::is_japanese(language) {
1182 &PHRASES_JA
1183 } else {
1184 &PHRASES_EN
1185 }
1186}
1187
1188fn language_of(task: &Task, fallback: &Path) -> String {
1192 crate::lang::of_repo(&repo_for(task, fallback))
1193}
1194
1195pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1199 if q.node == crate::conduct::NODE {
1200 return tasks.iter().find(|t| t.id == q.run);
1201 }
1202 tasks.iter().find(|t| t.runs.contains(&q.run))
1203}
1204
1205fn repair_choice_offers(questions: &Questions) {
1208 for q in questions.list().into_iter().filter(|q| {
1209 q.status.open()
1210 && q.node == crate::conduct::NODE
1211 && q.choices
1212 .iter()
1213 .any(|c| crate::choice_effect::describe(q, c).is_none())
1214 }) {
1215 let _ = questions.update(&q.id, |current| {
1216 if current.status.open() {
1217 crate::choice_effect::executable_conductor(current);
1218 if let Some(deputy) = &mut current.deputy {
1219 deputy.brief = crate::deputy::brief(
1220 ¤t.run,
1221 ¤t.detail,
1222 ¤t.choices,
1223 ¤t.actions,
1224 );
1225 }
1226 }
1227 Ok(())
1228 });
1229 }
1230}
1231
1232pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1235 apply_maintenance_choices(queue, questions, home).await;
1236 apply_task_choice_actions(queue, questions, home);
1237}
1238
1239async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1242 for q in questions.list() {
1243 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1244 continue;
1245 };
1246 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1249 continue;
1250 }
1251 let dir = home.join("choice-actions");
1252 if std::fs::create_dir_all(&dir).is_err() {
1253 continue;
1254 }
1255 let Ok(lock) = std::fs::OpenOptions::new()
1256 .create(true)
1257 .truncate(false)
1258 .write(true)
1259 .open(dir.join(format!("{}.lock", q.id)))
1260 else {
1261 continue;
1262 };
1263 if crate::pr_head::try_lock_file(&lock).is_err() {
1264 continue;
1265 }
1266 let fresh = !home
1267 .join("choice-actions")
1268 .join(format!("{}.json", q.id))
1269 .exists();
1270 let result = async {
1271 let receipt = dir.join(format!("{}.json", q.id));
1272 if receipt.exists() {
1273 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1274 anyhow::ensure!(
1275 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1276 "previous maintenance attempt did not confirm completion; inspect its receipt"
1277 );
1278 return Ok::<_, anyhow::Error>(());
1279 }
1280 let record = |status: &str| -> anyhow::Result<()> {
1283 let tmp = receipt.with_extension("tmp");
1284 std::fs::write(
1285 &tmp,
1286 serde_json::to_vec(&serde_json::json!({
1287 "action": action, "status": status,
1288 }))?,
1289 )?;
1290 std::fs::rename(tmp, &receipt)?;
1291 Ok(())
1292 };
1293 let applied = async {
1294 match action {
1295 ask::ChoiceAction::ResumeRun { run } => {
1296 crate::resume::validate(home, run)?;
1297 record("started")?;
1298 let decided = q.answered_at.unwrap_or(q.asked_at);
1299 if !crate::resume::pin_queued(queue, run, Some(decided), Some(&q.id))? {
1300 crate::resume::validate_unqueued(home, run)?;
1301 crate::resume::start(home, run.clone()).await?;
1302 }
1303 }
1304 ask::ChoiceAction::ReleaseTask {
1305 task,
1306 held_at,
1307 reason,
1308 } => {
1309 anyhow::ensure!(
1310 queue.resolve_id(task)? == *task,
1311 "task action requires an exact task ID"
1312 );
1313 let _claim = queue.claim(task)?;
1314 let mut task = queue.get(task)?;
1315 if task.action_applied(&q.id) {
1316 return Ok(());
1317 }
1318 if q.node == crate::conduct::NODE
1323 && q.run == task.id
1324 && task.status == TaskStatus::Blocked
1325 && task.blocked_from == Some(TaskStatus::Held)
1326 && task.blocked_by.as_slice() == [q.id.as_str()]
1327 {
1328 task.unblock(&q.id);
1329 }
1330 anyhow::ensure!(
1331 task.status == TaskStatus::Held
1332 && task.held_at == Some(*held_at)
1333 && task.hold_reason.as_deref() == reason.as_deref()
1334 && task
1335 .operator_decided_at
1336 .is_none_or(|at| at <= q.answered_at.unwrap_or(q.asked_at)),
1337 "task no longer has the hold this question names"
1338 );
1339 task.release_operator("choice", Some(&q.id));
1340 task.mark_action_applied(&q.id);
1341 queue.put(&mut task)?;
1342 }
1343 ask::ChoiceAction::AbandonRun { run } => {
1344 record("started")?;
1345 crate::driver::abandon_proven_dead(home, run)?;
1346 }
1347 ask::ChoiceAction::RerunChecks {
1348 repo,
1349 pr,
1350 head,
1351 runs,
1352 } => {
1353 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1354 .await?;
1355 }
1356 _ => unreachable!(),
1357 }
1358 Ok::<_, anyhow::Error>(())
1359 }
1360 .await;
1361 match applied {
1362 Ok(()) => record("applied"),
1363 Err(e) => {
1364 record(&format!("refused or failed: {e:#}"))?;
1365 Err(e)
1366 }
1367 }
1368 }
1369 .await;
1370 if home
1375 .join("choice-actions")
1376 .join(format!("{}.json", q.id))
1377 .exists()
1378 && let Some(listed) = task_of_question(&queue.list(), &q)
1379 && !listed.action_applied(&q.id)
1380 && let Ok(_claim) = queue.claim(&listed.id)
1381 && let Ok(mut origin) = queue.get(&listed.id)
1382 {
1383 if !origin.operator_held()
1384 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1385 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1386 {
1387 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1388 origin.hold_machine(Some(format!(
1389 "Maintenance choice {} settled; inspect its target before releasing this task",
1390 q.id
1391 )));
1392 } else {
1393 origin.unblock(&q.id);
1394 }
1395 }
1396 origin.mark_action_applied(&q.id);
1397 let _ = queue.put(&mut origin);
1398 }
1399 if let Err(e) = result
1400 && fresh
1401 {
1402 notices::raise_in(
1403 home,
1404 Notice::warn(
1405 &format!("action:{}", q.id),
1406 format!("Choice action {} was not completed: {e:#}", q.id),
1407 )
1408 .link(if q.run_names_task() {
1409 Link::Task { id: q.run.clone() }
1410 } else {
1411 Link::Run { id: q.run.clone() }
1412 }),
1413 );
1414 }
1415 }
1416}
1417
1418pub(crate) async fn rerun_checks_once(
1425 home: &Path,
1426 request: &str,
1427 repo: &Path,
1428 pr: &str,
1429 head: &str,
1430 runs: &[String],
1431 started: impl FnOnce() -> anyhow::Result<()>,
1432) -> anyhow::Result<()> {
1433 use crate::release_watch::{GhForge, ReleaseForge};
1434 anyhow::ensure!(
1435 !runs.is_empty()
1436 && runs.len() <= 20
1437 && runs
1438 .iter()
1439 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1440 && !pr.starts_with('-'),
1441 "invalid rerun targets"
1442 );
1443 let snap = GhForge.snapshot(repo, pr).await?;
1444 anyhow::ensure!(
1445 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1446 "pull request head or lifecycle changed"
1447 );
1448 anyhow::ensure!(
1449 runs.iter().all(|r| snap
1450 .checks
1451 .iter()
1452 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1453 "named workflow is no longer failed on this pull request"
1454 );
1455 started()?;
1456 let dir = home.join("choice-actions");
1457 std::fs::create_dir_all(&dir)?;
1458 let rerun_lock = std::fs::OpenOptions::new()
1459 .create(true)
1460 .truncate(false)
1461 .write(true)
1462 .open(dir.join("reruns.lock"))?;
1463 crate::pr_head::try_lock_file(&rerun_lock)?;
1464 let ledger = dir.join("reruns.json");
1465 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1466 serde_json::from_slice(&std::fs::read(&ledger)?)?
1467 } else {
1468 Default::default()
1469 };
1470 for run in runs {
1471 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1472 if !recorded.insert(key) {
1473 continue;
1474 }
1475 let tmp = ledger.with_extension("tmp");
1476 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1477 std::fs::rename(tmp, &ledger)?;
1478 GhForge.rerun(repo, run).await?;
1479 }
1480 Ok(())
1481}
1482
1483pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1492 let tasks = queue.list();
1493 for q in questions.list() {
1494 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1496 continue;
1497 }
1498 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1499 continue;
1500 }
1501 let Some(listed) = task_of_question(&tasks, &q) else {
1502 if q.node == land::APPROVAL_NODE
1503 && q.resolution().as_deref().is_some_and(|choice| {
1504 choice == land::REPAIR || crate::merge_decision::stops(choice)
1505 })
1506 {
1507 notices::raise_in(
1508 home,
1509 Notice::warn(
1510 &format!("repair-owner:{}", q.id),
1511 format!(
1512 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1513 q.run, q.run
1514 ),
1515 ).link(Link::Run { id: q.run.clone() }),
1516 );
1517 }
1518 continue;
1519 };
1520 if listed.action_applied(&q.id) {
1521 continue;
1522 }
1523 let Ok(_claim) = queue.claim(&listed.id) else {
1524 continue;
1525 };
1526 let Ok(mut task) = queue.get(&listed.id) else {
1527 continue;
1528 };
1529 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1532 let blockers: Vec<_> = task
1533 .blocked_by
1534 .iter()
1535 .filter(|id| {
1536 crate::queue::dependency(queue, questions, id)
1537 == crate::queue::Dependency::Question(q.id.clone())
1538 })
1539 .cloned()
1540 .collect();
1541 if !blockers.is_empty() {
1542 for id in blockers {
1543 task.unblock(&id);
1544 }
1545 let answer = match &q.answer {
1546 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1547 None => String::new(),
1548 };
1549 task.record_answer(q.summary.clone(), answer);
1550 }
1551 }
1552 let language = language_of(&task, Path::new("."));
1553 let decision = decide_action(&task, &q, phrases(&language), |id| {
1554 RunState::load_under(id, home)
1555 });
1556 let ran = matches!(
1557 decision,
1558 ActionDecision::Resume(_)
1559 | ActionDecision::Requeue
1560 | ActionDecision::Hold
1561 | ActionDecision::Done
1562 );
1563 if ran && asker_may_still_read(questions, &q, home) {
1564 continue;
1573 }
1574 let done = matches!(decision, ActionDecision::Done);
1575 match decision {
1576 ActionDecision::Skip => continue,
1577 ActionDecision::Resume(run) => {
1578 task.release_operator("choice", Some(&q.id));
1579 task.resume_override = Some(crate::queue::OperatorResume {
1580 question_id: q.id.clone(),
1581 at: Timestamp::now(),
1582 conductor_rehold: None,
1583 forced: true,
1584 pinned_run: Some(run),
1585 });
1586 }
1587 ActionDecision::Stale => {}
1588 ActionDecision::Requeue => {
1589 task.release_operator("choice-requeue", Some(&q.id));
1590 task.requeue();
1591 }
1592 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1593 ActionDecision::Done => task.succeed(),
1594 ActionDecision::Refuse(why) => {
1595 if task.operator_held() {
1596 task.hold_reason = Some(match task.hold_reason.take() {
1599 Some(reason) => format!("{reason}\n\n{why}"),
1600 None => why,
1601 });
1602 } else {
1603 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1604 task.hold_machine(Some(why));
1605 }
1606 notices::raise(
1607 Notice::warn(
1608 &format!("action:{}", q.id),
1609 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1610 )
1611 .link(Link::Task {
1612 id: task.id.clone(),
1613 }),
1614 );
1615 }
1616 }
1617 task.mark_action_applied(&q.id);
1618 record(queue, &mut task);
1621 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1622 supersede_prior_runs(&task, home);
1623 }
1624 let _ = questions.update(&q.id, |r| {
1627 r.answer_delivered = true;
1628 Ok(())
1629 });
1630 }
1631}
1632
1633fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1637 let now = Timestamp::now();
1638 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1639 return true;
1640 }
1641 q.node != crate::conduct::NODE
1642 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1643 s.seats_active()
1644 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1645 })
1646}
1647
1648fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1659 let tasks = queue.list();
1660 let by_id: std::collections::BTreeMap<&str, &Task> =
1661 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1662 let referenced: Vec<&str> = tasks
1663 .iter()
1664 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1665 .collect();
1666
1667 for mut question in questions.list() {
1668 if !question.status.open() || question.node != crate::conduct::NODE {
1669 continue;
1670 }
1671 if referenced
1676 .iter()
1677 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1678 {
1679 continue;
1680 }
1681 let Some(task) = by_id.get(question.run.as_str()) else {
1682 continue;
1683 };
1684 question.abandon(format!(
1685 "task {} no longer waits for this answer",
1686 task.short()
1687 ));
1688 if let Err(e) = questions.put(&mut question) {
1689 tracing::warn!(
1690 "could not retire question {} for task {}: {e:#}",
1691 question.short(),
1692 task.short()
1693 );
1694 }
1695 }
1696}
1697
1698#[derive(Debug, Clone, Copy)]
1705pub struct Verdict {
1706 pub status: RunStatus,
1708 pub left_pr: bool,
1710 pub quota_hit: bool,
1712 pub parked: bool,
1714 pub no_viable_candidates: bool,
1721}
1722
1723pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1782 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1783}
1784
1785fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1787 if verdict.parked {
1793 task.park(detail);
1794 return;
1795 }
1796 match verdict.status {
1797 RunStatus::Merged | RunStatus::Ready
1798 if task.pr_recovery.as_ref().is_some_and(|s| {
1799 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1800 }) =>
1801 {
1802 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1803 }
1804 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1805 RunStatus::AlreadyInBase => task.already_landed(detail),
1806 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1807 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1808 task.stall(detail)
1809 }
1810 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1811 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1812 RunStatus::Blocked => task.fail(detail, max_attempts),
1813 RunStatus::VerifiedNoop => task.handed_off(detail),
1814 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1815 }
1816}
1817
1818pub fn supersede_prior_runs(task: &Task, home: &Path) {
1867 let now = Timestamp::now();
1868 let last_run_succeeded = task
1875 .runs
1876 .last()
1877 .and_then(|id| RunState::load_under(id, home).ok())
1878 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1879 for id in task.superseded_attempts(last_run_succeeded) {
1880 let mut state = match RunState::load_under(id, home) {
1881 Ok(s) => s,
1882 Err(e) => {
1883 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1884 continue;
1885 }
1886 };
1887 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1888 continue;
1889 }
1890 let daemon_claims = is_working_on(home, id, now);
1891 if state.liveness(daemon_claims) == Liveness::Live {
1892 continue;
1893 }
1894 state.status = RunStatus::Superseded;
1895 if let Err(e) = state.save_under(home) {
1896 tracing::warn!("could not mark run {id} superseded: {e:#}");
1897 }
1898 }
1899}
1900
1901fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1923 for task in queue.list() {
1924 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1925 continue;
1926 }
1927 supersede_prior_runs(&task, home);
1928 }
1929}
1930
1931fn settle_and_diagnose(
1938 task: &mut Task,
1939 verdict: Verdict,
1940 detail: &str,
1941 max_attempts: usize,
1942 state: &RunState,
1943) {
1944 let p = phrases(&state.config.graph.language);
1945 if let Some(d) = &state.merge_decision {
1948 use crate::merge_decision as md;
1949 if task.runs.last().is_some_and(|id| id != &state.id) {
1950 return;
1951 }
1952 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1953 task.succeed();
1954 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1955 if state.liveness(false) != Liveness::Dead {
1956 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1957 return;
1958 }
1959 let replacement = md::Replacement {
1960 question: d.question.clone(),
1961 run: state.id.clone(),
1962 pr: d.target.pr.clone(),
1963 agent: agent.to_owned(),
1964 context: format!(
1965 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1966 d.target.pr,
1967 state.id,
1968 serde_json::to_string(&state.reviews).unwrap_or_default()
1969 ),
1970 };
1971 task.requeue();
1972 task.pr_recovery = None;
1973 task.review_of = None;
1974 task.replacement = Some(replacement);
1975 } else {
1976 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
1977 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
1978 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
1979 }
1980 task.mark_action_applied(&d.question);
1981 if let Some(pass) = state.passes.last() {
1982 task.record_pass_at(
1983 &state.id,
1984 pass.index,
1985 Some(pass.started_at),
1986 task.attempts,
1987 None,
1988 );
1989 }
1990 return;
1991 }
1992 if let Some(replacement) = &task.replacement
1993 && !state.parked
1994 && state.resource_block.is_none()
1995 && (state.viable().is_empty()
1996 || (state.status == RunStatus::Blocked && state.tally.is_none()))
1997 {
1998 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
1999 return;
2000 }
2001 if !verdict.parked && verdict.quota_hit && state.status == RunStatus::Stalled {
2002 let mut agents: Vec<_> = state
2003 .quota
2004 .iter()
2005 .filter(|q| {
2006 q.cause == crate::agent::QuotaCause::Authentication
2007 && state
2008 .passes
2009 .last()
2010 .is_none_or(|pass| q.at >= pass.started_at)
2011 })
2012 .map(|q| q.agent.as_str())
2013 .collect();
2014 agents.sort_unstable();
2015 agents.dedup();
2016 if !agents.is_empty() {
2017 task.stall(detail);
2018 task.notice_cause("agent-authentication", None, None);
2019 task.hold_machine(Some(format!("[authentication] Agents {} cannot authenticate; re-authenticate, then resume the held task. No attempt was spent.", agents.join(", "))));
2020 return;
2021 }
2022 }
2023 let attempts_before = task.attempts;
2024 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2031 task.notice_cause(
2032 &format!("resource:{}:{}", block.kind, block.node),
2033 None,
2034 None,
2035 );
2036 task.last_error = Some(detail.to_owned());
2037 task.diagnostic = None;
2038 task.attempts = task.attempts.saturating_sub(1);
2039 if block.kind == "agent authentication" {
2040 task.hold_machine(Some(format!(
2041 "[authentication] {}; no attempt was spent",
2042 block.detail
2043 )));
2044 } else if block.kind == "agent usage limit" {
2045 if let Some(agent) = &state.config.graph.selected_implementer {
2047 task.hold_machine(Some(format!("[agent usage limit] Selected implementer `{agent}` is unavailable. No other implementer was dispatched; resume after usage is available. No attempt was spent.")));
2048 } else {
2049 task.status = TaskStatus::Failed;
2050 task.last_error = Some(block.detail.clone());
2051 }
2052 } else if block.kind == "disk pressure" {
2053 task.hold_machine(Some(format!(
2054 "{}{}",
2055 crate::disk::RUNNING_PRESSURE,
2056 block.detail
2057 )));
2058 } else {
2059 task.hold_machine(Some(format!(
2060 "{}{}; no attempt was spent; resume to retry the affected operation",
2061 crate::cache::RESOURCE_HOLD,
2062 block.description()
2063 )));
2064 }
2065 if let Some(pass) = state.passes.last() {
2066 task.record_pass_at(
2067 &state.id,
2068 pass.index,
2069 Some(pass.started_at),
2070 attempts_before,
2071 None,
2072 );
2073 }
2074 return;
2075 }
2076 if let Some(hold) = state.land_unrelated.as_ref().filter(|hold| {
2077 state.parked
2078 && state.status == RunStatus::Landing
2079 && state
2080 .pr_pushed_head
2081 .as_ref()
2082 .is_none_or(|h| h == &hold.head)
2083 && task.runs.last() == Some(&state.id)
2084 }) {
2085 let pass = state.passes.last().map_or(0, |p| p.index);
2088 if !task
2089 .passes
2090 .iter()
2091 .any(|p| p.run == state.id && p.pass_index == pass)
2092 {
2093 task.attempts = task.attempts.saturating_sub(1);
2094 }
2095 task.hold_machine(Some(format!(
2096 "[ci-unrelated] CI failure reproduced on base; owner question {}",
2097 hold.question
2098 )));
2099 task.last_error = Some(detail.to_owned());
2100 task.diagnostic = None;
2101 task.record_pass_at(
2102 &state.id,
2103 pass,
2104 state.passes.last().map(|p| p.started_at),
2105 attempts_before,
2106 Some(crate::run::ParkCause::Question),
2107 );
2108 return;
2109 }
2110 let unresolved = task.hold_notice_cause.clone();
2111 if state.base_sync_blocked() {
2112 task.hold_machine(Some(detail.to_owned()));
2114 task.last_error = Some(detail.to_owned());
2115 } else if !verdict.parked
2116 && task
2117 .pr_recovery
2118 .as_ref()
2119 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2120 && matches!(
2121 verdict.status,
2122 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2123 )
2124 {
2125 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2126 task.succeed();
2127 } else {
2128 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2129 }
2130 } else if !verdict.parked
2131 && task.pr_recovery.is_some()
2132 && matches!(
2133 verdict.status,
2134 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2135 )
2136 && !verdict.quota_hit
2137 {
2138 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2139 } else {
2140 settle_in(task, verdict, detail, max_attempts, p);
2141 }
2142 if !verdict.parked
2143 && !verdict.quota_hit
2144 && matches!(
2145 verdict.status,
2146 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2147 )
2148 {
2149 task.recovery_failed(state);
2150 }
2151 if state.prep_failure.is_some() {
2152 task.hold_notice_cause = unresolved;
2155 task.startup_notice_cause("startup-candidates");
2156 }
2157 if let Some(pass) = state.passes.last() {
2160 let parked = match &pass.end {
2161 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2162 _ => None,
2163 };
2164 task.record_pass_at(
2165 &state.id,
2166 pass.index,
2167 Some(pass.started_at),
2168 attempts_before,
2169 parked,
2170 );
2171 }
2172 if task.status == TaskStatus::Held {
2173 if state.base_sync_blocked() {
2174 task.base_sync_notice(state);
2175 } else if state.prep_failure.is_none() {
2176 task.notice_cause(
2177 label(verdict.status),
2178 Some(&state.id),
2179 state.passes.last().map(|p| p.index),
2180 );
2181 }
2182 task.diagnostic = diagnostic(state);
2183 note_open_question(task, &state.id, p);
2184 }
2185}
2186
2187fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2202 let Some(home) = crate::run::try_home() else {
2203 return;
2204 };
2205 let open = Questions::at(home.join("questions")).open_for(run);
2206 let Some(q) = open.first() else {
2207 return;
2208 };
2209 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2210 decisions.sort_unstable();
2211 task.notice_cause(
2212 &format!("questions:{}", decisions.join(",")),
2213 Some(run),
2214 None,
2215 );
2216 let base = task.hold_reason.clone().unwrap_or_default();
2217 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2218}
2219
2220fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2231 match last_run {
2232 Some(state) => {
2233 let verdict = Verdict {
2234 status: state.status,
2235 left_pr: state.pr.is_some(),
2236 quota_hit: state.quota.iter().any(|loss| {
2237 state
2238 .passes
2239 .last()
2240 .is_none_or(|pass| loss.at >= pass.started_at)
2241 }),
2242 parked: state.parked,
2243 no_viable_candidates: state.viable().is_empty(),
2244 };
2245 let detail = format!(
2246 "{}{}",
2247 phrases(&state.config.graph.language).recovered_running,
2248 describe(&state)
2249 );
2250 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2251 }
2252 None => {
2253 let why = phrases(language).no_run_to_recover;
2256 task.last_error = Some(why.to_owned());
2257 task.notice_cause("missing-run", None, None);
2260 task.hold_machine(Some(why.to_owned()));
2261 }
2262 }
2263}
2264
2265fn reclaim_orphaned_running(
2287 queue: &Queue,
2288 max_attempts: usize,
2289 live_workers: &std::collections::BTreeSet<String>,
2290) -> Vec<String> {
2291 let mut reclaimed = Vec::new();
2292 for listed in queue.list() {
2293 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2294 continue;
2295 }
2296 let Ok(_claim) = queue.claim(&listed.id) else {
2297 continue;
2298 };
2299 let Ok(mut task) = queue.get(&listed.id) else {
2303 continue;
2304 };
2305 if task.status != TaskStatus::Running {
2306 continue;
2307 }
2308 let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
2309 if let Some(state) = &last_run
2319 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2320 {
2321 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2322 }
2323 let language = if last_run.is_none() {
2324 language_of(&task, Path::new("."))
2325 } else {
2326 String::new()
2327 };
2328 reclaim(&mut task, last_run, max_attempts, &language);
2329 if task.status == TaskStatus::Done {
2330 supersede_prior_runs(&task, &crate::run::home());
2331 }
2332 record(queue, &mut task);
2333 reclaimed.push(task.id.clone());
2334 }
2335 reclaimed
2336}
2337
2338fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2370 reclaim_abandoned_runs_with(
2371 home,
2372 now,
2373 crate::proc::pid_status,
2374 crate::proc::process_started_at,
2375 )
2376}
2377
2378fn reclaim_abandoned_runs_with<F, G>(
2384 home: &Path,
2385 now: Timestamp,
2386 query: F,
2387 identity: G,
2388) -> Vec<String>
2389where
2390 F: Fn(u32) -> Option<bool>,
2391 G: Fn(u32) -> Option<String>,
2392{
2393 let mut abandoned = Vec::new();
2394 for entry in std::fs::read_dir(home.join("runs"))
2395 .into_iter()
2396 .flatten()
2397 .flatten()
2398 {
2399 let id = entry.file_name().to_string_lossy().into_owned();
2400 if !crate::run::is_run_id(&id) {
2401 continue;
2402 }
2403 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2406 continue;
2407 };
2408 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2416 continue;
2417 };
2418 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2419 continue;
2420 };
2421 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2422 continue;
2423 }
2424 let daemon_claims = is_working_on(home, &id, now);
2436 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2437 != crate::run::Liveness::Dead
2438 {
2439 continue;
2440 }
2441 state.abandon("daemon");
2442 if let Err(e) = state.save_under(home) {
2443 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2444 continue;
2445 }
2446 if let Some(notice) = notices::run_ended(&state) {
2449 notices::raise_in(home, notice);
2450 }
2451 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2459 tracing::warn!("abandon questions for {id}: {e:#}");
2460 }
2461 abandoned.push(id);
2462 }
2463 abandoned
2464}
2465
2466#[derive(Debug, Clone, Serialize, Deserialize)]
2467struct TaskWorker {
2468 opts: Opts,
2469 queue: PathBuf,
2470 task: String,
2471 #[serde(default)]
2472 lane: PermitKind,
2473 #[serde(default)]
2474 standalone: bool,
2475}
2476
2477fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2478 Ok(crate::worker::active(home)?
2479 .into_iter()
2480 .filter(|job| job.kind == "task")
2481 .collect())
2482}
2483
2484fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2485 task_jobs(home)?
2486 .into_iter()
2487 .filter_map(
2488 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2489 Ok(request) if !request.standalone => Some(Ok(job)),
2490 Ok(_) => None,
2491 Err(error) => Some(Err(error.into())),
2492 },
2493 )
2494 .collect()
2495}
2496
2497#[must_use]
2499pub fn managed(home: &Path) -> bool {
2500 match crate::worker::active(home) {
2501 Err(error) => {
2502 tracing::error!("cannot determine scheduler ownership: {error:#}");
2503 true
2504 }
2505 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2506 Ok(_) => {
2507 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2508 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2509 }
2510 }
2511}
2512
2513pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2515 anyhow::ensure!(
2516 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2517 "the scheduler is stopping; wait for retirement before starting it again"
2518 );
2519 crate::worker::spawn(
2520 home,
2521 "scheduler",
2522 "scheduler",
2523 serde_json::to_value(opts.absolute()?)?,
2524 )
2525 .await
2526}
2527
2528fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2529 home.join("worker-parks").join(format!("{}.json", job.id))
2530}
2531
2532fn request_task_park(
2533 home: &Path,
2534 job: &crate::worker::Job,
2535 cause: crate::run::ParkCause,
2536) -> Result<()> {
2537 let path = park_request_path(home, job);
2538 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2539 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2540 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2541}
2542
2543pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2545 if control == crate::worker::Control::Park {
2546 for job in owned_task_jobs(home)? {
2547 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2548 }
2549 }
2550 crate::worker::request(home, "scheduler", control)
2551}
2552
2553pub async fn restart_detached(home: &Path) -> Result<()> {
2555 let active = crate::worker::active(home)?;
2556 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2557 if prior.is_none() && !managed(home) {
2558 return Ok(());
2559 }
2560 let opts: Opts = match prior {
2561 Some(job) => serde_json::from_value(job.payload)?,
2562 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2563 };
2564 request_control(home, crate::worker::Control::Stop)?;
2565 loop {
2566 let reserved = crate::worker::active(home)?
2567 .iter()
2568 .any(|job| job.kind == "scheduler");
2569 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2570 break;
2571 }
2572 tokio::time::sleep(Duration::from_millis(100)).await;
2573 }
2574 spawn_detached(home, &opts).await?;
2575 Ok(())
2576}
2577
2578pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2580 let opts: Opts = serde_json::from_value(payload)?;
2581 let _ = home;
2582 serve_until(opts, Stop::new()).await
2583}
2584
2585pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2587 let request: TaskWorker = serde_json::from_value(payload)?;
2588 let queue = Queue::at(request.queue);
2589 let key = format!("task:{}", request.task);
2590 let job = task_jobs(home)?
2591 .into_iter()
2592 .find(|job| job.key == key)
2593 .context("task worker reservation is missing")?;
2594 let park_path = park_request_path(home, &job);
2595 let _claim = loop {
2598 match queue.claim(&request.task) {
2599 Ok(claim) => break claim,
2600 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2603 return Err(e.context("claim the task for its worker"));
2604 }
2605 Err(_) => {
2606 sweep_stale_claims(&queue, STALE_CLAIM);
2607 tokio::time::sleep(Duration::from_millis(50)).await;
2608 }
2609 }
2610 };
2611 let mut task = queue.get(&request.task)?;
2612 if !task.status.runnable()
2613 || (!request.standalone
2614 && request.lane
2615 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2616 {
2617 return Ok(());
2618 }
2619 let stop = Stop::new();
2620 let watcher = {
2621 let home = home.to_path_buf();
2622 let stop = stop.clone();
2623 let park_path = park_path.clone();
2624 tokio::spawn(async move {
2625 loop {
2626 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2627 let cause = std::fs::read(&park_path)
2628 .ok()
2629 .and_then(|body| serde_json::from_slice(&body).ok())
2630 .unwrap_or(crate::run::ParkCause::Operator);
2631 stop.park(cause);
2632 return;
2633 }
2634 tokio::time::sleep(Duration::from_millis(200)).await;
2635 }
2636 })
2637 };
2638 let quota = if request.standalone {
2639 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2640 } else {
2641 let status = Arc::new(Mutex::new(Status::new()));
2642 Box::pin(attempt(
2643 &request.opts,
2644 &queue,
2645 &status,
2646 &stop,
2647 crate::graph::Pause::new(),
2648 &mut task,
2649 ))
2650 .await
2651 };
2652 watcher.abort();
2653 let _ = std::fs::remove_file(park_path);
2654 persist_cooldown(home, "a).await?;
2655 Ok(())
2656}
2657
2658pub async fn spawn_claimed(
2661 home: &Path,
2662 opts: &Opts,
2663 queue: &Queue,
2664 task: &Task,
2665 claim: crate::queue::Claim,
2666) -> Result<crate::worker::Job> {
2667 let request = TaskWorker {
2668 opts: opts.absolute()?,
2669 queue: std::path::absolute(queue.root())?,
2670 task: task.id.clone(),
2671 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2672 standalone: true,
2673 };
2674 let job = crate::worker::spawn(
2675 home,
2676 "task",
2677 &format!("task:{}", task.id),
2678 serde_json::to_value(request)?,
2679 )
2680 .await?;
2681 drop(claim);
2682 Ok(job)
2683}
2684
2685pub async fn run_detached_claimed(
2688 opts: &Opts,
2689 queue: &Queue,
2690 task: &mut Task,
2691 claim: crate::queue::Claim,
2692) -> Result<()> {
2693 let home = crate::run::home();
2694 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2695 loop {
2696 crate::worker::active(&home)?;
2699 let current = crate::worker::get(&home, &job.id)?;
2700 if matches!(
2701 current.status,
2702 crate::worker::Status::Finished | crate::worker::Status::Failed
2703 ) {
2704 *task = queue.get(&task.id)?;
2705 anyhow::ensure!(
2706 current.status != crate::worker::Status::Failed,
2707 "task worker failed: {}",
2708 current.result.as_deref().unwrap_or("unknown failure")
2709 );
2710 return Ok(());
2711 }
2712 tokio::time::sleep(Duration::from_millis(200)).await;
2713 }
2714}
2715
2716async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2717 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2718 let _lease = loop {
2719 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2720 break lease;
2721 }
2722 tokio::time::sleep(Duration::from_millis(10)).await;
2723 };
2724 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2725 let path = home.join("task-quota-cooldown.json");
2726 let temporary = path.with_extension("json.tmp");
2727 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2728 std::fs::rename(temporary, path)?;
2729 }
2730 }
2731 Ok(())
2732}
2733
2734fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2735 match std::fs::read(home.join("task-quota-cooldown.json")) {
2736 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2737 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2738 Err(error) => Err(error.into()),
2739 }
2740}
2741
2742pub async fn serve(opts: Opts) -> Result<()> {
2748 tracing::info!(
2749 "automatic update and restart requires magi web; standalone serve does not restart"
2750 );
2751 serve_until(opts, Stop::new()).await
2752}
2753
2754pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2765 let opts = opts.absolute()?;
2766 let home = crate::run::home();
2767 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2768 .context("another scheduler already owns this home")?;
2769 if let Some(reading) = read_status(&home)
2770 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2771 {
2772 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2773 }
2774 if !crate::worker::active(&home)?
2775 .iter()
2776 .any(|job| job.kind == "scheduler")
2777 {
2778 crate::worker::clear_control(&home, "scheduler")?;
2779 }
2780 std::fs::write(
2781 home.join("scheduler-options.json"),
2782 serde_json::to_vec(&opts)?,
2783 )?;
2784 let control = {
2785 let home = home.clone();
2786 let stop = stop.clone();
2787 tokio::spawn(async move {
2788 loop {
2789 match crate::worker::control(&home, "scheduler").ok().flatten() {
2790 Some(crate::worker::Control::Park) => {
2791 let _ = request_control(&home, crate::worker::Control::Park);
2792 stop.park(crate::run::ParkCause::Operator);
2793 break;
2794 }
2795 Some(crate::worker::Control::Stop) => {
2796 stop.stop();
2797 break;
2798 }
2799 None => {}
2800 }
2801 tokio::time::sleep(Duration::from_millis(100)).await;
2802 }
2803 })
2804 };
2805 let signal = {
2806 let stop = stop.clone();
2807 tokio::spawn(async move {
2808 if tokio::signal::ctrl_c().await.is_ok() {
2809 stop.stop();
2810 tracing::info!("scheduler shutdown requested; independent workers continue");
2811 }
2812 })
2813 };
2814
2815 let worktrees_root = opts
2816 .worktrees_root
2817 .clone()
2818 .unwrap_or_else(crate::run::default_worktree_root);
2819 let outcome = drive(
2820 &opts,
2821 &Queue::open(),
2822 &status_path(),
2823 &crate::run::home(),
2824 &worktrees_root,
2825 &stop,
2826 )
2827 .await;
2828
2829 signal.abort();
2830 control.abort();
2831 outcome
2832}
2833
2834async fn drive(
2847 opts: &Opts,
2848 queue: &Queue,
2849 status_file: &Path,
2850 home: &Path,
2851 worktrees_root: &Path,
2852 stop: &Stop,
2853) -> Result<()> {
2854 let recovery_cfg = prepare(&opts.repo, opts).ok();
2862 let daemon_cfg = recovery_cfg
2863 .as_ref()
2864 .map(|c| c.daemon.clone())
2865 .unwrap_or_default();
2866 let reconciled = crate::driver::reconcile(home);
2867 if !reconciled.is_empty() {
2868 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2869 }
2870 let mut initial_status = Status::new();
2871 let owned = owned_task_jobs(home)?;
2872 initial_status.current = current_work(home, Timestamp::now())
2873 .into_iter()
2874 .filter(|current| {
2875 owned
2876 .iter()
2877 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2878 })
2879 .collect();
2880 initial_status.idle = initial_status.current.is_empty();
2881 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2882 recovery_cfg.as_ref(),
2883 daemon_cfg.max_deputies,
2884 ));
2885 let status = Arc::new(Mutex::new(initial_status));
2886 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2887 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2888
2889 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2895
2896 tracing::info!(
2897 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2898 queue.root().display(),
2899 opts.poll.as_secs(),
2900 opts.max_attempts,
2901 concurrency,
2902 if daemon_cfg.pause_for_interrupts {
2903 ", interrupts enabled"
2904 } else {
2905 ""
2906 }
2907 );
2908
2909 let outcome = tokio::select! {
2910 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2911 max_concurrent: concurrency,
2912 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2913 recovery_cfg,
2914 }) => outcome,
2915 () = async {
2916 while !stop.stopped() {
2917 tokio::time::sleep(Duration::from_millis(25)).await;
2918 }
2919 } => {
2920 if stop.parking() {
2921 request_control(home, crate::worker::Control::Park)?;
2922 }
2923 Ok(())
2924 }
2925 };
2926
2927 stop.busy.store(0, Ordering::SeqCst);
2928 beat.abort();
2929 clear_status_at(status_file);
2930 outcome
2931}
2932
2933pub(crate) fn release_watch_settings_from_config(
2936 home: &Path,
2937 repo: &Path,
2938 cfg: &Config,
2939) -> (Vec<PathBuf>, u64) {
2940 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2941 .into_iter()
2942 .map(|r| r.path)
2943 .collect();
2944 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2947 .list()
2948 .into_iter()
2949 .map(|t| t.repo)
2950 .collect();
2951 let runs = home.join("runs");
2952 for id in crate::run::list_ids_in(&runs) {
2953 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2954 .ok()
2955 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2956 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2957 known.extend(repo);
2958 }
2959 for k in known {
2960 let k = k.canonicalize().unwrap_or(k);
2961 if k.join(".git").exists() && !paths.contains(&k) {
2962 paths.push(k);
2963 }
2964 }
2965 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2966 if !paths.contains(&own) {
2967 paths.push(own);
2968 }
2969 (paths, cfg.daemon.release_stall_minutes)
2970}
2971
2972const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
2974
2975async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
2981 loop {
2982 tokio::time::sleep(HEARTBEAT).await;
2983 let snapshot = {
2984 let mut guard = lock(&status);
2985 guard.updated_at = Timestamp::now();
2986 guard.clone()
2987 };
2988 if let Err(e) = write_status_to(&path, &snapshot) {
2989 tracing::warn!("could not refresh the daemon status file: {e:#}");
2992 }
2993 }
2994}
2995
2996#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2999enum LandResume {
3000 NotLanding,
3003 StillWaiting,
3008 Ready,
3012}
3013
3014fn land_resume_state(task: &Task) -> LandResume {
3018 let Some(run_id) = task.runs.last() else {
3019 return LandResume::NotLanding;
3020 };
3021 let Ok(state) = RunState::load(run_id) else {
3022 return LandResume::NotLanding;
3023 };
3024 if !state.parked {
3025 return LandResume::NotLanding;
3026 }
3027 if state.status == RunStatus::Landing
3028 && state.land_approval.as_ref().is_some_and(|binding| {
3029 ask::Questions::open()
3030 .get(&binding.question)
3031 .is_ok_and(|q| {
3032 q.run == state.id
3033 && q.resolution()
3034 .as_deref()
3035 .is_some_and(crate::merge_decision::stops)
3036 })
3037 })
3038 {
3039 return LandResume::Ready;
3040 }
3041 if !implementers_match(task, &state) {
3042 return LandResume::NotLanding;
3043 }
3044 let node = if state.status == RunStatus::Landing {
3048 land::APPROVAL_NODE
3049 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
3050 crate::github_text::ASK_NODE
3051 } else {
3052 return LandResume::NotLanding;
3053 };
3054 let store = ask::Questions::open();
3055 let waiting = store
3056 .list()
3057 .into_iter()
3058 .filter(|q| &q.run == run_id && q.node == node)
3059 .max_by(|a, b| a.id.cmp(&b.id));
3060 let Some(mut q) = waiting else {
3061 return LandResume::Ready;
3062 };
3063 if !q.status.open() {
3064 if q.resolution().as_deref() == Some(land::REPAIR) {
3067 return if state.liveness(false) == Liveness::Live {
3068 LandResume::StillWaiting
3069 } else {
3070 LandResume::NotLanding
3071 };
3072 }
3073 return LandResume::Ready;
3074 }
3075 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
3082 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
3083 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3084 q.abandon(format!(
3085 "no answer within {}s of asking",
3086 timeout.as_secs().max(1)
3087 ));
3088 if store.put(&mut q).is_ok() {
3091 return LandResume::Ready;
3092 }
3093 }
3094 LandResume::StillWaiting
3095}
3096
3097const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3105
3106const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3118
3119#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3132enum Interrupt {
3133 Idle,
3135 Parking {
3147 parked: Vec<String>,
3148 interrupt_task: String,
3149 },
3150 Running {
3158 parked: Vec<String>,
3159 interrupt_task: String,
3160 },
3161 Resuming { parked: Vec<String> },
3168}
3169
3170fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3191 match state {
3192 Interrupt::Idle => {
3193 if in_flight.len() != 1 {
3205 return Interrupt::Idle;
3206 }
3207 match runnable.iter().find(|t| t.interrupt) {
3208 Some(t) => Interrupt::Parking {
3209 parked: in_flight.to_vec(),
3210 interrupt_task: t.id.clone(),
3211 },
3212 None => Interrupt::Idle,
3213 }
3214 }
3215 Interrupt::Parking {
3216 parked,
3217 interrupt_task,
3218 } => {
3219 if in_flight.iter().any(|id| parked.contains(id)) {
3220 Interrupt::Parking {
3222 parked,
3223 interrupt_task,
3224 }
3225 } else if in_flight.contains(&interrupt_task) {
3226 Interrupt::Running {
3227 parked,
3228 interrupt_task,
3229 }
3230 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3231 Interrupt::Parking {
3235 parked,
3236 interrupt_task,
3237 }
3238 } else {
3239 Interrupt::Resuming { parked }
3244 }
3245 }
3246 Interrupt::Running {
3247 parked,
3248 interrupt_task,
3249 } => {
3250 if in_flight.contains(&interrupt_task) {
3251 Interrupt::Running {
3252 parked,
3253 interrupt_task,
3254 }
3255 } else {
3256 Interrupt::Resuming { parked }
3262 }
3263 }
3264 Interrupt::Resuming { parked } => {
3265 if in_flight.iter().any(|id| parked.contains(id)) {
3266 Interrupt::Idle
3272 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3273 Interrupt::Resuming { parked }
3274 } else {
3275 Interrupt::Idle
3278 }
3279 }
3280 }
3281}
3282
3283fn advance_interrupt_tick(
3289 enabled: bool,
3290 state: Interrupt,
3291 in_flight: &[String],
3292 runnable: &[Task],
3293) -> Interrupt {
3294 if !enabled {
3295 return Interrupt::Idle;
3296 }
3297 advance_interrupt(state, in_flight, runnable)
3298}
3299
3300fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3305 match state {
3306 Interrupt::Idle => candidates,
3307 Interrupt::Parking {
3308 parked,
3309 interrupt_task,
3310 } => {
3311 if in_flight.iter().any(|id| parked.contains(id)) {
3312 Vec::new()
3313 } else {
3314 candidates
3315 .into_iter()
3316 .filter(|t| &t.id == interrupt_task)
3317 .collect()
3318 }
3319 }
3320 Interrupt::Running { .. } => Vec::new(),
3321 Interrupt::Resuming { parked } => candidates
3329 .into_iter()
3330 .find(|t| parked.contains(&t.id))
3331 .into_iter()
3332 .collect(),
3333 }
3334}
3335
3336struct DispatchLimits {
3340 max_concurrent: usize,
3343 pause_for_interrupts: bool,
3345 recovery_cfg: Option<Config>,
3346}
3347
3348#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3357enum PermitKind {
3358 None,
3363 Urgent,
3370 #[default]
3373 Ordinary,
3374}
3375
3376fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3379 if priority {
3380 PermitKind::None
3381 } else if urgent {
3382 PermitKind::Urgent
3383 } else {
3384 PermitKind::Ordinary
3385 }
3386}
3387
3388fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3389 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3390 (0usize, 0usize),
3391 |(ordinary, urgent), request| match request.lane {
3392 PermitKind::Ordinary => (ordinary + 1, urgent),
3393 PermitKind::Urgent => (ordinary, urgent + 1),
3394 PermitKind::None => (ordinary, urgent),
3395 },
3396 );
3397 (
3398 limit.saturating_sub(ordinary),
3399 1usize.saturating_sub(urgent),
3400 )
3401}
3402
3403struct DispatchClaim {
3404 task: Task,
3405 claim: crate::queue::Claim,
3406 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3407}
3408
3409fn claim_dispatch(
3412 queue: &Queue,
3413 candidate: &Task,
3414 cooling_down: bool,
3415 ordinary: &Arc<tokio::sync::Semaphore>,
3416 urgent: &Arc<tokio::sync::Semaphore>,
3417) -> Option<DispatchClaim> {
3418 let Ok(claim) = queue.claim(&candidate.id) else {
3419 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3420 return None;
3421 };
3422 let task = match queue.get(&candidate.id) {
3423 Ok(t) if t.status.runnable() => t,
3424 Ok(_) => return None,
3425 Err(e) => {
3426 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3427 return None;
3428 }
3429 };
3430 let resume = land_resume_state(&task);
3431 if resume == LandResume::StillWaiting {
3432 return None;
3433 }
3434 let priority = resume == LandResume::Ready;
3435 if !priority && cooling_down {
3436 return None;
3437 }
3438 let permit = match permit_kind(priority, task.urgent) {
3441 PermitKind::None => None,
3442 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3443 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3444 };
3445 let stamp = Timestamp::now();
3448 let mut task = task;
3449 task.dispatched_at = Some(stamp);
3450 if let Err(e) = queue.modify(&task.id, |t| {
3451 t.dispatched_at = Some(stamp);
3452 true
3453 }) {
3454 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3455 }
3456 Some(DispatchClaim {
3457 task,
3458 claim,
3459 permit,
3460 })
3461}
3462
3463#[derive(Serialize, Deserialize)]
3482struct ServiceWorker {
3483 opts: Opts,
3484 queue: PathBuf,
3485 worktrees_root: PathBuf,
3486 operation: String,
3487}
3488
3489fn load_conductor(home: &Path) -> Result<Conductor> {
3490 match std::fs::read(home.join("conductor-state.json")) {
3491 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3492 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3493 Err(error) => Err(error.into()),
3494 }
3495}
3496
3497fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3498 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3499 .into_iter()
3500 .map(|c| c.task)
3501 .collect();
3502 let stalled = stalled_tasks(queue, home, Timestamp::now());
3503 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3504 let finished = finished_tasks(queue)
3505 .into_iter()
3506 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3507 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3508 .collect();
3509 let queued = queued_tasks(queue)
3510 .into_iter()
3511 .filter(|t| !live.contains(&t.id))
3512 .collect();
3513 (queued, stalled, finished)
3514}
3515
3516async fn start_service(
3517 home: &Path,
3518 opts: &Opts,
3519 queue: &Queue,
3520 worktrees_root: &Path,
3521 operation: &str,
3522 cfg: Option<&Config>,
3523 busy: bool,
3524) -> Result<()> {
3525 let fetch_cfg = if operation == "fetch" {
3528 match prepare(&opts.repo, opts) {
3529 Ok(cfg) => Some(cfg),
3530 Err(error) => {
3531 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3532 return Ok(());
3533 }
3534 }
3535 } else {
3536 None
3537 };
3538 let cfg = fetch_cfg.as_ref().or(cfg);
3539 let ticks = home.join("service-ticks");
3540 std::fs::create_dir_all(&ticks)?;
3541 let receipt = ticks.join(format!("daemon-{operation}"));
3542 let last = std::fs::read(&receipt)
3543 .ok()
3544 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3545 let interval = match operation {
3546 "disk-pressure" => 30,
3547 "cache" => CACHE_CHECK_INTERVAL_SECS,
3548 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3549 _ => opts.poll.as_secs().max(1),
3550 };
3551 let now = Timestamp::now();
3552 if !cache_check_due(last, now, interval) {
3553 return Ok(());
3554 }
3555 let key = format!("service:daemon-{operation}");
3556 if crate::worker::active(home)?
3557 .iter()
3558 .any(|job| job.key == key)
3559 {
3560 return Ok(());
3561 }
3562 let eligible = match operation {
3563 "conductor" => {
3564 let (queued, stalled, finished) = conductor_work(queue, home);
3565 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3566 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3567 }
3568 "maintenance" => Questions::at(home.join("questions"))
3569 .list()
3570 .iter()
3571 .any(|q| {
3572 q.chosen_action().is_some_and(|action| action.maintenance())
3573 && !home
3574 .join("choice-actions")
3575 .join(format!("{}.json", q.id))
3576 .exists()
3577 }),
3578 "housekeeping" => !busy && runnable(queue).is_empty(),
3579 "cache" => {
3580 !busy
3581 && !runnable(queue).is_empty()
3582 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3583 }
3584 "disk-pressure" => busy,
3585 "fetch" => {
3586 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3587 }
3588 _ => false,
3589 };
3590 if !eligible {
3591 return Ok(());
3592 }
3593 let request = ServiceWorker {
3594 opts: opts.absolute()?,
3595 queue: std::path::absolute(queue.root())?,
3596 worktrees_root: std::path::absolute(worktrees_root)?,
3597 operation: operation.to_owned(),
3598 };
3599 if let Err(error) =
3600 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3601 {
3602 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3603 }
3604 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3605 Ok(())
3606}
3607
3608pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3611 let request: ServiceWorker = serde_json::from_value(payload)?;
3612 let opts = &request.opts;
3613 let queue = Queue::at(request.queue);
3614 let questions = Questions::at(home.join("questions"));
3615 match request.operation.as_str() {
3616 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3617 "conductor" => {
3618 let path = home.join("conductor-state.json");
3619 let mut conductor = load_conductor(home)?;
3620 let (queued, stalled, finished) = conductor_work(&queue, home);
3621 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3622 && conductor.worth_a_look(&queue, &stalled, &finished)
3623 {
3624 let cfg = prepare(&opts.repo, opts)?;
3625 conductor
3626 .maybe_run(
3627 &cfg,
3628 &opts.repo,
3629 &queue,
3630 &questions,
3631 home,
3632 &queued,
3633 &stalled,
3634 &finished,
3635 opts.max_attempts,
3636 )
3637 .await;
3638 let temporary = path.with_extension("json.tmp");
3639 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3640 std::fs::rename(temporary, path)?;
3641 }
3642 }
3643 "fetch" => {
3644 let cfg = prepare(&opts.repo, opts)?;
3645 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3646 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3647 tracing::debug!("fetch origins: {outcome:?}");
3648 }
3649 }
3650 "cache" => {
3651 if task_jobs(home)?.is_empty() {
3652 let mut checked = None;
3653 maybe_prune_cache_between_runs(
3654 &opts.repo,
3655 opts,
3656 home,
3657 &Stop::new(),
3658 &mut checked,
3659 Timestamp::now(),
3660 )
3661 .await;
3662 }
3663 }
3664 "disk-pressure" => {
3665 if let Ok(cfg) = prepare(&opts.repo, opts) {
3666 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3667 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3668 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3669 }
3670 for current in current_work(home, Timestamp::now()) {
3671 if let Ok(state) = RunState::load_under(¤t.run, home)
3672 && let Some(reason) = disk_gate(&state.repo, &state.config)
3673 {
3674 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3675 job.key.strip_prefix("task:") == Some(current.task.as_str())
3676 }) {
3677 request_task_park(
3678 home,
3679 &job,
3680 crate::run::ParkCause::Interrupt {
3681 reason: reason.clone(),
3682 },
3683 )?;
3684 }
3685 notices::raise_in(
3686 home,
3687 Notice::warn(
3688 "disk:pressure",
3689 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3690 ),
3691 );
3692 }
3693 }
3694 }
3695 }
3696 "housekeeping" => {
3697 if task_jobs(home)?.is_empty() {
3698 if let Ok(cfg) = prepare(&opts.repo, opts) {
3699 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3700 }
3701 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3702 resweep_superseded_attempts(&queue, home);
3703 triage_held(&queue, home, opts).await;
3704 }
3705 }
3706 operation => bail!("unknown daemon service {operation}"),
3707 }
3708 Ok(())
3709}
3710
3711fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3715 let key = format!("task:{}", request.task);
3716 let payload = serde_json::to_value(request)?;
3717 if crate::worker::active(home)?
3718 .iter()
3719 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3720 {
3721 tracing::warn!(
3722 "task {} acquired a competing worker reservation: {error:#}",
3723 crate::run::short_of(&request.task)
3724 );
3725 return Ok(());
3726 }
3727 tracing::error!("task worker startup stopped dispatch: {error:#}");
3728 notices::raise_in(
3729 home,
3730 Notice::error(
3731 "loop:worker-startup",
3732 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
3735 ).link(Link::Task { id: request.task.clone() }),
3736 );
3737 Err(error).with_context(|| {
3738 format!(
3739 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3740 crate::run::short_of(&request.task)
3741 )
3742 })
3743}
3744
3745async fn poll(
3746 opts: &Opts,
3747 queue: &Queue,
3748 status: &Arc<Mutex<Status>>,
3749 home: &Path,
3750 worktrees_root: &Path,
3751 stop: &Stop,
3752 limits: DispatchLimits,
3753) -> Result<()> {
3754 let mut attempted = Vec::new();
3755 let mut settling_once_services = false;
3756 let interrupt_path = home.join("scheduler-interrupt.json");
3757 let mut interrupt = match std::fs::read(&interrupt_path) {
3758 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3759 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3760 Err(error) => return Err(error.into()),
3761 };
3762 let mut previous = std::collections::BTreeSet::<String>::new();
3763 while !stop.stopped() {
3764 let jobs: Vec<_> = crate::worker::active(home)?
3765 .into_iter()
3766 .filter(|job| job.kind == "task")
3767 .collect();
3768 let requests: Vec<_> = jobs
3769 .iter()
3770 .map(|job| {
3771 serde_json::from_value::<TaskWorker>(job.payload.clone())
3772 .with_context(|| format!("read live task worker {}", job.id))
3773 })
3774 .collect::<Result<_>>()?;
3775 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3776 let owned: std::collections::BTreeSet<_> = requests
3777 .iter()
3778 .filter(|r| !r.standalone)
3779 .map(|r| r.task.clone())
3780 .collect();
3781 let current: Vec<_> = requests
3782 .iter()
3783 .zip(&jobs)
3784 .filter(|(request, _)| !request.standalone)
3785 .map(|(request, job)| Current {
3786 worker: Some(job.id.clone()),
3787 task: request.task.clone(),
3788 run: queue
3789 .get(&request.task)
3790 .ok()
3791 .and_then(|task| task.runs.last().cloned())
3792 .unwrap_or_default(),
3793 })
3794 .collect();
3795 {
3796 let mut snapshot = lock(status);
3797 snapshot.polls += 1;
3798 snapshot.completed += previous.difference(&owned).count();
3799 snapshot.current = current;
3800 snapshot.idle = owned.is_empty();
3801 }
3802 previous = owned.clone();
3803 stop.busy.store(owned.len(), Ordering::SeqCst);
3804
3805 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3808 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3809 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3810 let mut permits = Vec::new();
3811 sweep_stale_claims(queue, STALE_CLAIM);
3812 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3813 reclaim_abandoned_runs(home, Timestamp::now());
3814 let questions = Questions::at(home.join("questions"));
3815
3816 crate::task_chat::sweep(
3819 queue,
3820 &crate::talk::Talks::at(home.join("talks")),
3821 &questions,
3822 home,
3823 );
3824 repair_choice_offers(&questions);
3825 resolve_blockers(queue, &questions);
3826 apply_task_choice_actions(queue, &questions, home);
3827 triage::apply_answers(queue, &questions);
3828 reconcile_task_questions(queue, &questions);
3829 let talks = crate::talk::Talks::at(home.join("talks"));
3830 crate::chat_notify::sweep(queue, &talks, home);
3831 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
3832 crate::chat_watch::sweep(home, &talks);
3833 let mut turns = tokio::task::JoinSet::new();
3834 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3835 while turns.join_next().await.is_some() {}
3837
3838 crate::consult::start_services(
3839 home,
3840 &opts.repo,
3841 limits.recovery_cfg.as_ref(),
3842 opts.config.as_deref(),
3843 )
3844 .await?;
3845 for operation in [
3846 "conductor",
3847 "maintenance",
3848 "housekeeping",
3849 "disk-pressure",
3850 "cache",
3851 "fetch",
3852 ] {
3853 if settling_once_services {
3857 break;
3858 }
3859 start_service(
3860 home,
3861 opts,
3862 queue,
3863 worktrees_root,
3864 operation,
3865 limits.recovery_cfg.as_ref(),
3866 !live.is_empty(),
3867 )
3868 .await?;
3869 }
3870 let latest = crate::worker::active(home)?;
3871 if latest.iter().any(|job| job.key == "service:daemon-cache")
3872 || latest
3873 .iter()
3874 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3875 {
3876 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3877 continue;
3878 }
3879 let candidates: Vec<Task> = runnable(queue)
3880 .into_iter()
3881 .filter(|task| !live.contains(&task.id))
3882 .filter(|task| !opts.once || !attempted.contains(&task.id))
3883 .collect();
3884 let in_flight: Vec<_> = owned.iter().cloned().collect();
3885 let prior_interrupt = std::mem::discriminant(&interrupt);
3886 interrupt = advance_interrupt_tick(
3887 limits.pause_for_interrupts,
3888 interrupt,
3889 &in_flight,
3890 &candidates,
3891 );
3892 if prior_interrupt != std::mem::discriminant(&interrupt) {
3895 let temporary = interrupt_path.with_extension("json.tmp");
3896 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3897 std::fs::rename(temporary, &interrupt_path)?;
3898 }
3899 if let Interrupt::Parking {
3900 parked,
3901 interrupt_task,
3902 } = &interrupt
3903 {
3904 for job in jobs.iter().filter(|job| {
3905 parked
3906 .iter()
3907 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3908 }) {
3909 if crate::worker::control(home, &job.key)?.is_none() {
3910 request_task_park(
3911 home,
3912 job,
3913 crate::run::ParkCause::Interrupt {
3914 reason: format!(
3915 "task {} asked to run first",
3916 crate::run::short_of(interrupt_task)
3917 ),
3918 },
3919 )?;
3920 }
3921 }
3922 }
3923 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3924 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3925 let mut started = false;
3926 let mut resync = false;
3927 for candidate in candidates {
3928 if stop.stopped() {
3929 break;
3930 }
3931 let Some(DispatchClaim {
3932 task,
3933 claim,
3934 permit,
3935 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3936 else {
3937 continue;
3938 };
3939 if crate::worker::active(home)?
3940 .iter()
3941 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3942 {
3943 drop(claim);
3944 resync = true;
3945 break;
3946 }
3947 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3948 let request = TaskWorker {
3949 opts: opts.absolute()?,
3950 queue: std::path::absolute(queue.root())?,
3951 task: task.id.clone(),
3952 lane,
3953 standalone: false,
3954 };
3955 if let Err(error) = crate::worker::spawn(
3958 home,
3959 "task",
3960 &format!("task:{}", task.id),
3961 serde_json::to_value(&request)?,
3962 )
3963 .await
3964 {
3965 task_worker_spawn_error(home, &request, error)?;
3966 drop(claim);
3967 resync = true;
3968 break;
3969 }
3970 drop(claim);
3971 permits.extend(permit);
3972 attempted.push(task.id);
3973 started = true;
3974 }
3975 if started || resync {
3976 continue;
3977 }
3978 if owned.is_empty() && opts.once {
3979 settling_once_services = true;
3980 if crate::worker::active(home)?.iter().any(|job| {
3984 job.kind == "daemon-service"
3985 && matches!(
3986 job.key.as_str(),
3987 "service:daemon-housekeeping"
3988 | "service:daemon-conductor"
3989 | "service:daemon-maintenance"
3990 )
3991 }) {
3992 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3993 continue;
3994 }
3995 break;
3996 }
3997 stop.idle(if live.is_empty() {
3998 opts.poll
3999 } else {
4000 RECHECK_WHILE_BUSY.min(opts.poll)
4001 })
4002 .await;
4003 }
4004 stop.busy.store(0, Ordering::SeqCst);
4007 Ok(())
4008}
4009
4010async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
4014 let busy = |error: &anyhow::Error| {
4015 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
4016 error.kind() == std::io::ErrorKind::WouldBlock
4017 || cfg!(windows) && error.raw_os_error() == Some(33)
4018 })
4019 };
4020 match crate::pr_head::owner_lock(home, id) {
4021 Ok(lock) => return Ok(lock),
4022 Err(error) if busy(&error) => {}
4023 Err(error) => return Err(error),
4024 }
4025 let wait = RunState::load_under(id, home)?
4026 .config
4027 .graph
4028 .timeout_resource_wait;
4029 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
4030}
4031
4032async fn public_resume_owner(
4036 home: &Path,
4037 runs: &[String],
4038 repo: &Path,
4039) -> Result<Option<RunState>> {
4040 let mut owner = None;
4041 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
4042 let _lock = resume_owner_lock(home, id).await?;
4043 let state = RunState::load_under(id, home)?;
4044 if state.released()
4045 || state.retired_to.is_some()
4046 || state.pr_taken_over_by.is_some()
4047 || !crate::handover::landing_checkpoint(&state)
4048 {
4049 continue;
4050 }
4051 anyhow::ensure!(
4052 crate::git::toplevel(&state.repo).await?.canonicalize()?
4053 == crate::git::toplevel(repo).await?.canonicalize()?,
4054 "public owner repository differs; inspect run lineage"
4055 );
4056 anyhow::ensure!(
4057 owner.is_none(),
4058 "multiple retained public owners; inspect run lineage"
4059 );
4060 if let Some(reason) = crate::handover::resume_refusal_under(&state, Some(home)) {
4061 anyhow::bail!(reason.diagnostic(&state));
4062 }
4063 let claimed = is_working_on_other(home, id, Timestamp::now())?;
4064 anyhow::ensure!(
4065 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
4066 "public owner's driver is live or unknown; wait before retrying"
4067 );
4068 owner = Some(state);
4069 }
4070 Ok(owner)
4071}
4072
4073async fn attempt(
4079 opts: &Opts,
4080 queue: &Queue,
4081 status: &Arc<Mutex<Status>>,
4082 stop: &Stop,
4083 interrupt_pause: crate::graph::Pause,
4084 task: &mut Task,
4085) -> Vec<QuotaLoss> {
4086 let repo = repo_for(task, &opts.repo);
4087 tracing::info!(
4088 "task {} — {} (repo {})",
4089 task.short(),
4090 task.title,
4091 repo.display()
4092 );
4093
4094 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
4095 Ok(owner) => owner,
4096 Err(e) => {
4097 task.hold_machine(Some(format!(
4098 "Cannot verify existing public owner: {e:#}; no attempt started."
4099 )));
4100 record(queue, task);
4101 return Vec::new();
4102 }
4103 };
4104 let resume_runs = public_owner
4105 .as_ref()
4106 .map(|s| vec![s.id.clone()])
4107 .unwrap_or_else(|| task.runs.clone());
4108
4109 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4113 && previous
4114 .merge_decision
4115 .as_ref()
4116 .is_some_and(|decision| !decision.closed)
4117 && previous.status != RunStatus::Landing
4118 && !crate::merge_decision::resumable_hold(&previous)
4119 {
4120 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4121 record(queue, task);
4122 return Vec::new();
4123 }
4124 let stop_owner = resume_runs
4125 .last()
4126 .and_then(|id| RunState::load(id).ok())
4127 .filter(|state| {
4128 crate::merge_decision::resumable_hold(state)
4129 || state.status == RunStatus::Landing
4130 && state.land_approval.as_ref().is_some_and(|binding| {
4131 Questions::open().get(&binding.question).is_ok_and(|q| {
4132 q.run == state.id
4133 && q.resolution()
4134 .as_deref()
4135 .is_some_and(crate::merge_decision::stops)
4136 })
4137 })
4138 });
4139 if let Some(owner) = stop_owner {
4140 let mut runner = match Runner::resume(&owner.id) {
4141 Ok(runner) => runner,
4142 Err(e) => {
4143 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4144 record(queue, task);
4145 return Vec::new();
4146 }
4147 };
4148 runner.on_pause(stop.pause());
4149 runner.watch_interrupt(interrupt_pause);
4150 let quota_before = runner.state.quota.clone();
4151 task.start(owner.id.clone());
4152 record(queue, task);
4153 lock(status).current.push(Current {
4154 worker: None,
4155 task: task.id.clone(),
4156 run: owner.id,
4157 });
4158 runner.state.next_pass_task_entry = true;
4159 let result = runner.execute().await;
4160 return finish_attempt(
4161 opts.max_attempts,
4162 queue,
4163 task,
4164 &runner.state,
4165 "a_before,
4166 result,
4167 );
4168 }
4169
4170 if let Some(owner) = &public_owner
4171 && (task.fresh_start || !may_resume_run(owner, task))
4172 {
4173 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4174 record(queue, task);
4175 return Vec::new();
4176 }
4177
4178 if let Some(held) = crate::base_sync_recovery::held_run(queue, task)
4179 && !task.fresh_start
4180 && task.review_branch.is_none()
4181 && task.pr_recovery.is_none()
4182 {
4183 let authorized = if task.resumption.is_some() && compatible_run_instruction(&held, task) {
4186 let home = queue.root().parent().expect("queue has a home");
4187 match Runner::authorize_base_recovery_from_resumption(home, &held.id, &task.id).await {
4188 Ok(()) => true,
4189 Err(error) => {
4190 tracing::debug!(run = %held.id, %error, "base recovery resumption refused");
4191 false
4192 }
4193 }
4194 } else {
4195 false
4196 };
4197 if !authorized {
4198 task.hold_machine(Some("Base synchronization remains stopped. Inspect the retained run; for movement exhaustion, explicitly use `magi run --resume <run> --recover-base`. A release without a new owner approval grants no additional rebase and starts no replacement competition.".into()));
4199 task.base_sync_notice(&held);
4201 task.resume_override = None;
4202 record(queue, task);
4203 return Vec::new();
4204 }
4205 }
4206 let unfinished = (!task.fresh_start)
4207 .then(|| unfinished_run(&resume_runs, task.short()))
4208 .flatten()
4209 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4210 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4211 if !task.fresh_start
4212 && task.pr_recovery.is_none()
4213 && task.review_branch.is_none()
4214 && task.resume_override.is_none()
4215 && previous
4216 .as_ref()
4217 .is_some_and(|run| run.retired_to.is_some())
4218 && unfinished.is_none()
4219 {
4220 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4221 record(queue, task);
4222 return Vec::new();
4223 }
4224
4225 let saved_recovery = previous.as_ref().filter(|run| {
4226 task.replacement.is_none()
4227 && task.pr_recovery.as_ref().is_some_and(|selected| {
4228 selected.check_task(&task.id, &task.instruction).is_ok()
4229 && selected_starter(selected, Some(run), unfinished.as_deref())
4230 == Starter::Resume(run.id.clone())
4231 })
4232 });
4233
4234 let explicit = match &task.overrides {
4238 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4239 Some(_) => opts.config.as_deref(),
4240 None => opts.config.as_deref(),
4241 };
4242 let (mut config, layers) = if let Some(run) = saved_recovery {
4243 let mut saved = run.clone();
4246 if let Err(e) = saved.configure_recovery_resume() {
4247 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4248 record(queue, task);
4249 return Vec::new();
4250 }
4251 (saved.config, Vec::new())
4252 } else {
4253 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4254 Ok(loaded) => loaded,
4255 Err(e) => {
4256 let reason = format!("[config] {e:#}");
4257 record_config_failure(task, &e);
4258 task.last_error = Some(reason.clone());
4259 task.hold_machine(Some(reason));
4260 record(queue, task);
4261 return Vec::new();
4262 }
4263 };
4264 if let Err(e) = apply_options(&mut config, opts, task) {
4265 task.hold_manual(Some(format!(
4266 "[config] {e:#}; correct the merge override before releasing the task"
4267 )));
4268 record(queue, task);
4269 return Vec::new();
4270 }
4271 if let Some(selected) = &task.pr_recovery
4272 && let Err(e) = selected.configure(&mut config)
4273 {
4274 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4275 record(queue, task);
4276 return Vec::new();
4277 }
4278 (config, layers)
4279 };
4280 let config_evidence = crate::pr_recovery::ConfigEvidence {
4281 layers,
4282 overrides: task.overrides.clone(),
4283 daemon_merge: task
4284 .overrides
4285 .as_ref()
4286 .is_none_or(|o| o.inherit_daemon_options)
4287 .then(|| opts.merge.clone())
4288 .flatten(),
4289 };
4290 apply_task_implementers(&mut config, task);
4291 if let Some(replacement) = &task.replacement {
4292 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4293 old.liveness(false) == Liveness::Dead
4294 && old.merge_decision.as_ref().is_some_and(|d| {
4295 d.closed
4296 && d.question == replacement.question
4297 && d.replacement_run.as_ref().is_none_or(|run| {
4298 task.runs.last() == Some(run)
4299 && RunState::load(run).is_ok_and(|current| {
4300 current.replaces.as_ref().is_some_and(|pin| {
4301 pin.question == replacement.question
4302 && pin.agent == replacement.agent
4303 })
4304 })
4305 })
4306 })
4307 });
4308 if !safe {
4309 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4310 record(queue, task);
4311 return Vec::new();
4312 }
4313 match config.agent(&replacement.agent) {
4314 Ok(agent) if crate::agent::installed(agent) => {}
4315 _ => {
4316 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4317 record(queue, task);
4318 return Vec::new();
4319 }
4320 }
4321 config.graph.implementers = 1;
4322 config.graph.land_approval = true;
4323 config.merge.mode = MergeMode::Pr;
4324 config.graph.selected_implementer = Some(replacement.agent.clone());
4325 }
4326 let p = phrases(&config.graph.language);
4327 let start_failed = p.could_not_start;
4328 if let Ok(base) =
4333 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4334 .await
4335 && let Ok(head) =
4336 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4337 {
4338 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4339 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4340 let source = crate::git::rev_parse(&repo, branch)
4341 .await
4342 .unwrap_or_default();
4343 inputs.push_str(&format!(";review:{branch}:{source}"));
4344 }
4345 if let Some(reason) = task.recovery_exhausted(
4346 &head,
4347 &inputs,
4348 task.attempt_limit
4349 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4350 ) {
4351 task.hold_unrequested_retry(Some(reason));
4352 record(queue, task);
4353 return Vec::new();
4354 }
4355 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4356 }
4357
4358 if let Some(reason) = disk_gate(&repo, &config) {
4366 task.notice_cause("disk-space", None, None);
4367 task.last_error = Some(reason.clone());
4368 task.hold_machine(Some(reason.clone()));
4369 record(queue, task);
4370 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4371 notices::raise(
4374 Notice::warn(
4375 &format!("disk:{}", repo.display()),
4376 format!("Task {} held for want of free disk space: {reason}. Free some, then release it from the queue.", task.id),
4377 )
4378 .link(Link::Task {
4379 id: task.id.clone(),
4380 }),
4381 );
4382 return Vec::new();
4383 }
4384
4385 let review_branch = take_review_branch(task, previous.as_ref());
4410 let branch_exists = match &review_branch {
4411 Some(branch) => crate::git::branch_exists(&repo, branch)
4412 .await
4413 .unwrap_or(false),
4414 None => false,
4415 };
4416 let proposed = review_retry_starter(
4420 task,
4421 previous.as_ref(),
4422 review_branch.as_deref(),
4423 branch_exists,
4424 unfinished.as_deref(),
4425 );
4426 let retry_selected = task.replacement.is_some()
4427 && task.resumption.is_some()
4428 && !task.fresh_start
4429 && previous.as_ref().is_some_and(|run| {
4430 run.replaces.is_some()
4431 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4432 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4433 });
4434 let stop_pin = task
4435 .resume_override
4436 .as_ref()
4437 .filter(|pin| {
4438 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4439 q.node == land::APPROVAL_NODE
4440 && q.resolution()
4441 .as_deref()
4442 .is_some_and(crate::merge_decision::stops)
4443 })
4444 })
4445 .and_then(|pin| pin.pinned_run.clone());
4446 let starter = if let Some(owner) = &public_owner {
4447 Starter::Resume(owner.id.clone())
4448 } else if let Some(run) = stop_pin {
4449 Starter::Resume(run)
4452 } else if retry_selected {
4453 Starter::Resume(previous.as_ref().unwrap().id.clone())
4454 } else if let Some(selected) = &task.pr_recovery {
4455 if selected.check_task(&task.id, &task.instruction).is_ok() {
4456 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4457 } else {
4458 Starter::Review(selected.branch.clone())
4459 }
4460 } else {
4461 starter_with_implementers(task, previous.as_ref(), proposed)
4462 };
4463 if let Some(pin) = &task.resume_override
4466 && let Some(run) = &pin.pinned_run
4467 && starter != Starter::Resume(run.clone())
4468 {
4469 let run = run.clone();
4470 task.notice_cause("resume-owner", Some(&run), None);
4471 task.hold_machine(Some(format!(
4472 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4473 task.id
4474 )));
4475 record(queue, task);
4476 return Vec::new();
4477 }
4478 if task.replacement.is_some()
4479 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4480 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4481 {
4482 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4483 record(queue, task);
4484 return Vec::new();
4485 }
4486 if let Some(reason) = retained_attempt_hold(
4487 task,
4488 &starter,
4489 config.disk.retained_attempt_limit,
4490 RunState::load,
4491 ) {
4492 if let Starter::Review(branch) = &starter {
4495 task.review_branch = Some(branch.clone());
4496 }
4497 task.hold_unrequested_retry(Some(reason));
4498 record(queue, task);
4499 return Vec::new();
4500 }
4501 let attachments = match task_attachments(queue, task) {
4506 Ok(a) => a,
4507 Err(e) => {
4508 task.attempts += 1;
4509 task.fail_startup_target(
4510 "startup-attachments",
4511 e.name.clone(),
4512 format!("{start_failed}{e:#}"),
4513 opts.max_attempts,
4514 );
4515 record(queue, task);
4516 return Vec::new();
4517 }
4518 };
4519 let startup_pause = stop.pause();
4520 let startup_interrupt = interrupt_pause.clone();
4521 let wait = crate::repo_lock::WaitPolicy::new(
4522 Duration::from_secs(config.graph.timeout_resource_wait),
4523 move || startup_pause.parked() || startup_interrupt.parked(),
4524 );
4525 let started = wait.scope(Box::pin(async { match &starter {
4528 Starter::Review(_) if task.pr_recovery.is_some() => {
4529 let selected = task.pr_recovery.clone().expect("selected recovery");
4530 match selected.check_task(&task.id, &task.instruction) {
4531 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4532 .await
4533 .map(|mut r| {
4534 r.state.attachments = attachments.clone();
4535 r
4536 }),
4537 Err(e) => Err(e),
4538 }
4539 }
4540 Starter::Review(branch) => {
4541 tracing::info!(
4542 "task {} reopens `{branch}` as a review-only pass",
4543 task.short()
4544 );
4545 let takeover = crate::handover::Takeover {
4548 earlier: task.earlier_attempts().to_vec(),
4549 home: crate::run::home(),
4550 choice: take_divergence_answer(branch, &config.merge.remote, task),
4551 task_instruction: Some(instruction_for(task)),
4552 };
4553 Runner::review_taking_over(
4554 &repo,
4555 branch,
4556 config,
4557 Some(takeover),
4558 crate::run::Origin::queue(&task.id),
4559 )
4560 .await
4561 }
4562 Starter::Resume(id) => {
4563 tracing::info!("resuming run {id} rather than competing again");
4564 Runner::resume(id).and_then(|mut r| {
4565 if retry_selected {
4566 for candidate in &mut r.state.candidates {
4569 candidate.failed = None;
4570 candidate.empty = false;
4571 candidate.commits = 0;
4572 }
4573 r.state.status = RunStatus::Implementing;
4574 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4575 }
4576 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4577 r.state.sources = task.sources.clone();
4578 }
4579 if r.state.prep_failure.is_some() {
4580 let approval = task
4581 .resumption
4582 .as_ref()
4583 .context("preparation retry requires an explicit task release")?;
4584 r.retry_preparation(approval.at)?;
4585 }
4586 if let Some(instruction) =
4587 prepare_instruction(&starter, Some(&r.state.instruction), task)
4588 {
4589 r.state.instruction = instruction;
4590 }
4591 r.state.attachments = attachments.clone();
4592 if task.replacement.is_some() {
4593 r.state.config.merge.mode = MergeMode::Pr;
4594 r.state.config.graph.land_approval = true;
4595 }
4596 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4600 .overrides
4601 .as_ref()
4602 .and_then(|o| o.merge.as_deref())
4603 .and_then(|m| merge_mode(m).ok())
4604 {
4605 r.state.config.merge.mode = mode;
4606 }
4607 Ok(r)
4608 })
4609 }
4610 Starter::Start => {
4611 if let Some(branch) = &review_branch {
4612 tracing::warn!(
4613 "conductor chose review for task {} but branch `{branch}` no longer \
4614 exists; requeuing as a fresh competition instead",
4615 task.short()
4616 );
4617 }
4618 let instruction = prepare_instruction(&starter, None, task)
4619 .unwrap_or_else(|| task.instruction.clone());
4620 Runner::start_naming(
4621 &repo,
4622 instruction,
4623 &task.title,
4624 config,
4625 crate::run::Origin::queue(&task.id),
4626 )
4627 .await
4628 .map(|mut r| {
4629 r.state.attachments = attachments.clone();
4630 r.state.sources = task.sources.clone();
4631 r
4632 })
4633 }
4634 }})).await;
4635 if let Err(error) = &started
4636 && let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>()
4637 {
4638 task.record_startup_failure(failure.clone());
4639 }
4640 let mut runner = match started {
4641 Ok(r) => r,
4642 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4643 task.last_error = Some(e.to_string());
4644 if stop.parking() || interrupt_pause.parked() {
4645 let attempts = task.attempts;
4646 task.park(e.to_string());
4647 task.attempts = attempts;
4649 } else {
4650 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4651 }
4652 record(queue, task);
4653 return Vec::new();
4654 }
4655 Err(e) if task.pr_recovery.is_some() => {
4659 task.startup_notice_cause("startup-public-head");
4660 task.attempts += 1;
4661 task.last_error = Some(format!("{start_failed}{e:#}"));
4662 task.hold_unrequested_retry(Some(format!(
4663 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4664 )));
4665 record(queue, task);
4666 return Vec::new();
4667 }
4668 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4669 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4670 Some(r) => (p.handover_refused)(r),
4671 None => format!("{e:#}"),
4672 };
4673 task.startup_notice_cause("startup-handover");
4674 let reason = format!("{start_failed}{detail}");
4675 task.last_error = Some(reason.clone());
4676 let branch = match &starter {
4679 Starter::Review(branch) => Some(branch.clone()),
4680 _ => None,
4681 };
4682 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4686 record(queue, task);
4687 tracing::warn!(
4688 "holding {} for a branch it cannot take over: {e:#}",
4689 task.short()
4690 );
4691 return Vec::new();
4692 }
4693 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4698 let d = e
4699 .downcast_ref::<crate::reconcile::Diverged>()
4700 .expect("checked by the guard");
4701 let mut q = ask::Question::new(
4702 task.id.clone(),
4703 crate::reconcile::NODE.to_owned(),
4704 crate::reconcile::SEAT.to_owned(),
4705 d.summary(),
4706 d.detail(),
4707 d.choices(),
4708 );
4709 match Questions::open().put(&mut q) {
4710 Ok(()) => {
4711 task.last_error = Some(format!("{e:#}"));
4712 task.review_branch = Some(d.branch.clone());
4715 task.review_instruction = Some(task.instruction.clone());
4716 task.block(vec![q.id.clone()], Some(d.summary()));
4717 }
4718 Err(put) => {
4719 tracing::warn!("could not file the divergence question: {put:#}");
4720 task.attempts += 1;
4721 task.fail_startup(
4722 "startup-question-store",
4723 format!("{start_failed}{e:#}"),
4724 opts.max_attempts,
4725 );
4726 }
4727 }
4728 record(queue, task);
4729 return Vec::new();
4730 }
4731 Err(e) => {
4732 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4738 && let Starter::Review(branch) = &starter
4739 {
4740 task.review_branch = Some(branch.clone());
4741 task.review_instruction = Some(task.instruction.clone());
4742 task.last_error = Some(format!("{start_failed}{e:#}"));
4743 task.status = crate::queue::TaskStatus::Failed;
4744 record(queue, task);
4745 return Vec::new();
4746 }
4747 task.attempts += 1;
4748 task.last_error = Some(format!("{start_failed}{e:#}"));
4749 task.startup_notice_cause("startup-source");
4750 task.hold_unrequested_retry(Some(format!(
4751 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4752 )));
4753 record(queue, task);
4754 return Vec::new();
4755 }
4756 };
4757 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4758 runner.state.recovery_config = Some(config_evidence);
4759 }
4760 if let Some(replacement) = &task.replacement {
4761 if matches!(starter, Starter::Start) {
4762 runner.state.instruction.push_str(&format!(
4763 "\n\n# Replacement review context\n{}",
4764 replacement.context
4765 ));
4766 }
4767 runner.state.replaces = Some(replacement.clone());
4768 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4769 if let Ok(mut old) = RunState::load(&replacement.run)
4770 && let Some(d) = &mut old.merge_decision
4771 {
4772 d.replacement_run = Some(runner.state.id.clone());
4773 old.event(
4774 "replacement",
4775 format!(
4776 "replacement run {} owns the next attempt; old PR remains closed",
4777 runner.state.id
4778 ),
4779 );
4780 if let Err(e) = old.save() {
4781 let error = runner.fail_startup("replacement-history", e);
4782 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4783 task.record_startup_failure(failure.clone());
4784 }
4785 task.hold_manual(Some(format!(
4786 "Could not link replacement history: {error:#}; no agents dispatched"
4787 )));
4788 record(queue, task);
4789 return Vec::new();
4790 }
4791 }
4792 }
4793 runner.state.task_instruction = Some(task.instruction.clone());
4794 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4797 if runner.state.origin_chat.is_none() {
4798 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4799 }
4800 runner.on_pause(stop.pause());
4802 runner.watch_interrupt(interrupt_pause);
4806
4807 let run = runner.state.id.clone();
4810 let before_start = task.clone();
4811 task.start(run.clone());
4812 if let Err(error) = queue.put(task) {
4813 *task = before_start;
4814 let error = runner.fail_startup("task-start", error);
4815 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4816 task.record_startup_failure(failure.clone());
4817 }
4818 task.hold_manual(Some(format!(
4819 "Could not persist task start; no agents dispatched: {error:#}"
4820 )));
4821 record(queue, task);
4822 return Vec::new();
4823 }
4824 lock(status).current.push(Current {
4825 worker: None,
4826 task: task.id.clone(),
4827 run,
4828 });
4829
4830 let quota_before = runner.state.quota.clone();
4833 runner.state.next_pass_task_entry = true;
4834 let result = runner.execute().await;
4835 finish_attempt(
4836 opts.max_attempts,
4837 queue,
4838 task,
4839 &runner.state,
4840 "a_before,
4841 result,
4842 )
4843}
4844
4845pub fn finish_attempt(
4850 max_attempts: usize,
4851 queue: &Queue,
4852 task: &mut Task,
4853 state: &RunState,
4854 quota_before: &[QuotaLoss],
4855 result: Result<()>,
4856) -> Vec<QuotaLoss> {
4857 let detail = match result {
4858 Ok(()) => describe(state),
4859 Err(e) => format!("{e:#}"),
4860 };
4861 let fresh = losses_this_attempt(quota_before, &state.quota);
4862 let verdict = Verdict {
4863 status: state.status,
4864 left_pr: state.pr.is_some(),
4867 quota_hit: !fresh.is_empty(),
4873 parked: state.parked,
4877 no_viable_candidates: state.viable().is_empty(),
4880 };
4881 if let Some(replacement) = &state.replaces
4882 && let Some(pr) = &state.pr
4883 && let Ok(mut old) = RunState::load(&replacement.run)
4884 && let Some(d) = &mut old.merge_decision
4885 {
4886 d.replacement_pr = Some(pr.url.clone());
4887 let _ = old.save();
4888 }
4889 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4890 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4891 task.hold_unrequested_retry(Some(format!(
4892 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4893 task.short()
4894 )));
4895 }
4896 if task.status == TaskStatus::Done {
4897 supersede_prior_runs(task, &crate::run::home());
4898 }
4899 record(queue, task);
4900 tracing::info!(
4901 "task {} is {} after run {} ({})",
4902 task.short(),
4903 task.status.as_str(),
4904 state.short(),
4905 label(state.status)
4906 );
4907 fresh
4908}
4909
4910pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4914 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4917 && !awaiting_resume_with(task, RunState::load)
4918 && withheld_text_wait(task).is_none()
4922 && !(task.fresh_start && task.replacement.is_some())
4923 {
4924 let why = task.last_error.clone().map_or_else(
4925 || "the run did not finish".to_owned(),
4926 |e| format!("the run did not finish: {e}"),
4927 );
4928 task.hold_unrequested_retry(Some(format!(
4929 "{why}. It was started by hand, so it is not retried \
4930 automatically; `magi task release` retries it."
4931 )));
4932 record(queue, task);
4933 }
4934}
4935
4936#[must_use]
4940pub fn loop_would_resume(run: &str) -> bool {
4941 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
4942}
4943
4944#[must_use]
4954pub fn foreign_loop(
4955 reading: Option<&Reading>,
4956 now: Timestamp,
4957 own_pid: u32,
4958) -> Option<Option<u32>> {
4959 let reading = reading.filter(|r| r.running(now))?;
4960 match reading.pid {
4961 Some(pid) if pid == own_pid => None,
4962 pid => Some(pid),
4963 }
4964}
4965
4966fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
4967 reading.worker_protocol != Some(1)
4968 && (foreign_loop(Some(reading), now, own_pid).is_some()
4969 || reading
4970 .pid
4971 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
4972}
4973
4974pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
4985 run_claimed_until(opts, queue, task, &Stop::new()).await;
4986}
4987
4988async fn run_claimed_until(
4989 opts: &Opts,
4990 queue: &Queue,
4991 task: &mut Task,
4992 stop: &Stop,
4993) -> Vec<QuotaLoss> {
4994 let status = Arc::new(Mutex::new(Status::new()));
4995 let mut quota = Box::pin(attempt(
4998 opts,
4999 queue,
5000 &status,
5001 stop,
5002 crate::graph::Pause::new(),
5003 task,
5004 ))
5005 .await;
5006 let mut resumes = 0;
5015 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
5016 let Some(timeout) = withheld_text_wait(task) else {
5017 break;
5018 };
5019 eprintln!(
5020 "waiting for the owner's answer on a withheld pull request text \
5021 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
5022 );
5023 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
5026 let ready = loop {
5027 match land_resume_state(task) {
5028 LandResume::StillWaiting
5029 if !stop.parking() && std::time::Instant::now() < limit =>
5030 {
5031 tokio::time::sleep(WITHHELD_POLL).await;
5032 }
5033 LandResume::Ready => break true,
5034 _ => break false,
5035 }
5036 };
5037 if !ready || stop.parking() {
5038 break;
5039 }
5040 resumes += 1;
5041 quota.extend(
5042 Box::pin(attempt(
5043 opts,
5044 queue,
5045 &status,
5046 stop,
5047 crate::graph::Pause::new(),
5048 task,
5049 ))
5050 .await,
5051 );
5052 }
5053 hold_if_runnable(queue, task);
5054 quota
5055}
5056
5057const MAX_WITHHELD_RESUMES: u32 = 3;
5060
5061const WITHHELD_POLL: Duration = Duration::from_secs(15);
5063
5064fn withheld_text_wait(task: &Task) -> Option<u64> {
5067 let id = task.runs.last()?;
5068 let s = RunState::load(id).ok()?;
5069 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
5070 .then_some(s.config.graph.answer_timeout)
5071}
5072
5073fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
5083 after
5084 .iter()
5085 .filter(|q| !before.contains(q))
5086 .cloned()
5087 .collect()
5088}
5089
5090fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
5093 let quota: Vec<_> = quota
5094 .iter()
5095 .filter(|q| q.cause == crate::agent::QuotaCause::UsageLimit)
5096 .collect();
5097 if quota.is_empty() {
5098 return None;
5099 }
5100 let with_hint = quota.iter().find(|q| q.reset.is_some());
5101 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
5102 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
5103 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
5104 Some(
5105 now.checked_add(jiff::SignedDuration::from_secs(secs))
5106 .unwrap_or(Timestamp::MAX),
5107 )
5108}
5109
5110fn implementers_match(task: &Task, state: &RunState) -> bool {
5112 task.replacement.as_ref().map_or_else(
5113 || {
5114 task.implementers()
5115 .is_none_or(|n| n == state.config.graph.implementers)
5116 },
5117 |replacement| {
5118 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
5119 && state.config.graph.implementers == 1
5120 },
5121 )
5122}
5123
5124fn starter_with_implementers(
5126 task: &Task,
5127 previous: Option<&RunState>,
5128 proposed: Starter,
5129) -> Starter {
5130 if previous.is_some_and(|s| !implementers_match(task, s)) {
5131 task.review_of
5132 .as_ref()
5133 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
5134 } else {
5135 proposed
5136 }
5137}
5138
5139fn apply_task_implementers(config: &mut Config, task: &Task) {
5141 if let Some(n) = task.implementers() {
5142 config.graph.implementers = n;
5143 }
5144}
5145
5146#[cfg(test)]
5151fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5152 let explicit = match &task.overrides {
5153 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5154 _ => opts.config.as_deref(),
5155 };
5156 let (mut config, _) = Config::discover(repo, explicit)?;
5157 apply_options(&mut config, opts, task)?;
5158 Ok(config)
5159}
5160
5161fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5162 if task
5163 .overrides
5164 .as_ref()
5165 .is_none_or(|o| o.inherit_daemon_options)
5166 && let Some(mode) = opts.merge.as_deref()
5167 {
5168 config.merge.mode = merge_mode(mode)?;
5169 }
5170 if let Some(o) = &task.overrides {
5171 o.apply(config);
5172 }
5173 Ok(())
5174}
5175
5176fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5178 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5179 if let Some(mode) = &opts.merge {
5180 config.merge.mode = merge_mode(mode)?;
5181 }
5182 Ok(config)
5183}
5184
5185async fn maybe_prune_cache_between_runs(
5190 repo: &Path,
5191 opts: &Opts,
5192 home: &Path,
5193 stop: &Stop,
5194 last_checked: &mut Option<Timestamp>,
5195 now: Timestamp,
5196) {
5197 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5198 return;
5199 }
5200 *last_checked = Some(now);
5201 let cfg = match prepare(repo, opts) {
5202 Ok(cfg) => cfg,
5203 Err(e) => {
5204 tracing::warn!("cache check: no config: {e:#}");
5205 return;
5206 }
5207 };
5208 match clean::prune_cache_if_over_limit(&cfg, home) {
5209 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5210 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5211 pruned.files,
5212 pruned.freed
5213 ),
5214 Ok(_) => {}
5215 Err(e) => {
5216 tracing::warn!("housekeep: prune cache: {e:#}");
5217 notices::raise_in(
5218 home,
5219 Notice::warn(
5220 "housekeep:cache",
5221 format!(
5222 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5223 e.to_string()
5224 .lines()
5225 .next()
5226 .unwrap_or("unknown cleanup error")
5227 ),
5228 ),
5229 );
5230 }
5231 }
5232}
5233
5234fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5238 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5239}
5240
5241async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5260 let cfg = match prepare(repo, opts) {
5261 Ok(cfg) => cfg,
5262 Err(e) => {
5263 tracing::warn!("housekeep: no config: {e:#}");
5264 return;
5265 }
5266 };
5267 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5276 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5277 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5282 let mut extra = Vec::new();
5283 if out.unreadable > 0 {
5284 extra.push(format!("{} unreadable", out.unreadable));
5285 }
5286 if out.orphaned_worktrees > 0 {
5287 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5288 }
5289 let detail = if extra.is_empty() {
5290 String::new()
5291 } else {
5292 format!(" ({})", extra.join(", "))
5293 };
5294 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5295 }
5296 if out.external_merges_recorded > 0 {
5297 tracing::info!(
5298 "housekeep: recorded {} run(s) as merged externally",
5299 out.external_merges_recorded
5300 );
5301 }
5302 if out.stale_pr_states_repaired > 0 {
5303 tracing::info!(
5304 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5305 out.stale_pr_states_repaired
5306 );
5307 }
5308 if out.cache_files > 0 {
5309 tracing::info!(
5310 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5311 out.cache_files,
5312 out.cache_freed
5313 );
5314 }
5315 if out.questions_abandoned > 0 {
5316 tracing::info!(
5317 "housekeep: abandoned {} question(s) left open by a finished run",
5318 out.questions_abandoned
5319 );
5320 }
5321}
5322
5323async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5332 let questions = Questions::at(home.join("questions"));
5333 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5334 if report.is_empty() {
5335 return;
5336 }
5337 if !report.quarantined.is_empty() {
5338 tracing::info!(
5339 "triage: held {} blocked task(s) whose blocked-on task or \
5340 question no longer exists: {}",
5341 report.quarantined.len(),
5342 report.quarantined.join(", ")
5343 );
5344 }
5345 if !report.resumed.is_empty() {
5346 tracing::info!(
5347 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5348 report.resumed.len(),
5349 report.resumed.join(", ")
5350 );
5351 }
5352 if !report.asked.is_empty() {
5353 tracing::info!(
5354 "triage: asked about {} held task(s): {}",
5355 report.asked.len(),
5356 report.asked.join(", ")
5357 );
5358 }
5359 if !report.answered.is_empty() {
5360 tracing::info!(
5361 "triage: applied {} operator answer(s): {}",
5362 report.answered.len(),
5363 report.answered.join(", ")
5364 );
5365 }
5366}
5367
5368pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5375 disk_gate_with(repo, config, crate::disk::free_bytes)
5376}
5377
5378fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5382 repo: &Path,
5383 config: &Config,
5384 free_bytes: F,
5385) -> Option<String> {
5386 let min = config.disk.min_free_bytes;
5387 if min == 0 {
5388 return None;
5389 }
5390 let mut paths = vec![repo.to_owned()];
5391 if let Some(cache) = config.cache_dir() {
5392 paths.push(cache);
5393 }
5394 if let Some(root) = &config.graph.worktree_root {
5395 paths.push(root.clone());
5396 }
5397 for path in paths {
5398 let probe = if path == repo {
5399 path.as_path()
5400 } else {
5401 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5402 };
5403 let reason = match free_bytes(probe) {
5404 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5405 Err(e) => Some(crate::disk::unmeasured_in(
5406 probe,
5407 &e,
5408 &config.graph.language,
5409 )),
5410 };
5411 if reason.is_some() {
5412 return reason;
5413 }
5414 }
5415 None
5416}
5417
5418const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5425
5426const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5430
5431fn quota_wait(
5440 reset_at: Option<Timestamp>,
5441 now: Timestamp,
5442 fallback: Duration,
5443 cap: Duration,
5444) -> Duration {
5445 match reset_at {
5446 Some(at) if at > now => {
5447 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5448 Duration::from_secs(secs).min(cap)
5449 }
5450 _ => fallback,
5451 }
5452}
5453
5454fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5468 parse_reset_hint_zoned(text, now)
5469 .or_else(|| parse_reset_hint_dated(text))
5470 .or_else(|| parse_reset_hint_relative(text, recorded))
5471}
5472
5473fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5477 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5478 let mut rest = rest.trim();
5479 if rest.is_empty() {
5480 return None;
5481 }
5482 let mut total: i64 = 0;
5483 let mut matched = false;
5484 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5485 if let Some((digits, tail)) = rest.split_once(unit)
5486 && !digits.is_empty()
5487 && digits.bytes().all(|b| b.is_ascii_digit())
5488 {
5489 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5490 rest = tail;
5491 matched = true;
5492 }
5493 }
5494 if !rest.is_empty() || !matched {
5495 return None;
5496 }
5497 recorded
5498 .checked_add(jiff::SignedDuration::from_secs(total))
5499 .ok()
5500}
5501
5502fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5506 let clock = clock.trim().to_lowercase();
5507 let (digits, pm) = clock
5508 .strip_suffix("am")
5509 .map(|d| (d, false))
5510 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5511 let (h, m) = digits.trim().split_once(':')?;
5512 let mut hour: i8 = h.trim().parse().ok()?;
5513 let minute: i8 = m.trim().parse().ok()?;
5514 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5515 return None;
5516 }
5517 if pm && hour != 12 {
5518 hour += 12;
5519 } else if !pm && hour == 12 {
5520 hour = 0;
5521 }
5522 Some((hour, minute))
5523}
5524
5525fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5530 let open = text.find('(')?;
5531 let close = text.rfind(')')?;
5532 if close <= open {
5533 return None;
5534 }
5535 let zone = text[open + 1..close].trim();
5536 let (hour, minute) = parse_12h_clock(&text[..open])?;
5537 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5538 let candidate = now
5539 .to_zoned(tz)
5540 .with()
5541 .hour(hour)
5542 .minute(minute)
5543 .second(0)
5544 .millisecond(0)
5545 .microsecond(0)
5546 .nanosecond(0)
5547 .build()
5548 .ok()?;
5549 let mut at = candidate.timestamp();
5550 if at <= now {
5551 at += jiff::SignedDuration::from_hours(24);
5552 }
5553 Some(at)
5554}
5555
5556fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5565 let words: Vec<&str> = text.split_whitespace().collect();
5566 if words.len() < 5 {
5567 return None;
5568 }
5569 (0..=words.len() - 5)
5570 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5571}
5572
5573fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5579 if trailing.is_some_and(|next| next.starts_with('(')) {
5580 return None;
5581 }
5582 let month = month_number(window[0])?;
5583 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5584 let day_digits = ["st", "nd", "rd", "th"]
5585 .iter()
5586 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5587 let day: i8 = day_digits.parse().ok()?;
5588 let year_token = window[2];
5589 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5590 return None;
5591 }
5592 let year: i16 = year_token.parse().ok()?;
5593 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5597 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5598 let date = jiff::civil::Date::new(year, month, day).ok()?;
5599 let candidate = date
5600 .at(hour, minute, 0, 0)
5601 .to_zoned(jiff::tz::TimeZone::UTC)
5602 .ok()?;
5603 Some(candidate.timestamp())
5604}
5605
5606fn month_number(name: &str) -> Option<i8> {
5609 const NAMES: [&str; 12] = [
5610 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5611 ];
5612 let lower = name.to_lowercase();
5613 NAMES
5614 .iter()
5615 .position(|n| *n == lower.as_str())
5616 .map(|i| i as i8 + 1)
5617}
5618
5619#[cfg(test)]
5621fn exhausted_review_budget(state: &RunState) -> bool {
5622 !crate::handover::can_resume(state)
5625 && state.status == RunStatus::Blocked
5626 && !state.released()
5627 && state.retired_to.is_none()
5628}
5629
5630fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5667 unfinished_run_with(runs, short, RunState::load)
5668}
5669
5670pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5673where
5674 F: FnMut(&str) -> Result<RunState>,
5675{
5676 let mut id = runs.last()?.clone();
5677 let mut visited = std::collections::BTreeSet::new();
5678 let mut repo = None;
5679 loop {
5680 if !runs.contains(&id) || !visited.insert(id.clone()) {
5681 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5682 return None;
5683 }
5684 let state = match load(&id) {
5685 Ok(state) if state.id == id => state,
5686 Ok(_) => return None,
5687 Err(e) => {
5688 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5689 return None;
5690 }
5691 };
5692 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5693 return None;
5694 }
5695 repo = Some(state.repo.clone());
5696 if let Some(authority) = state.retired_to {
5697 id = authority;
5698 continue;
5699 }
5700 return (crate::handover::can_resume(&state)
5701 && state.liveness(false) != crate::run::Liveness::Live)
5702 .then_some(id);
5703 }
5704}
5705
5706fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5714where
5715 F: FnOnce(&str) -> Result<RunState>,
5716{
5717 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5719 || task.fresh_start
5720 || task.review_branch.is_some()
5721 {
5722 return false;
5723 }
5724 let Some(id) = task.runs.last() else {
5725 return false;
5726 };
5727 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5728}
5729
5730#[derive(Debug, Clone, PartialEq, Eq)]
5733enum Starter {
5734 Review(String),
5737 Resume(String),
5739 Start,
5741}
5742
5743fn retained_attempt_hold(
5748 task: &Task,
5749 starter: &Starter,
5750 limit: usize,
5751 mut load: impl FnMut(&str) -> Result<RunState>,
5752) -> Option<String> {
5753 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5754 return None;
5755 }
5756 let mut retained = 0;
5757 for id in task
5758 .runs
5759 .iter()
5760 .chain(task.startup_failures.iter().map(|failure| &failure.run))
5761 .collect::<std::collections::BTreeSet<_>>()
5762 {
5763 let state = match load(id) {
5764 Ok(state) if state.id == *id => state,
5765 _ => {
5766 return Some(format!(
5767 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5768 ));
5769 }
5770 };
5771 if state.retired_to.is_some() || state.released() {
5772 continue;
5773 }
5774 retained += 1;
5777 }
5778 (retained >= limit).then(|| format!(
5779 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5780 ))
5781}
5782
5783fn selected_starter(
5785 selected: &crate::pr_recovery::Selection,
5786 previous: Option<&RunState>,
5787 unfinished: Option<&str>,
5788) -> Starter {
5789 if let Some(id) = unfinished
5790 && previous.is_some_and(|r| {
5791 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5792 })
5793 {
5794 Starter::Resume(id.to_owned())
5795 } else {
5796 Starter::Review(selected.branch.clone())
5797 }
5798}
5799
5800fn take_divergence_answer(
5804 branch: &str,
5805 remote: &str,
5806 task: &mut Task,
5807) -> Option<crate::reconcile::Choice> {
5808 let summary = crate::reconcile::summary_for(branch, remote);
5809 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5810 (a.question == summary)
5811 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5812 .flatten()
5813 .map(|c| (i, c))
5814 })?;
5815 task.answers.remove(idx);
5816 Some(choice)
5817}
5818
5819fn choose_starter(
5831 review_branch: Option<&str>,
5832 branch_exists: bool,
5833 unfinished: Option<&str>,
5834) -> Starter {
5835 match review_branch {
5836 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5837 Some(_) => Starter::Start,
5838 None => match unfinished {
5839 Some(id) => Starter::Resume(id.to_owned()),
5840 None => Starter::Start,
5841 },
5842 }
5843}
5844
5845fn review_retry_starter(
5849 task: &Task,
5850 previous: Option<&RunState>,
5851 branch: Option<&str>,
5852 exists: bool,
5853 unfinished: Option<&str>,
5854) -> Starter {
5855 if let Some(run) = previous
5856 && unfinished == Some(run.id.as_str())
5857 && run.reviewed_commits.is_some()
5858 && branch.is_some_and(|branch| {
5859 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
5860 })
5861 && may_resume_run(run, task)
5862 {
5863 return Starter::Resume(run.id.clone());
5864 }
5865 match (branch, unfinished, &task.review_of) {
5866 (None, None, Some(branch)) => Starter::Review(branch.clone()),
5867 _ => choose_starter(branch, exists, unfinished),
5868 }
5869}
5870
5871fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5873 let branch = task.review_branch.take();
5874 let instruction = task.review_instruction.take();
5875 branch.filter(|branch| match &instruction {
5876 Some(authored) => *authored == task.instruction,
5877 None => previous.map_or_else(
5880 || task.review_of.as_ref() == Some(branch),
5881 |run| compatible_run_instruction(run, task),
5882 ),
5883 })
5884}
5885
5886fn may_resume_run(run: &RunState, task: &Task) -> bool {
5889 task.resume_override
5890 .as_ref()
5891 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5892 || compatible_run_instruction(run, task)
5893}
5894
5895pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5897 if let Some(authored) = &run.task_instruction {
5898 return compatible_instruction(authored, task);
5899 }
5900 if compatible_instruction(&run.instruction, task) {
5901 return true;
5902 }
5903 if run.reviewed_commits.is_some() {
5906 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5907 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5908 }
5909 if let Some(branch) = &task.review_of {
5910 let canonical = format!(
5911 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5912 );
5913 let generated = format!(
5914 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5915 );
5916 return task.instruction == canonical && run.instruction.starts_with(&generated);
5917 }
5918 }
5919 false
5920}
5921
5922pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5924 strip_answers_block(old, task) == task.instruction
5925}
5926
5927fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5930 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5931 return fallback.to_path_buf();
5932 }
5933 task.repo.clone()
5934}
5935
5936const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
5940
5941fn answers_block(task: &Task, count: usize) -> String {
5943 let mut s = ANSWERS_HEADER.to_owned();
5944 for a in &task.answers[..count] {
5945 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
5946 }
5947 s
5948}
5949
5950fn resumption_block(r: &crate::queue::Resumption) -> String {
5953 format!(
5954 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
5955 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
5956 r.generation, r.at, r.instruction, r.answers_through,
5957 )
5958}
5959
5960fn append_answers(base: &str, task: &Task) -> String {
5962 let mut s = base.to_owned();
5963 if !task.answers.is_empty() {
5964 s.push_str(&answers_block(task, task.answers.len()));
5965 }
5966 if let Some(r) = &task.resumption {
5967 s.push_str(&resumption_block(r));
5968 }
5969 s
5970}
5971
5972fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
5976 let mut instruction = instruction;
5977 for approval in task
5978 .resumption
5979 .iter()
5980 .chain(task.resumption_history.iter().rev())
5981 {
5982 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
5983 instruction = base;
5984 break;
5985 }
5986 }
5987 for count in (1..=task.answers.len()).rev() {
5988 let block = answers_block(task, count);
5989 if let Some(base) = instruction.strip_suffix(&block) {
5990 return base;
5991 }
5992 }
5993 instruction
5994}
5995
5996fn instruction_for(task: &Task) -> String {
6004 append_answers(&task.instruction, task)
6005}
6006
6007fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
6019 append_answers(strip_answers_block(old_instruction, task), task)
6020}
6021
6022fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
6024 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
6025 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
6026 if let Some(cause) = task.hold_notice_cause.as_mut() {
6027 cause.target = failure.map(|f| f.target.clone());
6028 }
6029}
6030
6031#[derive(Debug, thiserror::Error)]
6032#[error("attachment `{name}` is recorded on the task but {path} is missing")]
6033struct MissingAttachment {
6034 name: String,
6035 path: PathBuf,
6036}
6037
6038fn task_attachments(
6040 queue: &Queue,
6041 task: &Task,
6042) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
6043 let paths = queue.attachment_paths(task);
6044 for (name, path) in task.attachments.iter().zip(&paths) {
6045 if !path.is_file() {
6046 return Err(MissingAttachment {
6047 name: name.clone(),
6048 path: path.clone(),
6049 });
6050 }
6051 }
6052 Ok(paths)
6053}
6054
6055fn prepare_instruction(
6066 starter: &Starter,
6067 old_instruction: Option<&str>,
6068 task: &Task,
6069) -> Option<String> {
6070 match starter {
6071 Starter::Start => Some(instruction_for(task)),
6072 Starter::Resume(_) => Some(resumed_instruction(
6073 old_instruction.expect("a resumed run always has a prior instruction"),
6074 task,
6075 )),
6076 Starter::Review(_) => None,
6077 }
6078}
6079
6080fn record(queue: &Queue, task: &mut Task) {
6084 if let Err(e) = queue.put(task) {
6085 tracing::error!("could not record task {}: {e:#}", task.short());
6086 let notice = Notice::error(
6087 "loop:record",
6088 format!(
6089 "Task {} state could not be saved: {}. Check the disk.",
6090 task.id,
6091 e.to_string().lines().next().unwrap_or("unknown save error")
6092 ),
6093 )
6094 .link(Link::Task {
6095 id: task.id.clone(),
6096 });
6097 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
6099 Some(home) => notices::raise_in(home, notice),
6100 None => notices::raise(notice),
6101 }
6102 }
6103}
6104
6105fn runnable(queue: &Queue) -> Vec<Task> {
6111 let mut tasks: Vec<Task> = queue
6112 .list()
6113 .into_iter()
6114 .filter(|t| t.status.runnable())
6115 .collect();
6116 tasks.sort_unstable_by(crate::queue::dispatch_order);
6117 tasks
6118}
6119
6120fn describe(state: &RunState) -> String {
6134 let p = phrases(&state.config.graph.language);
6135 let mut detail = if state.status == RunStatus::Stalled {
6136 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
6137 seats.sort_unstable();
6138 seats.dedup();
6139 if seats.is_empty() {
6140 p.quorum_lost.to_owned()
6141 } else {
6142 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
6143 }
6144 } else {
6145 format!("{}{}", p.run_ended, state.status.display_label())
6146 };
6147 if let Some(cause) = state.verification_blocking_cause() {
6148 detail.push_str(&format!(" ({cause})"));
6149 } else if let Some(last) = state.events.last() {
6150 let unanswered = state
6154 .reviews
6155 .last()
6156 .filter(|r| {
6157 state.status == RunStatus::Blocked
6158 && last.node == "review"
6159 && r.incomplete()
6160 && r.blocking == 0
6161 && r.round == state.config.graph.review_rounds
6162 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6163 })
6164 .map(|r| (r.expected - r.answered, r.round));
6165 match unanswered {
6166 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6167 detail.push_str(&format!(
6168 " ({}: {})",
6169 last.node,
6170 (p.reviewers_never_answered)(missing, rounds)
6171 ));
6172 }
6173 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6174 }
6175 }
6176 detail.push_str(&format!(" [run {}]", state.id));
6177 detail
6178}
6179
6180const DIAGNOSTIC_MAX: usize = 4_000;
6186
6187const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6192
6193fn diagnostic(state: &RunState) -> Option<String> {
6207 let mut parts: Vec<String> = Vec::new();
6208
6209 for o in state.gate.iter().filter(|o| !o.ok()) {
6211 parts.push(format!(
6212 "gate `{}` failed ({}):\n{}",
6213 o.command,
6214 notices::exit_status(o.code),
6215 crate::run::summarize_verify(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6216 ));
6217 }
6218
6219 if let Some(last) = state
6222 .events
6223 .iter()
6224 .rev()
6225 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6226 {
6227 parts.push(last.message.clone());
6228 }
6229
6230 if state.viable().is_empty() {
6237 for c in &state.candidates {
6238 if let Some(evidence) = &c.verified_noop {
6239 parts.push(format!(
6240 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6241 c.label
6242 ));
6243 } else if !c.summary.trim().is_empty() {
6244 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6245 } else if let Some(why) = &c.failed {
6246 parts.push(format!("candidate {}: {why}", c.label));
6247 }
6248 }
6249 }
6250
6251 if parts.is_empty() {
6252 return None;
6253 }
6254 Some(crate::run::tail(
6259 &parts.join("\n\n"),
6260 DIAGNOSTIC_MAX.saturating_sub(100),
6261 ))
6262}
6263
6264fn label(status: RunStatus) -> &'static str {
6272 status.as_str()
6273}
6274
6275pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6277 match mode {
6278 "none" => Ok(MergeMode::None),
6279 "local" => Ok(MergeMode::Local),
6280 "pr" => Ok(MergeMode::Pr),
6281 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6282 }
6283}
6284
6285fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6290 mutex
6291 .lock()
6292 .unwrap_or_else(std::sync::PoisonError::into_inner)
6293}
6294
6295#[cfg(test)]
6296mod tests {
6297 use super::*;
6298 use crate::queue::{Source, TaskStatus};
6299 use crate::run::{Candidate, CommandOutcome};
6300 use pretty_assertions::assert_eq;
6301
6302 fn task() -> Task {
6303 Task::new(
6304 "add retries".to_owned(),
6305 "add retries".to_owned(),
6306 PathBuf::from("/repo"),
6307 Source::Human,
6308 )
6309 }
6310
6311 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6312 let job = crate::worker::Job {
6313 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6314 sequence: 1,
6315 kind: "task".into(),
6316 key: format!("task:{}", request.task),
6317 payload: serde_json::to_value(request).unwrap(),
6318 generation: "a".repeat(64),
6319 version: None,
6320 status: crate::worker::Status::Starting,
6321 result: None,
6322 pid: None,
6323 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6324 finished_at: None,
6325 };
6326 let dir = home.join("workers/jobs").join(&job.id);
6327 std::fs::create_dir_all(&dir).unwrap();
6328 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6329 job
6330 }
6331
6332 #[test]
6333 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6334 let home = tempfile::tempdir().unwrap();
6335 let request = TaskWorker {
6336 opts: Opts::default(),
6337 queue: home.path().join("queue"),
6338 task: "startup".into(),
6339 lane: PermitKind::Ordinary,
6340 standalone: false,
6341 };
6342 let job = starting_task_reservation(home.path(), &request);
6343 let error = task_worker_spawn_error(
6344 home.path(),
6345 &request,
6346 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6347 )
6348 .expect_err("our own reservation must not turn a failed spawn into a resync");
6349 let active = crate::worker::active(home.path()).unwrap();
6350 assert_eq!(active.len(), 1);
6351 assert_eq!(active[0].id, job.id);
6352 assert_eq!(active[0].status, crate::worker::Status::Starting);
6353 assert!(
6354 crate::worker::control(home.path(), &job.key)
6355 .unwrap()
6356 .is_none()
6357 );
6358 let notices = notices::Notices::at(home.path().join("notifications")).list();
6359 assert_eq!(notices.len(), 1);
6360 assert_eq!(notices[0].key, "loop:worker-startup");
6361 assert!(!notices[0].message.contains("private-error-detail"));
6362 assert!(notices[0].message.contains(&request.task));
6363 assert!(
6364 notices[0]
6365 .message
6366 .contains("could not be started or acknowledged")
6367 );
6368 assert!(notices[0].message.contains("worker.log"));
6369 assert_eq!(
6370 notices[0].link,
6371 Some(Link::Task {
6372 id: request.task.clone()
6373 })
6374 );
6375 assert!(format!("{error:#}").contains("private-error-detail"));
6376 }
6377
6378 #[test]
6379 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6380 let home = tempfile::tempdir().unwrap();
6381 let mut request = TaskWorker {
6382 opts: Opts::default(),
6383 queue: home.path().join("queue"),
6384 task: "handoff".into(),
6385 lane: PermitKind::Ordinary,
6386 standalone: true,
6387 };
6388 let job = starting_task_reservation(home.path(), &request);
6389 request.standalone = false;
6390 task_worker_spawn_error(
6391 home.path(),
6392 &request,
6393 anyhow::anyhow!("operation key is already reserved for a different request"),
6394 )
6395 .expect("a different standalone request is the handoff race, not our startup failure");
6396 let active = crate::worker::active(home.path()).unwrap();
6397 assert_eq!(active.len(), 1);
6398 assert_eq!(active[0].id, job.id);
6399 assert_eq!(active[0].payload, job.payload);
6400 assert!(
6401 notices::Notices::at(home.path().join("notifications"))
6402 .list()
6403 .is_empty()
6404 );
6405 }
6406
6407 #[test]
6408 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6409 let home = tempfile::tempdir().unwrap();
6410 let queue = Queue::at(home.path().join("queue"));
6411 let mut task = task();
6412 task.runs = vec!["previous".into()];
6413 queue.put(&mut task).unwrap();
6414 let request = TaskWorker {
6415 opts: Opts::default(),
6416 queue: home.path().join("queue"),
6417 task: task.id.clone(),
6418 lane: PermitKind::Ordinary,
6419 standalone: false,
6420 };
6421 let job = starting_task_reservation(home.path(), &request);
6422 let claims = current_work(home.path(), Timestamp::now());
6423 assert_eq!(claims.len(), 1);
6424 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6425 assert_eq!(claims[0].run, "previous");
6426 assert!(other_claims_run(&claims, "previous", None));
6427 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6430 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6431 }
6432
6433 #[test]
6434 fn admission_excludes_only_the_callers_exact_reservation() {
6435 let own = Current {
6436 worker: Some("own".into()),
6437 task: "same-task".into(),
6438 run: "previous".into(),
6439 };
6440 assert!(!other_claims_run(
6441 std::slice::from_ref(&own),
6442 "previous",
6443 Some("own")
6444 ));
6445 assert!(other_claims_run(
6446 std::slice::from_ref(&own),
6447 "previous",
6448 None
6449 ));
6450 for worker in [None, Some("other".into())] {
6451 let other = Current {
6452 worker,
6453 ..own.clone()
6454 };
6455 assert!(other_claims_run(
6456 &[own.clone(), other],
6457 "previous",
6458 Some("own")
6459 ));
6460 }
6461 let legacy: Current =
6462 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6463 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6464 }
6465
6466 #[test]
6467 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6468 let mut task = task();
6469 let mut run = run_state(RunStatus::Reviewing);
6470 run.task_instruction = Some(task.instruction.clone());
6471 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6472 let branch = "magi/test/B";
6473 run.candidates.push(crate::run::Candidate {
6474 branch: branch.into(),
6475 ..candidate('B', "preserved work", false, None)
6476 });
6477 task.runs = vec![run.id.clone()];
6478 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6479 assert_eq!(
6480 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6481 Starter::Resume(run.id.clone())
6482 );
6483 assert!(matches!(
6484 review_retry_starter(
6485 &task,
6486 Some(&run),
6487 Some("different"),
6488 true,
6489 unfinished.as_deref()
6490 ),
6491 Starter::Review(_)
6492 ));
6493 assert_eq!(
6494 review_retry_starter(
6495 &task,
6496 Some(&run),
6497 Some(branch),
6498 false,
6499 unfinished.as_deref()
6500 ),
6501 Starter::Start
6502 );
6503 task.instruction.push_str(" changed criteria");
6504 assert!(matches!(
6505 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6506 Starter::Review(_)
6507 ));
6508 assert!(matches!(
6509 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6510 Starter::Review(_)
6511 ));
6512 }
6513
6514 #[test]
6515 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6516 let dir = tempfile::tempdir().unwrap();
6517 let queue = Queue::at(dir.path().join("queue"));
6518 let mut task = task();
6519 task.status = TaskStatus::Running;
6520 task.attempts = 1;
6521 queue.put(&mut task).unwrap();
6522 let live = std::collections::BTreeSet::from([task.id.clone()]);
6523 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6524 drop(queue.claim(&task.id).unwrap());
6525 let saved = queue.get(&task.id).unwrap();
6526 assert_eq!(saved.status, TaskStatus::Running);
6527 assert_eq!(saved.attempts, 1);
6528 assert!(saved.last_error.is_none());
6529 }
6530
6531 #[test]
6532 fn standalone_workers_do_not_consume_scheduler_lanes() {
6533 let request = |task: &str, lane, standalone| TaskWorker {
6534 opts: Opts::default(),
6535 queue: PathBuf::from("/queue"),
6536 task: task.to_owned(),
6537 lane,
6538 standalone,
6539 };
6540 let mut jobs = vec![
6541 request("ordinary", PermitKind::Ordinary, false),
6542 request("standalone-urgent", PermitKind::Urgent, true),
6543 request("standalone-ordinary", PermitKind::Ordinary, true),
6544 request("decision", PermitKind::None, false),
6545 ];
6546 assert_eq!(available_slots(&jobs, 2), (1, 1));
6547 jobs.push(request("urgent", PermitKind::Urgent, false));
6548 assert_eq!(available_slots(&jobs, 2), (1, 0));
6549 assert_eq!(available_slots(&jobs, 0), (0, 0));
6550 }
6551
6552 #[test]
6553 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6554 let now = Timestamp::now();
6555 let mut reading = Reading {
6556 pid: Some(std::process::id()),
6557 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6558 ..Reading::default()
6559 };
6560 assert!(!reading.running(now));
6561 assert!(legacy_scheduler_running(&reading, now, 0));
6562 reading.worker_protocol = Some(1);
6563 assert!(!legacy_scheduler_running(&reading, now, 0));
6564 }
6565
6566 #[test]
6567 fn selected_public_head_overrides_historical_resume_and_pin() {
6568 let mut t = task();
6569 let mut old = run_state(RunStatus::Blocked);
6570 let selection = crate::pr_recovery::Selection {
6571 url: "url".into(),
6572 branch: format!("magi/task-{}", t.id),
6573 head: "a".repeat(40),
6574 source_head: None,
6575 source_run: None,
6576 source_proof: None,
6577 disposition: Some(crate::pr_recovery::Disposition::Review),
6578 instruction: t.instruction.clone(),
6579 approved_at: Timestamp::now(),
6580 };
6581 t.resume_override = Some(crate::queue::OperatorResume {
6582 pinned_run: Some(old.id.clone()),
6583 question_id: "choice".into(),
6584 at: Timestamp::now(),
6585 conductor_rehold: None,
6586 forced: false,
6587 });
6588 assert_eq!(
6589 selected_starter(&selection, Some(&old), Some(&old.id)),
6590 Starter::Review(selection.branch.clone())
6591 );
6592 old.pr_recovery = Some(selection.clone());
6593 assert_eq!(
6594 selected_starter(&selection, Some(&old), Some(&old.id)),
6595 Starter::Review(selection.branch.clone())
6596 );
6597 old.reviewed_commits = Some(vec!["implementation".into()]);
6598 assert_eq!(
6599 selected_starter(&selection, Some(&old), Some(&old.id)),
6600 Starter::Resume(old.id.clone())
6601 );
6602 let mut changed = selection.clone();
6603 changed.head = "b".repeat(40);
6604 assert_eq!(
6605 selected_starter(&changed, Some(&old), Some(&old.id)),
6606 Starter::Review(changed.branch)
6607 );
6608 }
6609
6610 #[test]
6611 fn candidate_transplant_failure_is_one_actionable_hold() {
6612 let dir = tempfile::tempdir().unwrap();
6613 let q = Queue::at(dir.path().join("queue"));
6614 let mut t = task();
6615 let mut state = run_state(RunStatus::Blocked);
6616 state.event("prep", "cannot apply referenced commit: conflict");
6617 state.prep_failure = Some("conflict".into());
6618 t.start(state.id.clone());
6619 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6620 assert_eq!(t.attempts, 1);
6621 assert_eq!(t.status, TaskStatus::Held);
6622 assert!(t.operator_held());
6623 assert!(!t.status.runnable());
6624 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6625 let first = t.chat_notice_key().unwrap();
6626 t.chat_notices.push(first.clone());
6627 t.release_operator("test", None);
6628 state.id = "another-unprepared-run".into();
6629 t.start(state.id.clone());
6630 finish_attempt(
6631 3,
6632 &q,
6633 &mut t,
6634 &state,
6635 &[],
6636 Err(anyhow::anyhow!("still conflicts")),
6637 );
6638 assert_eq!(
6639 t.chat_notice_key().as_ref(),
6640 Some(&first),
6641 "no work happened"
6642 );
6643 t.release_operator("test", None);
6644 state.prep_failure = None;
6645 state.status = RunStatus::Failed;
6646 state.begin_pass();
6647 t.start(state.id.clone());
6648 finish_attempt(
6649 1,
6650 &q,
6651 &mut t,
6652 &state,
6653 &[],
6654 Err(anyhow::anyhow!("new gate failure")),
6655 );
6656 let failure = t.chat_notice_key().unwrap();
6657 assert_ne!(failure, first, "approved recovery reached a real failure");
6658 t.release_operator("test", None);
6659 state.begin_pass();
6660 t.start(state.id.clone());
6661 finish_attempt(
6662 1,
6663 &q,
6664 &mut t,
6665 &state,
6666 &[],
6667 Err(anyhow::anyhow!("new failure on resume")),
6668 );
6669 assert_ne!(
6670 t.chat_notice_key().as_ref(),
6671 Some(&failure),
6672 "a new settled pass"
6673 );
6674 }
6675
6676 #[test]
6677 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6678 for source in ["cli", "web"] {
6679 let mut t = task();
6680 let mut run = run_state(RunStatus::Blocked);
6681 run.instruction = "Run A original criteria".into();
6682 run.task_instruction = Some(run.instruction.clone());
6683 t.start(run.id.clone());
6684 t.release_operator("choice", Some("resume-question"));
6685 t.resume_override = Some(crate::queue::OperatorResume {
6686 question_id: "resume-question".into(),
6687 at: Timestamp::now(),
6688 conductor_rehold: None,
6689 forced: true,
6690 pinned_run: Some(run.id.clone()),
6691 });
6692 t.hold_machine(Some("not enough free space to start a run".into()));
6693 let mut unchanged = t.clone();
6694 unchanged
6695 .edit("title changed".into(), unchanged.instruction.clone())
6696 .unwrap();
6697 unchanged.release_operator(source, None);
6698 assert_eq!(
6699 unchanged
6700 .resume_override
6701 .as_ref()
6702 .unwrap()
6703 .pinned_run
6704 .as_deref(),
6705 Some(run.id.as_str())
6706 );
6707 assert!(!compatible_run_instruction(&run, &unchanged));
6708 assert!(may_resume_run(&run, &unchanged));
6709
6710 t.edit(
6711 "approved recovery".into(),
6712 "Reuse measured PR head B; fix Clippy; review".into(),
6713 )
6714 .unwrap();
6715 assert!(t.resume_override.is_none());
6716 t.release_operator(source, None);
6717 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6718 assert!(reloaded.resume_override.is_none());
6719 assert!(!may_resume_run(&run, &reloaded));
6720 let unfinished =
6721 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6722 .filter(|_| may_resume_run(&run, &reloaded));
6723 assert_eq!(
6724 choose_starter(None, false, unfinished.as_deref()),
6725 Starter::Start
6726 );
6727 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6728 assert_eq!(
6729 reloaded.resumption_history[0].question_id.as_deref(),
6730 Some("resume-question")
6731 );
6732
6733 unchanged.instruction = "Reuse measured PR head B".into();
6736 unchanged.hold_machine(Some("disk hold".into()));
6737 unchanged.release_operator(source, None);
6738 assert!(unchanged.resume_override.is_none());
6739 assert!(!may_resume_run(&run, &unchanged));
6740 }
6741 }
6742
6743 #[test]
6744 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6745 let mut t = task();
6746 let mut run = run_state(RunStatus::Blocked);
6747 t.review_of = Some("approved-head".into());
6748 t.instruction =
6749 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6750 .into();
6751 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6752 run.reviewed_commits = Some(vec!["Commit log".into()]);
6753 t.start(run.id.clone());
6754 t.hold_machine(Some("interrupted review".into()));
6755 t.release_operator("cli", None);
6756 assert!(
6757 compatible_run_instruction(&run, &t),
6758 "legacy explicit review remains resumable"
6759 );
6760 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6761 .filter(|_| compatible_run_instruction(&run, &t));
6762 assert_eq!(
6763 choose_starter(None, false, unfinished.as_deref()),
6764 Starter::Resume(run.id.clone())
6765 );
6766 run.task_instruction = Some(t.instruction.clone());
6767 assert!(compatible_run_instruction(&run, &t));
6768 let reloaded: RunState =
6769 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6770 assert!(compatible_run_instruction(&reloaded, &t));
6771 t.edit(
6772 "new approved head".into(),
6773 "Reuse measured PR head B".into(),
6774 )
6775 .unwrap();
6776 assert!(!compatible_run_instruction(&run, &t));
6777
6778 t.review_of = None;
6780 t.instruction = "Fix Clippy on approved head".into();
6781 run.task_instruction = None;
6782 run.instruction = format!(
6783 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6784 t.instruction
6785 );
6786 assert!(compatible_run_instruction(&run, &t));
6787 t.instruction.push('\n');
6788 assert!(
6789 compatible_run_instruction(&run, &t),
6790 "legacy generated review trimmed the original criteria"
6791 );
6792 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6793 let mut unchanged = t.clone();
6794 assert_eq!(
6795 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6796 Some("approved-head")
6797 );
6798 t.edit("new plan".into(), "Reuse measured PR head B".into())
6799 .unwrap();
6800 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6801 assert!(!compatible_run_instruction(&run, &t));
6802 t.review_branch = Some("approved-head".into());
6803 t.review_instruction = None;
6804 assert!(
6805 take_review_branch(&mut t, Some(&run)).is_none(),
6806 "legacy retained candidate cannot replace a revised approved head"
6807 );
6808 }
6809
6810 #[test]
6811 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6812 crate::run::pin_test_home();
6813 let dir = tempfile::tempdir().unwrap();
6814 let queue = Queue::at(dir.path().join("queue"));
6815 for resume in [true, false] {
6816 let state = run_state(RunStatus::Blocked);
6817 let mut t = task();
6818 t.start(state.id.clone());
6819 let action = if resume {
6820 ask::ChoiceAction::Resume {
6821 run: state.id.clone(),
6822 }
6823 } else {
6824 ask::ChoiceAction::Requeue
6825 };
6826 let mut q = action_question(&state.id, action);
6827 q.status = ask::QuestionStatus::Open;
6828 q.answer = None;
6829 q.answered_at = None;
6830 t.fail("run stopped with a question", 3);
6831 hold_if_runnable(&queue, &mut t);
6832 assert!(t.operator_decided_at.is_none());
6833 assert!(t.operator_held());
6834 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6835 let expected = if resume {
6836 ActionDecision::Resume(state.id.clone())
6837 } else {
6838 ActionDecision::Requeue
6839 };
6840 assert_eq!(
6841 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6842 expected
6843 );
6844 t.hold_manual(Some("later explicit hold".into()));
6845 assert_eq!(
6846 decide_action(&t, &q, phrases("en"), |_| Ok(state.clone())),
6847 ActionDecision::Stale
6848 );
6849 }
6850 }
6851
6852 #[test]
6853 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6854 let mut t = task();
6855 t.answers.push(crate::queue::AnsweredQuestion {
6856 question: "Inspect the remote?".into(),
6857 answer: "Inspect then wait".into(),
6858 });
6859 let old = instruction_for(&t);
6860 assert!(compatible_instruction(&old, &t));
6861 t.instruction = "Reuse approved PR head and review".into();
6862 t.release_operator("cli", None);
6863 assert!(!compatible_instruction(&old, &t));
6864 let approved = instruction_for(&t);
6865 assert!(approved.contains("waiting premises are superseded"));
6866 assert!(compatible_instruction(&approved, &t));
6867 assert_eq!(resumed_instruction(&approved, &t), approved);
6868 t.release_operator("web", None);
6869 assert!(compatible_instruction(&approved, &t));
6870 assert_eq!(t.resumption_history.len(), 1);
6871 t.start("run".into());
6872 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6873 }
6874
6875 #[test]
6879 fn unrelated_ci_refunds_once_per_pass_and_holds_for_the_owner() {
6880 let mut state = RunState::new(
6881 PathBuf::from("/repo"),
6882 "main".into(),
6883 "base".into(),
6884 "task".into(),
6885 Config::default(),
6886 );
6887 state.status = RunStatus::Landing;
6888 state.parked = true;
6889 state.pr_pushed_head = Some("head".into());
6890 state.land_unrelated = Some(
6891 serde_json::from_value(serde_json::json!({
6892 "head": "head", "question": "ci-question",
6893 "evidence": {"head": "head", "base": "base", "changed": [], "checks": []}
6894 }))
6895 .unwrap(),
6896 );
6897 let mut t = task();
6898 t.start(state.id.clone());
6899 let verdict = Verdict {
6900 status: RunStatus::Landing,
6901 left_pr: true,
6902 quota_hit: false,
6903 parked: true,
6904 no_viable_candidates: false,
6905 };
6906 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6907 assert_eq!(t.attempts, 0);
6908 assert_eq!(t.status, TaskStatus::Held);
6909 assert!(!t.operator_held());
6910 assert!(
6911 t.hold_reason
6912 .as_deref()
6913 .unwrap()
6914 .starts_with("[ci-unrelated]")
6915 );
6916 t.attempts = 1;
6918 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6919 assert_eq!(t.attempts, 1);
6920 assert_eq!(t.passes.len(), 1);
6921 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
6922 state.pr_pushed_head = Some("new-head".into());
6923 state.parked = false;
6924 state.status = RunStatus::Blocked;
6925 let stale = Verdict {
6926 status: RunStatus::Blocked,
6927 parked: false,
6928 ..verdict
6929 };
6930 settle_and_diagnose(&mut t, stale, "stale CI hold", 2, &state);
6931 assert_eq!(t.attempts, 1);
6932 state.begin_pass();
6934 state.begin_pass();
6935 state.status = RunStatus::Landing;
6936 state.parked = true;
6937 state.pr_pushed_head = Some("head".into());
6938 settle_and_diagnose(&mut t, verdict, "CI hold on resume", 2, &state);
6939 assert_eq!(t.attempts, 0);
6940 assert_eq!(t.passes.len(), 2);
6941 }
6942
6943 #[test]
6944 fn orphan_recovery_does_not_refund_a_previous_pass_loss() {
6945 let mut state = run_state(RunStatus::Failed);
6946 state
6947 .quota
6948 .push(loss("advisor-1", "2026-10-22T14:38:00Z", None));
6949 state.begin_pass();
6950 let mut t = task();
6951 t.start(state.id.clone());
6952 reclaim(&mut t, Some(state), 2, "en");
6953 assert_eq!(t.attempts, 1);
6954 }
6955
6956 #[test]
6957 fn an_all_seats_unauthenticated_node_refunds_and_holds_without_retry() {
6958 let mut state = run_state(RunStatus::Blocked);
6959 let losses: Vec<_> = ["alpha", "beta"]
6960 .into_iter()
6961 .enumerate()
6962 .map(|(i, agent)| {
6963 QuotaLoss::from_report(
6964 format!("advisor-{i}"),
6965 "advise",
6966 Some(&crate::agent::Quota {
6967 cause: crate::agent::QuotaCause::Authentication,
6968 agent: agent.to_owned(),
6969 reset: None,
6970 }),
6971 )
6972 })
6973 .collect();
6974 crate::graph::block_machine_outage(&mut state, "advise", &losses);
6975 state.quota = losses;
6976 let mut t = task();
6977 t.start(state.id.clone());
6978 let verdict = Verdict {
6979 status: state.status,
6980 left_pr: false,
6981 quota_hit: true,
6982 parked: false,
6983 no_viable_candidates: true,
6984 };
6985 settle_and_diagnose(&mut t, verdict, "authentication unavailable", 2, &state);
6986 assert_eq!(t.attempts, 0);
6987 assert_eq!(t.status, TaskStatus::Held);
6988 assert!(!t.operator_held());
6989 let reason = t.hold_reason.as_deref().unwrap();
6990 assert!(reason.starts_with("[authentication]"));
6991 assert!(
6992 reason.contains("alpha")
6993 && reason.contains("beta")
6994 && reason.contains("re-authenticate")
6995 );
6996 assert!(cooldown_until(&state.quota, Timestamp::now()).is_none());
6997 let mut orphan = task();
6999 orphan.start(state.id.clone());
7000 reclaim(&mut orphan, Some(state), 2, "en");
7001 assert_eq!(orphan.attempts, 0);
7002 assert_eq!(orphan.status, TaskStatus::Held);
7003 }
7004
7005 #[test]
7006 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
7007 let mut state = RunState::new(
7008 PathBuf::from("/repo"),
7009 "main".to_owned(),
7010 "abc1234".to_owned(),
7011 "task".to_owned(),
7012 Config::default(),
7013 );
7014 state.status = RunStatus::Blocked;
7015 state.resource_block = Some(crate::run::ResourceBlock {
7016 cause: crate::run::ResourceBlockCause::Other,
7017 node: "graft".to_owned(),
7018 kind: "shared build cache".to_owned(),
7019 detail: "timed out".to_owned(),
7020 at: jiff::Timestamp::now(),
7021 });
7022 let mut t = task();
7023 t.start("20260902-000000-aaaa".to_owned());
7024 assert_eq!(t.attempts, 1);
7025 let verdict = Verdict {
7026 status: RunStatus::Blocked,
7027 left_pr: false,
7028 quota_hit: false,
7029 parked: false,
7030 no_viable_candidates: false,
7031 };
7032 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
7033 assert_eq!(t.status, TaskStatus::Held);
7034 assert_eq!(t.attempts, 0, "the attempt is refunded");
7035 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
7036 assert!(
7037 !t.operator_held(),
7038 "a machine hold, released without a human"
7039 );
7040
7041 let mut selected = task();
7042 selected.start("20260902-000000-cccc".into());
7043 selected.replacement = Some(crate::merge_decision::Replacement {
7044 question: "fixture".into(),
7045 run: "earlier".into(),
7046 pr: "https://example.invalid/pr/1".into(),
7047 agent: "selected".into(),
7048 context: "fixture".into(),
7049 });
7050 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
7051 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
7052 assert_eq!(selected.attempts, 0);
7053 assert!(
7054 !selected.operator_held(),
7055 "a selected agent also retains a machine resource hold"
7056 );
7057 assert!(
7058 selected
7059 .hold_reason
7060 .as_deref()
7061 .unwrap()
7062 .starts_with("[resource] ")
7063 );
7064
7065 let mut p = task();
7067 p.start("20260902-000000-bbbb".to_owned());
7068 settle_and_diagnose(
7069 &mut p,
7070 Verdict {
7071 parked: true,
7072 ..verdict
7073 },
7074 "parked",
7075 2,
7076 &state,
7077 );
7078 assert_eq!(p.status, TaskStatus::Parked);
7079 assert_eq!(
7080 p.attempts, 0,
7081 "parking refunds once rather than also settling the resource hold"
7082 );
7083 }
7084
7085 #[test]
7086 fn record_failure_notice_links_the_task() {
7087 let dir = tempfile::tempdir().unwrap();
7088 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
7091 let queue = Queue::at(dir.path().join("queue"));
7092 let mut t = interrupt_task("rec-fail-1");
7093 record(&queue, &mut t);
7094
7095 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
7096 assert_eq!(found.len(), 1);
7097 assert_eq!(found[0].key, "loop:record");
7098 assert_eq!(
7099 found[0].link,
7100 Some(Link::Task {
7101 id: "rec-fail-1".to_owned()
7102 })
7103 );
7104 }
7105
7106 fn interrupt_task(id: &str) -> Task {
7109 let mut t = task();
7110 t.id = id.to_owned();
7111 t.interrupt = true;
7112 t
7113 }
7114
7115 fn task_with_id(id: &str) -> Task {
7117 let mut t = task();
7118 t.id = id.to_owned();
7119 t
7120 }
7121
7122 fn urgent_task(id: &str) -> Task {
7124 let mut t = task();
7125 t.id = id.to_owned();
7126 t.urgent = true;
7127 t
7128 }
7129
7130 #[test]
7131 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
7132 let dir = tempfile::tempdir().unwrap();
7133 let queue = Queue::at(dir.path().join("queue"));
7134 let mut parked = task_with_id("parked");
7135 parked.start("run".into());
7136 parked.park("upgrade");
7137 parked.record_pass_at(
7138 "run",
7139 0,
7140 Some(Timestamp::now()),
7141 1,
7142 Some(crate::run::ParkCause::Upgrade {
7143 from: "old".into(),
7144 to: None,
7145 }),
7146 );
7147 let mut fresh = task_with_id("fresh");
7148 fresh.priority = 100;
7149 let mut urgent_task = urgent_task("urgent");
7150 for t in [&mut parked, &mut fresh, &mut urgent_task] {
7151 queue.put(t).unwrap();
7152 }
7153 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7154 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7155 let candidates = runnable(&queue);
7156 assert_eq!(candidates[0].id, parked.id);
7157 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
7158 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
7159 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
7160 drop(first);
7161 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
7162 let next = runnable(&queue)
7163 .into_iter()
7164 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
7165 .unwrap();
7166 assert_eq!(next.task.id, fresh.id);
7167 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
7168 drop(claimed_elsewhere);
7169 drop(next);
7170 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
7171 }
7172
7173 #[test]
7174 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
7175 let dir = tempfile::tempdir().unwrap();
7176 let queue = Queue::at(dir.path().join("queue"));
7177 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7178 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7179 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
7180 let mut first = task();
7181 queue.put(&mut first).unwrap();
7182 let stale_plain = queue.get(&first.id).unwrap();
7183 first.set_urgent(true).unwrap();
7184 queue.put(&mut first).unwrap();
7185 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
7186 let mut second = urgent_task("second");
7187 queue.put(&mut second).unwrap();
7188 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
7189 drop(queue.claim(&second.id).unwrap());
7190 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
7191 drop(dispatched);
7192 let stale_urgent = queue.get(&first.id).unwrap();
7193 first.set_urgent(false).unwrap();
7194 queue.put(&mut first).unwrap();
7195 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
7196 assert_eq!(urgent.available_permits(), 1);
7197 drop(queue.claim(&first.id).unwrap());
7198 drop(occupied);
7199 }
7200
7201 #[test]
7202 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
7203 let dir = tempfile::tempdir().unwrap();
7204 let queue = Queue::at(dir.path().join("queue"));
7205 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7206 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7207 let mut t = task();
7208 queue.put(&mut t).unwrap();
7209 for enabled in [true, false] {
7210 let stale = queue.get(&t.id).unwrap();
7211 t.set_urgent(enabled).unwrap();
7212 queue.put(&mut t).unwrap();
7213 assert_ne!(stale.urgent, enabled);
7214 let DispatchClaim {
7215 task: fresh,
7216 claim,
7217 permit,
7218 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
7219 assert_eq!(fresh.urgent, enabled);
7220 assert_eq!(ordinary.available_permits(), usize::from(enabled));
7221 assert_eq!(urgent.available_permits(), usize::from(!enabled));
7222 assert!(queue.claim(&t.id).is_err());
7223 assert_eq!(fresh.attempts, stale.attempts);
7224 drop((claim, permit));
7225 assert_eq!(ordinary.available_permits(), 1);
7226 assert_eq!(urgent.available_permits(), 1);
7227 }
7228 t.set_urgent(true).unwrap();
7229 queue.put(&mut t).unwrap();
7230 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
7231 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7232 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
7233 drop(queue.claim(&t.id).unwrap());
7234 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
7235 drop(occupied);
7236 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
7237 drop(queue.claim(&t.id).unwrap());
7238 let held = queue.claim(&t.id).unwrap();
7239 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7240 drop(held);
7241 t.status = TaskStatus::Held;
7242 queue.put(&mut t).unwrap();
7243 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7244 drop(queue.claim(&t.id).unwrap());
7245 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
7246 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7247 drop(queue.claim(&t.id).unwrap());
7248 assert_eq!(urgent.available_permits(), 1);
7249 }
7250
7251 #[test]
7256 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
7257 assert_eq!(permit_kind(true, false), PermitKind::None);
7258 assert_eq!(permit_kind(true, true), PermitKind::None);
7259 }
7260
7261 #[test]
7266 fn permit_kind_separates_urgent_from_ordinary() {
7267 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
7268 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
7269 }
7270
7271 #[test]
7278 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
7279 let cfg = Config::default();
7280 let repo = Path::new("/any/repo/path");
7281
7282 let reason =
7283 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
7284 assert!(reason.contains("1024"), "{reason}");
7285 assert!(
7286 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7287 "{reason}"
7288 );
7289
7290 assert_eq!(
7291 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
7292 None,
7293 "exactly at the floor is open"
7294 );
7295 assert_eq!(
7296 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
7297 None,
7298 "comfortably above the floor is open"
7299 );
7300 }
7301
7302 #[test]
7303 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
7304 let mut cfg = Config::default();
7305 cfg.disk.min_free_bytes = 0;
7306 let repo = Path::new("/any/repo/path");
7307 assert_eq!(
7308 disk_gate_with(repo, &cfg, |_| Ok(0)),
7309 None,
7310 "a zero floor never measures at all"
7311 );
7312 }
7313
7314 #[test]
7315 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7316 let cfg = Config::default();
7317 let repo = Path::new("/any/repo/path");
7318 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7319 .expect("a measurement failure must close the gate, not open it");
7320 assert!(reason.contains("could not measure"), "{reason}");
7321 }
7322
7323 #[test]
7324 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7325 let ordinary = task();
7326 let next = advance_interrupt(
7327 Interrupt::Idle,
7328 std::slice::from_ref(&ordinary.id),
7329 std::slice::from_ref(&ordinary),
7330 );
7331 assert_eq!(next, Interrupt::Idle);
7332 }
7333
7334 #[test]
7335 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7336 let marked = interrupt_task("marked");
7339 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7340 assert_eq!(next, Interrupt::Idle);
7341 }
7342
7343 #[test]
7344 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7345 let marked = interrupt_task("marked");
7346 let next = advance_interrupt(
7347 Interrupt::Idle,
7348 &["running".to_owned()],
7349 std::slice::from_ref(&marked),
7350 );
7351 assert_eq!(
7352 next,
7353 Interrupt::Parking {
7354 parked: vec!["running".to_owned()],
7355 interrupt_task: "marked".to_owned(),
7356 }
7357 );
7358 }
7359
7360 #[test]
7369 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7370 let marked = interrupt_task("marked");
7371
7372 let two = advance_interrupt(
7373 Interrupt::Idle,
7374 &["a".to_owned(), "b".to_owned()],
7375 std::slice::from_ref(&marked),
7376 );
7377 assert_eq!(two, Interrupt::Idle);
7378
7379 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7380 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7381 }
7382
7383 #[test]
7384 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7385 let state = Interrupt::Parking {
7386 parked: vec!["running".to_owned()],
7387 interrupt_task: "marked".to_owned(),
7388 };
7389 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7391 assert_eq!(still_going, state);
7392
7393 let stopped_but_not_yet_dispatched =
7397 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7398 assert_eq!(stopped_but_not_yet_dispatched, state);
7399
7400 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7402 assert_eq!(
7403 dispatched,
7404 Interrupt::Running {
7405 parked: vec!["running".to_owned()],
7406 interrupt_task: "marked".to_owned(),
7407 }
7408 );
7409 }
7410
7411 #[test]
7412 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7413 let state = Interrupt::Running {
7414 parked: vec!["running".to_owned()],
7415 interrupt_task: "marked".to_owned(),
7416 };
7417 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7418 assert_eq!(still_running, state);
7419
7420 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7427 assert_eq!(
7428 ended,
7429 Interrupt::Resuming {
7430 parked: vec!["running".to_owned()]
7431 }
7432 );
7433 }
7434
7435 #[test]
7436 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7437 let state = Interrupt::Resuming {
7438 parked: vec!["running".to_owned()],
7439 };
7440 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7441 assert_eq!(still_waiting, state);
7442
7443 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7444 assert_eq!(dispatched, Interrupt::Idle);
7445 }
7446
7447 #[test]
7453 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7454 {
7455 let state = Interrupt::Parking {
7456 parked: vec!["running".to_owned()],
7457 interrupt_task: "marked".to_owned(),
7458 };
7459 let next = advance_interrupt(state, &[], &[]);
7462 assert_eq!(
7463 next,
7464 Interrupt::Resuming {
7465 parked: vec!["running".to_owned()]
7466 },
7467 "abandoning the interrupt must not abandon the resume it owes"
7468 );
7469 }
7470
7471 #[test]
7474 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7475 let state = Interrupt::Resuming {
7476 parked: vec!["running".to_owned()],
7477 };
7478 let next = advance_interrupt(state, &[], &[]);
7479 assert_eq!(
7480 next,
7481 Interrupt::Idle,
7482 "nothing is left to wait for; the loop must not stay wedged"
7483 );
7484 }
7485
7486 #[test]
7487 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7488 let marked = interrupt_task("marked");
7489 let next = advance_interrupt_tick(
7490 false,
7491 Interrupt::Idle,
7492 &["running".to_owned()],
7493 std::slice::from_ref(&marked),
7494 );
7495 assert_eq!(
7496 next,
7497 Interrupt::Idle,
7498 "an unmarked, unconfigured daemon must behave exactly as before"
7499 );
7500 }
7501
7502 #[test]
7503 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7504 let state = Interrupt::Parking {
7505 parked: vec!["running".to_owned()],
7506 interrupt_task: "marked".to_owned(),
7507 };
7508 let candidates = vec![interrupt_task("marked"), task()];
7509 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7510 assert!(
7511 allowed.is_empty(),
7512 "nothing may dispatch - not even the interrupt task itself - \
7513 until the parked run has actually stopped"
7514 );
7515 }
7516
7517 #[test]
7531 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7532 for state in [
7533 Interrupt::Parking {
7534 parked: vec!["running".to_owned()],
7535 interrupt_task: "marked".to_owned(),
7536 },
7537 Interrupt::Running {
7538 parked: vec!["running".to_owned()],
7539 interrupt_task: "marked".to_owned(),
7540 },
7541 Interrupt::Resuming {
7542 parked: vec!["running".to_owned()],
7543 },
7544 ] {
7545 let candidates = vec![
7546 interrupt_task("marked"),
7547 urgent_task("hot"),
7548 task_with_id("ordinary"),
7549 ];
7550 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7551 assert!(
7552 !allowed.iter().any(|t| t.id == "hot"),
7553 "an urgent candidate must wait out the same gate as anything \
7554 else while the run it would run alongside has not actually \
7555 left flight, for state {state:?}: {allowed:?}"
7556 );
7557 }
7558 }
7559
7560 #[test]
7566 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7567 let state = Interrupt::Resuming {
7568 parked: vec!["hot".to_owned()],
7569 };
7570 let candidates = vec![urgent_task("hot"), task()];
7571 let allowed = interrupt_gate(&state, &[], candidates);
7572 assert_eq!(
7573 allowed.iter().filter(|t| t.id == "hot").count(),
7574 1,
7575 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7576 );
7577 }
7578
7579 #[test]
7580 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7581 let state = Interrupt::Parking {
7582 parked: vec!["running".to_owned()],
7583 interrupt_task: "marked".to_owned(),
7584 };
7585 let other = task();
7586 let candidates = vec![interrupt_task("marked"), other.clone()];
7587 let allowed = interrupt_gate(&state, &[], candidates);
7588 assert_eq!(allowed.len(), 1);
7589 assert_eq!(allowed[0].id, "marked");
7590 }
7591
7592 #[test]
7593 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7594 let state = Interrupt::Running {
7595 parked: vec!["running".to_owned()],
7596 interrupt_task: "marked".to_owned(),
7597 };
7598 let candidates = vec![task(), task()];
7599 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7600 assert!(allowed.is_empty());
7601 }
7602
7603 #[test]
7610 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7611 let state = Interrupt::Resuming {
7612 parked: vec!["a".to_owned(), "c".to_owned()],
7613 };
7614 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7615 let allowed = interrupt_gate(&state, &[], candidates);
7616 assert_eq!(
7617 allowed.len(),
7618 1,
7619 "at most one candidate may be offered while resuming: {allowed:?}"
7620 );
7621 assert_eq!(allowed[0].id, "a");
7622 }
7623
7624 #[test]
7625 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7626 let state = Interrupt::Resuming {
7627 parked: vec!["a".to_owned()],
7628 };
7629 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7630 assert!(allowed.is_empty());
7631 }
7632
7633 #[test]
7639 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7640 let running = task(); let marked = interrupt_task("marked");
7642
7643 let mut state = Interrupt::Idle;
7644 let in_flight = vec![running.id.clone()];
7646 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7647 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7648 assert!(gated.is_empty(), "still waiting on `running` to park");
7649
7650 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7652 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7653 assert_eq!(
7654 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7655 vec!["marked"],
7656 "only the interrupt task may be offered to the dispatcher now"
7657 );
7658
7659 state = advance_interrupt_tick(
7661 true,
7662 state,
7663 &["marked".to_owned()],
7664 std::slice::from_ref(&running),
7665 );
7666 let gated = interrupt_gate(
7667 &state,
7668 &["marked".to_owned()],
7669 vec![marked.clone(), running.clone()],
7670 );
7671 assert!(
7672 gated.is_empty(),
7673 "the parked run must not be offered back while the interrupt \
7674 task is still running"
7675 );
7676
7677 let other = task_with_id("other");
7681 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7682 assert_eq!(
7683 state,
7684 Interrupt::Resuming {
7685 parked: vec![running.id.clone()]
7686 }
7687 );
7688 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7689 assert_eq!(
7690 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7691 vec![running.id.as_str()],
7692 "exactly the parked run resumes - not the unrelated task, even \
7693 though it was offered first"
7694 );
7695
7696 state = advance_interrupt_tick(
7700 true,
7701 state,
7702 std::slice::from_ref(&running.id),
7703 std::slice::from_ref(&other),
7704 );
7705 assert_eq!(state, Interrupt::Idle);
7706 let gated = interrupt_gate(
7707 &state,
7708 std::slice::from_ref(&running.id),
7709 vec![other.clone()],
7710 );
7711 assert_eq!(
7712 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7713 vec![other.id.as_str()],
7714 "ordinary dispatch is unrestricted again"
7715 );
7716 }
7717
7718 #[test]
7719 fn every_run_status_settles_the_task_it_came_from() {
7720 let table = [
7722 (RunStatus::Merged, TaskStatus::Done, 1),
7723 (RunStatus::Ready, TaskStatus::Done, 1),
7724 (RunStatus::Stalled, TaskStatus::Failed, 0),
7725 (RunStatus::Blocked, TaskStatus::Failed, 1),
7726 (RunStatus::Failed, TaskStatus::Failed, 1),
7727 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7728 (RunStatus::Prep, TaskStatus::Failed, 1),
7729 (RunStatus::Implementing, TaskStatus::Failed, 1),
7730 (RunStatus::Judging, TaskStatus::Failed, 1),
7731 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7732 (RunStatus::Voting, TaskStatus::Failed, 1),
7733 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7734 (RunStatus::Gating, TaskStatus::Failed, 1),
7735 ];
7736 for (run, want, attempts) in table {
7737 let mut t = task();
7738 t.start("20260902-000000-aaaa".to_owned());
7739 settle(
7740 &mut t,
7741 Verdict {
7742 status: run,
7743 left_pr: false,
7744 parked: false,
7745 quota_hit: matches!(run, RunStatus::Stalled),
7746 no_viable_candidates: false,
7747 },
7748 "why",
7749 2,
7750 );
7751 assert_eq!(t.status, want, "task status after {}", label(run));
7752 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7753 }
7754 }
7755
7756 #[test]
7757 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7758 let mut stalled = task();
7759 stalled.start("20260902-000000-aaaa".to_owned());
7760 settle(
7761 &mut stalled,
7762 Verdict {
7763 status: RunStatus::Stalled,
7764 left_pr: false,
7765 parked: false,
7766 quota_hit: true,
7767 no_viable_candidates: false,
7768 },
7769 "quota",
7770 1,
7771 );
7772 assert_eq!(stalled.attempts, 0);
7773 assert!(
7774 stalled.status.runnable(),
7775 "a machine problem must leave the task in line"
7776 );
7777
7778 let mut blocked = task();
7779 blocked.start("20260902-000000-aaaa".to_owned());
7780 settle(
7781 &mut blocked,
7782 Verdict {
7783 status: RunStatus::Blocked,
7784 left_pr: false,
7785 parked: false,
7786 quota_hit: false,
7787 no_viable_candidates: false,
7788 },
7789 "findings open",
7790 1,
7791 );
7792 assert_eq!(blocked.attempts, 1);
7793 assert_eq!(
7794 blocked.status,
7795 TaskStatus::Held,
7796 "the last attempt hands the task to a human"
7797 );
7798 }
7799
7800 #[test]
7801 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7802 let mut delivered = task();
7805 delivered.start("20260903-080619-01c2".to_owned());
7806 settle(
7807 &mut delivered,
7808 Verdict {
7809 status: RunStatus::Blocked,
7810 left_pr: true,
7811 parked: false,
7812 quota_hit: false,
7813 no_viable_candidates: false,
7814 },
7815 "no check status",
7816 4,
7817 );
7818 assert_eq!(
7819 delivered.status,
7820 TaskStatus::Held,
7821 "a pull request waiting on CI or a person is not a retryable failure"
7822 );
7823 assert!(
7824 !delivered.status.runnable(),
7825 "the loop must not pick this task up again"
7826 );
7827 assert_eq!(
7828 delivered.last_error.as_deref(),
7829 Some("no check status"),
7830 "the operator needs to be told what the gate was waiting for"
7831 );
7832
7833 let mut empty_handed = task();
7836 empty_handed.start("20260903-080619-01c2".to_owned());
7837 settle(
7838 &mut empty_handed,
7839 Verdict {
7840 status: RunStatus::Blocked,
7841 left_pr: false,
7842 parked: false,
7843 quota_hit: false,
7844 no_viable_candidates: false,
7845 },
7846 "findings open",
7847 4,
7848 );
7849 assert_eq!(empty_handed.status, TaskStatus::Failed);
7850 assert!(empty_handed.status.runnable());
7851 }
7852
7853 #[test]
7854 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7855 let mut noop = task();
7861 noop.start("20260912-131304-391f".to_owned());
7862 settle(
7863 &mut noop,
7864 Verdict {
7865 status: RunStatus::VerifiedNoop,
7866 left_pr: false,
7867 parked: false,
7868 quota_hit: false,
7869 no_viable_candidates: true,
7870 },
7871 "candidate A: already fixed by b32cfc4, on main",
7872 4,
7873 );
7874 assert_eq!(
7875 noop.status,
7876 TaskStatus::Held,
7877 "an unverified claim is a request for a human, not a failure"
7878 );
7879 assert!(
7880 !noop.status.runnable(),
7881 "the loop must not requeue this on the same unverified claim"
7882 );
7883 assert_eq!(noop.attempts, 1);
7888 }
7889
7890 #[test]
7891 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7892 let mut parked = task();
7897 parked.start("20260903-183634-2d98".to_owned());
7898 settle(
7899 &mut parked,
7900 Verdict {
7901 status: RunStatus::Implementing,
7902 left_pr: false,
7903 quota_hit: false,
7904 parked: true,
7905 no_viable_candidates: false,
7906 },
7907 "parked after `implementing`",
7908 2,
7909 );
7910 assert_eq!(parked.attempts, 0, "a park is refunded");
7911 assert!(
7912 parked.status.runnable(),
7913 "and the task stays in line so the next loop resumes its run"
7914 );
7915 assert_eq!(parked.status, TaskStatus::Parked);
7916 assert_eq!(
7917 parked.park_reason.as_deref(),
7918 Some("parked after `implementing`"),
7919 "the card says where it stopped"
7920 );
7921 assert_eq!(parked.last_error, None, "nothing failed");
7922
7923 let mut broken = task();
7927 broken.start("20260903-183634-2d98".to_owned());
7928 settle(
7929 &mut broken,
7930 Verdict {
7931 status: RunStatus::Implementing,
7932 left_pr: false,
7933 quota_hit: false,
7934 parked: false,
7935 no_viable_candidates: false,
7936 },
7937 "returned mid-flight",
7938 2,
7939 );
7940 assert_eq!(broken.attempts, 1);
7941 }
7942
7943 #[test]
7944 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
7945 let mut flaky = task();
7950 flaky.start("20260903-123023-e633".to_owned());
7951 settle(
7952 &mut flaky,
7953 Verdict {
7954 status: RunStatus::Stalled,
7955 left_pr: false,
7956 parked: false,
7957 quota_hit: false,
7958 no_viable_candidates: false,
7959 },
7960 "verdict rests on 1 of 3 judges",
7961 2,
7962 );
7963 assert_eq!(
7964 flaky.attempts, 1,
7965 "flakiness spends an attempt, so `max_attempts` still bounds it"
7966 );
7967 assert!(flaky.status.runnable(), "and it is still worth retrying");
7968
7969 let mut limited = task();
7971 limited.start("20260903-123023-e633".to_owned());
7972 settle(
7973 &mut limited,
7974 Verdict {
7975 status: RunStatus::Stalled,
7976 left_pr: false,
7977 parked: false,
7978 quota_hit: true,
7979 no_viable_candidates: false,
7980 },
7981 "judge-2, judge-3 out of quota",
7982 2,
7983 );
7984 assert_eq!(limited.attempts, 0, "a quota window is refunded");
7985 assert!(limited.status.runnable());
7986
7987 let mut worn = task();
7990 for _ in 0..2 {
7991 worn.release();
7992 }
7993 worn.start("20260903-123023-e633".to_owned());
7994 worn.attempts = 2;
7995 settle(
7996 &mut worn,
7997 Verdict {
7998 status: RunStatus::Stalled,
7999 left_pr: false,
8000 parked: false,
8001 quota_hit: false,
8002 no_viable_candidates: false,
8003 },
8004 "no quorum again",
8005 2,
8006 );
8007 assert_eq!(worn.status, TaskStatus::Held);
8008 assert!(!worn.status.runnable());
8009 }
8010
8011 #[test]
8012 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
8013 let mut wiped_out = task();
8020 wiped_out.start("20260907-025000-a1b2".to_owned());
8021 settle(
8022 &mut wiped_out,
8023 Verdict {
8024 status: RunStatus::Failed,
8025 left_pr: false,
8026 parked: false,
8027 quota_hit: true,
8028 no_viable_candidates: true,
8029 },
8030 "no candidate produced a change; nothing to judge",
8031 2,
8032 );
8033 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
8034 assert!(
8035 wiped_out.status.runnable(),
8036 "a machine problem must leave the task in line"
8037 );
8038
8039 let mut partial_progress = task();
8045 partial_progress.start("20260907-025500-c3d4".to_owned());
8046 settle(
8047 &mut partial_progress,
8048 Verdict {
8049 status: RunStatus::Failed,
8050 left_pr: false,
8051 parked: false,
8052 quota_hit: true,
8053 no_viable_candidates: false,
8054 },
8055 "gate failed on the winning candidate",
8056 2,
8057 );
8058 assert_eq!(
8059 partial_progress.attempts, 1,
8060 "a candidate that actually produced a change spends the attempt \
8061 even though some other seat hit its quota"
8062 );
8063 assert!(partial_progress.status.runnable());
8064 }
8065
8066 #[test]
8067 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
8068 let mut t = task();
8074 t.start("20260907-025000-a1b2".to_owned());
8075 let mut state = run_state(RunStatus::Failed);
8076 state.quota.push(QuotaLoss {
8077 cause: crate::agent::QuotaCause::UsageLimit,
8078 agent: String::new(),
8079 seat: "cand-a".to_owned(),
8080 node: "implement".to_owned(),
8081 at: Timestamp::now(),
8082 reset: None,
8083 });
8084 assert!(
8085 state.viable().is_empty(),
8086 "no candidate was added, so nothing is viable"
8087 );
8088 reclaim(&mut t, Some(state), 2, "en");
8089 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
8090 assert!(t.status.runnable());
8091 }
8092
8093 #[test]
8094 fn a_held_task_is_never_offered_to_the_loop() {
8095 let dir = tempfile::tempdir().unwrap();
8096 let queue = Queue::at(dir.path().to_path_buf());
8097 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
8098 let mut t = task();
8099 t.id = format!("2026090{n}-000000-000{n}");
8100 t.priority = priority;
8101 queue.put(&mut t).unwrap();
8102 }
8103 let mut held = task();
8104 held.id = "20260909-000000-9999".to_owned();
8105 held.priority = 99;
8106 held.hold_machine(None);
8107 queue.put(&mut held).unwrap();
8108
8109 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
8110 assert_eq!(order.len(), 3);
8111 assert!(!order.contains(&held.id));
8112 assert_eq!(
8113 order.first().cloned(),
8114 queue.next_runnable().map(|t| t.id),
8115 "the loop's first candidate is exactly what the queue offers"
8116 );
8117 assert_eq!(
8118 order,
8119 vec![
8120 "20260902-000000-0002".to_owned(),
8121 "20260903-000000-0003".to_owned(),
8122 "20260901-000000-0001".to_owned(),
8123 ],
8124 "priority first, then oldest, so nothing starves"
8125 );
8126 }
8127
8128 #[test]
8129 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
8130 let dir = tempfile::tempdir().unwrap();
8131 let queue = Queue::at(dir.path().to_path_buf());
8132 let mut old = task();
8133 old.id = "20260101-000000-old0".to_owned();
8134 queue.put(&mut old).unwrap();
8135 let mut fresh = task();
8136 fresh.id = "20260101-000000-new0".to_owned();
8137 queue.put(&mut fresh).unwrap();
8138
8139 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
8143 std::thread::sleep(Duration::from_millis(60));
8144 let live = queue.claim(&fresh.id).unwrap();
8145
8146 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8147 assert_eq!(swept, vec![old.id.clone()]);
8148 assert!(
8149 queue.claim(&old.id).is_ok(),
8150 "an unparseable lock older than the threshold is swept"
8151 );
8152 assert!(
8153 queue.claim(&fresh.id).is_err(),
8154 "a live pid protects its lock regardless of age"
8155 );
8156 drop(live);
8157 }
8158
8159 #[test]
8160 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
8161 let dir = tempfile::tempdir().unwrap();
8164 let queue = Queue::at(dir.path().to_path_buf());
8165 let mut t = task();
8166 t.id = "20260101-000000-live".to_owned();
8167 queue.put(&mut t).unwrap();
8168
8169 let claim = queue.claim(&t.id).unwrap();
8170 std::thread::sleep(Duration::from_millis(60));
8171
8172 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8173 assert!(
8174 swept.is_empty(),
8175 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
8176 );
8177 assert!(
8178 queue.claim(&t.id).is_err(),
8179 "the lock still protects its task"
8180 );
8181 drop(claim);
8182 }
8183
8184 fn injected_dead_pid() -> u32 {
8187 std::process::id().checked_add(1).unwrap_or(1)
8188 }
8189
8190 #[test]
8191 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
8192 let dir = tempfile::tempdir().unwrap();
8193 let queue = Queue::at(dir.path().to_path_buf());
8194 let mut t = task();
8195 t.id = "20260101-000000-dead".to_owned();
8196 queue.put(&mut t).unwrap();
8197 let dead_pid = injected_dead_pid();
8198
8199 std::fs::write(
8204 dir.path().join(format!("{}.lock", t.id)),
8205 dead_pid.to_string(),
8206 )
8207 .unwrap();
8208
8209 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8210 pid != dead_pid
8211 });
8212 assert_eq!(
8213 swept,
8214 vec![t.id.clone()],
8215 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
8216 );
8217 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
8218 }
8219
8220 #[test]
8221 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
8222 let dir = tempfile::tempdir().unwrap();
8223 let queue = Queue::at(dir.path().to_path_buf());
8224 let mut t = task();
8225 t.id = "20260101-000000-late".to_owned();
8226 queue.put(&mut t).unwrap();
8227 let dead_pid = injected_dead_pid();
8228
8229 assert!(
8232 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
8233 "nothing has claimed the task yet"
8234 );
8235
8236 std::fs::write(
8239 dir.path().join(format!("{}.lock", t.id)),
8240 dead_pid.to_string(),
8241 )
8242 .unwrap();
8243
8244 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8248 pid != dead_pid
8249 });
8250 assert_eq!(swept, vec![t.id.clone()]);
8251 }
8252
8253 #[test]
8254 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
8255 crate::run::pin_test_home();
8260 let dir = tempfile::tempdir().unwrap();
8261 let queue = Queue::at(dir.path().to_path_buf());
8262 let mut t = task();
8263 t.id = "20260101-000000-crsh".to_owned();
8264 t.status = TaskStatus::Running;
8265 t.attempts = 1;
8266 t.runs.push("20260904-000000-4043".to_owned());
8270 queue.put(&mut t).unwrap();
8271 let dead_pid = injected_dead_pid();
8272
8273 std::fs::write(
8276 dir.path().join(format!("{}.lock", t.id)),
8277 dead_pid.to_string(),
8278 )
8279 .unwrap();
8280
8281 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
8287 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8288
8289 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8290 pid != dead_pid
8291 });
8292 assert_eq!(swept, vec![t.id.clone()]);
8293
8294 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8295 assert_eq!(reclaimed, vec![t.id.clone()]);
8296 let after = queue.get(&t.id).unwrap();
8297 assert_eq!(
8298 after.status,
8299 TaskStatus::Held,
8300 "no run.json to recover from, so a human is asked"
8301 );
8302 assert_eq!(
8303 after.runs,
8304 vec!["20260904-000000-4043".to_owned()],
8305 "the crashed run's id is kept as evidence, not discarded"
8306 );
8307 }
8308
8309 #[test]
8310 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
8311 let dir = tempfile::tempdir().unwrap();
8312 let queue = Queue::at(dir.path().to_path_buf());
8313 let mut t = task();
8314 t.id = "20260101-000000-unknown".to_owned();
8315 queue.put(&mut t).unwrap();
8316 let dead_pid = injected_dead_pid();
8317 std::fs::write(
8318 dir.path().join(format!("{}.lock", t.id)),
8319 dead_pid.to_string(),
8320 )
8321 .unwrap();
8322
8323 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8324 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8325 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8326 }
8327
8328 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8329 let mut s = run_state(status);
8330 s.config.graph.language = language.to_owned();
8331 s
8332 }
8333
8334 fn unstarted_verdict(status: RunStatus) -> Verdict {
8335 Verdict {
8336 status,
8337 left_pr: false,
8338 quota_hit: false,
8339 parked: false,
8340 no_viable_candidates: false,
8341 }
8342 }
8343
8344 #[test]
8345 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8346 let mut t = task();
8347 t.attempts = 1;
8348 settle_in(
8349 &mut t,
8350 unstarted_verdict(RunStatus::AlreadyInBase),
8351 "already in main",
8352 1,
8353 phrases("en"),
8354 );
8355 assert_eq!(t.status, TaskStatus::Done);
8356 assert_eq!(t.attempts, 0);
8357 }
8358
8359 #[test]
8360 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8361 let reason = |language: &str| {
8362 let mut t = task();
8363 settle_in(
8364 &mut t,
8365 unstarted_verdict(RunStatus::Judging),
8366 "boom",
8367 1,
8368 phrases(language),
8369 );
8370 t.last_error.or(t.hold_reason).unwrap_or_default()
8371 };
8372 assert!(
8373 reason("en").starts_with("the graph stopped at `"),
8374 "{}",
8375 reason("en")
8376 );
8377 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8378 assert!(reason("日本語").contains("boom"));
8379 assert_eq!(reason("fr"), reason("en"));
8380 }
8381
8382 #[test]
8383 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8384 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8385 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8386 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8387 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8388 assert!(ja.contains("[run "), "{ja}");
8389 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8390 assert!(ended.starts_with("run 終了: "), "{ended}");
8391 let mut de = run_state_in(RunStatus::Failed, "de");
8392 let mut en = run_state_in(RunStatus::Failed, "en");
8393 de.id = "same".to_owned();
8394 en.id = "same".to_owned();
8395 assert_eq!(describe(&de), describe(&en));
8396 }
8397
8398 #[test]
8399 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8400 use crate::handover::Refused;
8401 let cases = [
8402 Refused::Foreign {
8403 branch: "b".into(),
8404 path: "/w/x".into(),
8405 why: "made by hand".into(),
8406 },
8407 Refused::Unsafe {
8408 branch: "b".into(),
8409 path: "/w/x".into(),
8410 why: "its worktree has uncommitted changes (a.rs)".into(),
8411 },
8412 Refused::ReleaseFailed {
8413 branch: "b".into(),
8414 path: "/w/x".into(),
8415 run: "ab12".into(),
8416 },
8417 ];
8418 for r in &cases {
8419 let en = (phrases("en").handover_refused)(r);
8420 assert_eq!(en, r.to_string());
8421 let ja = (phrases("ja").handover_refused)(r);
8422 assert!(
8423 !ja.contains("is checked out") && !ja.contains("try again"),
8424 "{ja}"
8425 );
8426 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8427 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8428 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8429 }
8430 }
8431 assert!(phrases("en").handover_hint.contains("release the task"));
8432 assert!(phrases("ja").handover_hint.contains("解放"));
8433 }
8434
8435 #[test]
8436 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8437 let build = |lang: &str| {
8438 let mut s = run_state_in(RunStatus::Blocked, lang);
8439 s.id = "same".to_owned();
8440 s.config.graph.review_rounds = 3;
8441 let mut r = review_round(3);
8442 r.expected = 3;
8443 r.answered = 1;
8444 s.reviews.push(r);
8445 s.event(
8446 "review",
8447 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8448 );
8449 s
8450 };
8451 let en = describe(&build("en"));
8452 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8453 let ja = describe(&build("ja"));
8454 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8455 assert!(!ja.contains("never answered"), "{ja}");
8456 assert!(ja.contains("[run "), "{ja}");
8457
8458 let mut failed = build("ja");
8461 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8462 command: "cargo test".to_owned(),
8463 code: Some(1),
8464 output_tail: String::new(),
8465 duration_ms: 0,
8466 resource_blocked: false,
8467 });
8468 failed.event("review", "stopped; e2e failed: cargo test");
8469 let ja = describe(&failed);
8470 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8471 assert!(!ja.contains("席のレビュアー"), "{ja}");
8472 }
8473
8474 #[test]
8475 fn refusals_and_recovery_prose_follow_the_language() {
8476 let t = held_task_with("r1");
8477 let q = action_question(
8478 "r1",
8479 ask::ChoiceAction::Resume {
8480 run: "r1".to_owned(),
8481 },
8482 );
8483 let refuse = |p: &Phrases| match decide_action(&t, &q, p, |_: &str| bail!("gone")) {
8484 ActionDecision::Refuse(s) => s,
8485 other => panic!("{other:?}"),
8486 };
8487 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8488 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8489 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8490
8491 let mut held = task();
8492 reclaim(&mut held, None, 2, "ja");
8493 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8494 let mut held = task();
8495 reclaim(&mut held, None, 2, "xx");
8496 assert!(held.hold_reason.unwrap().contains("held for a human"));
8497 }
8498
8499 fn run_state(status: RunStatus) -> RunState {
8500 let mut state = RunState::new(
8501 PathBuf::from("/repo"),
8502 "main".to_owned(),
8503 "abc1234def".to_owned(),
8504 "add retries".to_owned(),
8505 Config::default(),
8506 );
8507 state.status = status;
8508 state
8509 }
8510
8511 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8512 Candidate {
8513 index: 0,
8514 label,
8515 agent: "claude".to_owned(),
8516 branch: format!("magi/x/{label}"),
8517 worktree: PathBuf::from("/repo"),
8518 summary: summary.to_owned(),
8519 stat: String::new(),
8520 files: 0,
8521 commits: usize::from(!empty),
8522 empty,
8523 failed: failed.map(str::to_owned),
8524 verified_noop: None,
8525 duration_ms: 0,
8526 folded: false,
8527 }
8528 }
8529
8530 #[test]
8531 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8532 let mut state = run_state(RunStatus::Blocked);
8533 state.gate = vec![
8534 CommandOutcome {
8535 command: "cargo make check".to_owned(),
8536 code: Some(0),
8537 output_tail: "ok".to_owned(),
8538 duration_ms: 0,
8539 resource_blocked: false,
8540 },
8541 CommandOutcome {
8542 command: "cargo test".to_owned(),
8543 code: Some(101),
8544 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8545 duration_ms: 0,
8546 resource_blocked: false,
8547 },
8548 ];
8549 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8550 assert!(d.contains("cargo test"), "{d}");
8551 assert!(
8552 !d.contains("cargo make check"),
8553 "a passing check is not a diagnostic: {d}"
8554 );
8555 assert!(d.contains("assertion failed"), "{d}");
8556 }
8557
8558 #[test]
8559 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8560 let mut state = run_state(RunStatus::Blocked);
8561 state.event(
8562 "land",
8563 "stopped: the fixer produced no commit while 2 check(s) were failing \
8564 (build, lint); stopping instead of looping on an unchanged tree",
8565 );
8566 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8567 assert!(d.contains("build"), "{d}");
8568 assert!(d.contains("lint"), "{d}");
8569 assert!(d.contains("fixer produced no commit"), "{d}");
8570 }
8571
8572 #[test]
8573 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8574 let state = run_state(RunStatus::VerifiedNoop);
8579 let d = describe(&state);
8580 assert!(
8581 d.contains("agent-verified no-op"),
8582 "expected the display label, not the wire spelling: {d}"
8583 );
8584 assert!(!d.contains("verified_noop"), "{d}");
8585 }
8586
8587 #[test]
8588 fn diagnostic_keeps_gate_panic_before_large_backtrace() {
8589 let mut state = run_state(RunStatus::Failed);
8590 state.gate.push(crate::run::CommandOutcome {
8591 command: "cargo test".into(),
8592 code: Some(101),
8593 output_tail: format!(
8594 "---- widget stdout ----\nwidget log\nthread 'widget' panicked at widget.rs:12:\nassertion failed: widget missing\n{}test result: FAILED\n",
8595 " 41: 0x1234 - backtrace frame\n".repeat(10000)
8596 ),
8597 duration_ms: 1,
8598 resource_blocked: false,
8599 });
8600 let d = diagnostic(&state).unwrap();
8601 assert!(d.contains("panicked at widget.rs:12"), "{d}");
8602 assert!(d.contains("assertion failed: widget missing"));
8603 assert!(d.contains("widget log"));
8604 }
8605
8606 #[test]
8607 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8608 let mut state = run_state(RunStatus::Failed);
8614 state.candidates = vec![candidate(
8615 'A',
8616 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8617 true,
8618 None,
8619 )];
8620 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8621 assert!(d.contains("candidate A"), "{d}");
8622 assert!(d.contains("tagged v1.2.3"), "{d}");
8623 }
8624
8625 #[test]
8626 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8627 let mut state = run_state(RunStatus::Failed);
8628 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8629 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8630 assert!(d.contains("candidate A"), "{d}");
8631 assert!(d.contains("agent timed out"), "{d}");
8632 }
8633
8634 #[test]
8635 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8636 let mut state = run_state(RunStatus::Failed);
8639 state.candidates = vec![candidate('A', "did the work", false, None)];
8640 assert!(diagnostic(&state).is_none());
8641 }
8642
8643 #[test]
8644 fn diagnostic_is_bounded_however_much_a_run_printed() {
8645 let mut state = run_state(RunStatus::Blocked);
8646 state.gate = vec![
8647 CommandOutcome {
8648 command: "cargo test".to_owned(),
8649 code: Some(101),
8650 output_tail: "x".repeat(50_000),
8651 duration_ms: 0,
8652 resource_blocked: false,
8653 },
8654 CommandOutcome {
8655 command: "cargo clippy".to_owned(),
8656 code: Some(1),
8657 output_tail: "y".repeat(50_000),
8658 duration_ms: 0,
8659 resource_blocked: false,
8660 },
8661 ];
8662 state.candidates = vec![
8663 candidate('A', &"z".repeat(50_000), true, None),
8664 candidate('B', &"w".repeat(50_000), true, None),
8665 ];
8666 let d = diagnostic(&state).expect("plenty here to diagnose");
8667 assert!(
8668 d.len() <= DIAGNOSTIC_MAX,
8669 "diagnostic grew to {} bytes, unbounded",
8670 d.len()
8671 );
8672 }
8673
8674 #[test]
8675 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8676 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8677 let mut t = task();
8678 let mut state = run_state(status);
8679 t.pr_recovery = Some(crate::pr_recovery::Selection {
8680 url: "url".into(),
8681 branch: format!("magi/task-{}", t.id),
8682 head: "a".repeat(40),
8683 source_head: None,
8684 source_run: None,
8685 source_proof: None,
8686 disposition: Some(crate::pr_recovery::Disposition::Land),
8687 instruction: t.instruction.clone(),
8688 approved_at: Timestamp::now(),
8689 });
8690 state.pr_recovery = t.pr_recovery.clone();
8691 t.start(state.id.clone());
8692 settle_and_diagnose(
8693 &mut t,
8694 Verdict {
8695 status,
8696 left_pr: false,
8697 quota_hit: false,
8698 parked: false,
8699 no_viable_candidates: false,
8700 },
8701 "delivery stopped",
8702 5,
8703 &state,
8704 );
8705 assert_eq!(t.status, TaskStatus::Held);
8706 let reason = t.hold_reason.as_deref().unwrap();
8707 if status == RunStatus::Ready {
8708 assert!(reason.contains("Selected PR delivery is incomplete"));
8709 assert!(t.recovery_budget.is_none());
8710 } else {
8711 assert!(reason.contains("Selected recovery stopped"));
8712 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8713 }
8714 assert!(t.pr_recovery.is_some());
8715 }
8716 }
8717
8718 #[test]
8719 fn base_recovery_settlement_holds_without_replacement_or_duplicate_notice_cause() {
8720 let mut state = run_state(RunStatus::Voting);
8721 state.base_sync = Some(crate::run::BaseSync {
8722 tip: "base-tip".into(),
8723 behind: 3,
8724 attempts: 4,
8725 conflict: Some("base stopped".into()),
8726 already_in: None,
8727 });
8728 state.record_base_sync_stop("candidate-head");
8729 let mut t = task();
8730 t.start(state.id.clone());
8731 let verdict = Verdict {
8732 status: RunStatus::Blocked,
8733 left_pr: false,
8734 quota_hit: false,
8735 parked: false,
8736 no_viable_candidates: false,
8737 };
8738 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
8739 assert_eq!(
8740 t.status,
8741 TaskStatus::Held,
8742 "unspent task attempts cannot start another competition"
8743 );
8744 let at = t.held_at;
8745 let cause = t.hold_notice_cause.clone();
8746 state.begin_pass();
8747 t.release_operator("test-resume", None);
8748 t.start(state.id.clone());
8749 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
8750 assert_eq!(t.status, TaskStatus::Held);
8751 assert_eq!(t.held_at, at);
8752 assert_eq!(t.hold_notice_cause, cause);
8753 }
8754
8755 #[test]
8756 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8757 let mut state = run_state(RunStatus::Blocked);
8758 state.gate = vec![CommandOutcome {
8759 command: "cargo test".to_owned(),
8760 code: Some(101),
8761 output_tail: "assertion failed".to_owned(),
8762 duration_ms: 0,
8763 resource_blocked: false,
8764 }];
8765 let verdict = Verdict {
8766 status: RunStatus::Blocked,
8767 left_pr: false,
8768 quota_hit: false,
8769 parked: false,
8770 no_viable_candidates: false,
8771 };
8772
8773 let mut t = task();
8776 t.start("run-1".to_owned());
8777 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8778 assert_eq!(t.status, TaskStatus::Failed);
8779 assert!(t.diagnostic.is_none());
8780
8781 t.start("run-2".to_owned());
8784 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8785 assert_eq!(t.status, TaskStatus::Held);
8786 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8787 assert!(d.contains("cargo test"), "{d}");
8788 }
8789
8790 #[test]
8791 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8792 crate::run::pin_test_home();
8797 let home = crate::run::home();
8798 let state = run_state(RunStatus::VerifiedNoop);
8799 let mut q = ask::Question::new(
8800 state.id.clone(),
8801 "implement".to_owned(),
8802 "impl-A".to_owned(),
8803 "is this really a no-op?".to_owned(),
8804 String::new(),
8805 Vec::new(),
8806 );
8807 Questions::at(home.join("questions")).put(&mut q).unwrap();
8808
8809 let verdict = Verdict {
8810 status: RunStatus::VerifiedNoop,
8811 left_pr: false,
8812 quota_hit: false,
8813 parked: false,
8814 no_viable_candidates: false,
8815 };
8816 let mut t = task();
8817 t.start(state.id.clone());
8818 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8819
8820 assert_eq!(t.status, TaskStatus::Held);
8821 let reason = t.hold_reason.expect("a held task must record why");
8822 assert!(
8823 reason.starts_with("run ended agent-verified no-op"),
8824 "the original settle reason must survive unchanged: {reason}"
8825 );
8826 assert!(
8827 reason.contains(q.short()),
8828 "the open question's id must be named so the notice is actionable: {reason}"
8829 );
8830 }
8831
8832 #[test]
8833 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8834 crate::run::pin_test_home();
8835 let state = run_state(RunStatus::VerifiedNoop);
8836
8837 let verdict = Verdict {
8838 status: RunStatus::VerifiedNoop,
8839 left_pr: false,
8840 quota_hit: false,
8841 parked: false,
8842 no_viable_candidates: false,
8843 };
8844 let mut t = task();
8845 t.start(state.id.clone());
8846 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8847
8848 assert_eq!(t.status, TaskStatus::Held);
8849 assert_eq!(
8850 t.hold_reason.as_deref(),
8851 Some("run ended agent-verified no-op"),
8852 "nothing to append when the question was already answered or never asked"
8853 );
8854 }
8855
8856 #[test]
8857 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8858 crate::run::pin_test_home();
8859 let mut first = run_state(RunStatus::Blocked);
8860 first.id = "20260101-000000-sup1".to_owned();
8861 first.save().unwrap();
8862 let mut second = run_state(RunStatus::Merged);
8863 second.id = "20260101-000000-sup2".to_owned();
8864 second.save().unwrap();
8865
8866 let mut t = task();
8867 t.runs = vec![first.id.clone(), second.id.clone()];
8868 t.status = TaskStatus::Done;
8869
8870 supersede_prior_runs(&t, &crate::run::home());
8871
8872 assert_eq!(
8873 RunState::load(&first.id).unwrap().status,
8874 RunStatus::Superseded,
8875 "the first attempt's Blocked no longer needs anyone's attention"
8876 );
8877 assert_eq!(
8878 RunState::load(&second.id).unwrap().status,
8879 RunStatus::Merged,
8880 "the run that actually succeeded is left exactly as it was"
8881 );
8882 }
8883
8884 #[test]
8885 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8886 crate::run::pin_test_home();
8892 let mut first = run_state(RunStatus::Blocked);
8893 first.id = "20260101-000000-sup9".to_owned();
8894 first.driver_pid = Some(std::process::id());
8897 first.driver_started_at = Some(
8898 crate::proc::process_started_at(std::process::id())
8899 .expect("this test process's own start time must be queryable"),
8900 );
8901 first.save().unwrap();
8902 let mut second = run_state(RunStatus::Merged);
8903 second.id = "20260101-000000-supa".to_owned();
8904 second.save().unwrap();
8905
8906 let mut t = task();
8907 t.runs = vec![first.id.clone(), second.id.clone()];
8908 t.status = TaskStatus::Done;
8909
8910 supersede_prior_runs(&t, &crate::run::home());
8911
8912 assert_eq!(
8913 RunState::load(&first.id).unwrap().status,
8914 RunStatus::Blocked,
8915 "a live driver_pid means something is still actually working this run, \
8916 even though no daemon claims it - rewriting under it would just be \
8917 undone the next time that process saves"
8918 );
8919 }
8920
8921 #[test]
8922 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8923 let dir = tempfile::tempdir().unwrap();
8929 let home = dir.path().to_path_buf();
8930 let queue = Queue::at(dir.path().join("queue"));
8931
8932 let mut first = run_state(RunStatus::Blocked);
8933 first.id = "20260101-000000-supd".to_owned();
8934 first.driver_pid = Some(std::process::id());
8935 first.driver_started_at = Some(
8936 crate::proc::process_started_at(std::process::id())
8937 .expect("this test process's own start time must be queryable"),
8938 );
8939 first.save_under(&home).unwrap();
8940 let mut second = run_state(RunStatus::Merged);
8941 second.id = "20260101-000000-supe".to_owned();
8942 second.save_under(&home).unwrap();
8943
8944 let mut t = task();
8945 t.runs = vec![first.id.clone(), second.id.clone()];
8946 t.status = TaskStatus::Done;
8947 queue.put(&mut t).unwrap();
8948
8949 resweep_superseded_attempts(&queue, &home);
8950 assert_eq!(
8951 RunState::load_under(&first.id, &home).unwrap().status,
8952 RunStatus::Blocked,
8953 "still live on the first pass, so still untouched"
8954 );
8955
8956 let mut stale = RunState::load_under(&first.id, &home).unwrap();
8962 stale.driver_started_at = Some("1".to_owned());
8963 stale.save_under(&home).unwrap();
8964
8965 resweep_superseded_attempts(&queue, &home);
8966 assert_eq!(
8967 RunState::load_under(&first.id, &home).unwrap().status,
8968 RunStatus::Superseded,
8969 "the second pass catches up what the first one correctly skipped"
8970 );
8971 }
8972
8973 #[test]
8974 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
8975 crate::run::pin_test_home();
8976 let mut first = run_state(RunStatus::Blocked);
8977 first.id = "20260101-000000-sup3".to_owned();
8978 first.save().unwrap();
8979 let mut second = run_state(RunStatus::Blocked);
8980 second.id = "20260101-000000-sup4".to_owned();
8981 second.save().unwrap();
8982
8983 let mut t = task();
8984 t.runs = vec![first.id.clone(), second.id.clone()];
8985 t.status = TaskStatus::Failed;
8989
8990 supersede_prior_runs(&t, &crate::run::home());
8991
8992 assert_eq!(
8993 RunState::load(&first.id).unwrap().status,
8994 RunStatus::Blocked
8995 );
8996 assert_eq!(
8997 RunState::load(&second.id).unwrap().status,
8998 RunStatus::Blocked
8999 );
9000 }
9001
9002 #[test]
9003 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
9004 crate::run::pin_test_home();
9008 let mut first = run_state(RunStatus::Blocked);
9009 first.id = "20260101-000000-sup5".to_owned();
9010 first.save().unwrap();
9011
9012 let mut t = task();
9013 t.runs = vec![first.id.clone()];
9014 t.status = TaskStatus::Done;
9015
9016 supersede_prior_runs(&t, &crate::run::home());
9017
9018 assert_eq!(
9019 RunState::load(&first.id).unwrap().status,
9020 RunStatus::Blocked,
9021 "a single-attempt task has no earlier run to supersede"
9022 );
9023 }
9024
9025 #[test]
9026 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
9027 crate::run::pin_test_home();
9034 let mut first = run_state(RunStatus::Blocked);
9035 first.id = "20260101-000000-supb".to_owned();
9036 first.save().unwrap();
9037 let mut second = run_state(RunStatus::Failed);
9038 second.id = "20260101-000000-supc".to_owned();
9039 second.save().unwrap();
9040
9041 let mut t = task();
9042 t.runs = vec![first.id.clone(), second.id.clone()];
9043 t.status = TaskStatus::Done;
9044
9045 supersede_prior_runs(&t, &crate::run::home());
9046
9047 assert_eq!(
9048 RunState::load(&first.id).unwrap().status,
9049 RunStatus::Blocked,
9050 "the task's last attempt never landed, so there is nothing here \
9051 actually superseding it"
9052 );
9053 }
9054
9055 #[test]
9056 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
9057 crate::run::pin_test_home();
9061 let mut failed = run_state(RunStatus::Failed);
9062 failed.id = "20260101-000000-sup6".to_owned();
9063 failed.save().unwrap();
9064 let mut noop = run_state(RunStatus::VerifiedNoop);
9065 noop.id = "20260101-000000-sup7".to_owned();
9066 noop.save().unwrap();
9067 let mut winner = run_state(RunStatus::Ready);
9068 winner.id = "20260101-000000-sup8".to_owned();
9069 winner.save().unwrap();
9070
9071 let mut t = task();
9072 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
9073 t.status = TaskStatus::Done;
9074
9075 supersede_prior_runs(&t, &crate::run::home());
9076
9077 assert_eq!(
9078 RunState::load(&failed.id).unwrap().status,
9079 RunStatus::Failed
9080 );
9081 assert_eq!(
9082 RunState::load(&noop.id).unwrap().status,
9083 RunStatus::VerifiedNoop
9084 );
9085 }
9086
9087 fn approval_question(run: &str) -> ask::Question {
9088 ask::Question::new(
9089 run.to_owned(),
9090 land::APPROVAL_NODE.to_owned(),
9091 "land".to_owned(),
9092 "merge?".to_owned(),
9093 String::new(),
9094 vec!["merge".to_owned(), "hold".to_owned()],
9095 )
9096 }
9097
9098 #[test]
9099 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
9100 crate::run::pin_test_home();
9101 let mut state = run_state(RunStatus::Landing);
9102 state.id = "20260101-000000-fre1".to_owned();
9103 state.parked = true;
9104 state.save().unwrap();
9105 ask::Questions::open()
9106 .put(&mut approval_question(&state.id))
9107 .unwrap();
9108
9109 let mut t = task();
9110 t.runs.push(state.id.clone());
9111 assert_eq!(
9112 land_resume_state(&t),
9113 LandResume::StillWaiting,
9114 "nobody has answered and the timeout has not passed"
9115 );
9116 }
9117
9118 #[test]
9119 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
9120 crate::run::pin_test_home();
9121 let mut state = run_state(RunStatus::Gating);
9122 state.id = "20260101-000000-gt01".to_owned();
9123 state.parked = true;
9124 state.config.graph.answer_timeout = 60;
9125 state.github_text = Some(crate::run::GithubTextAsk {
9126 rewrite: None,
9127 fingerprint: "f".into(),
9128 title: true,
9129 body: false,
9130 categories: vec!["title-language".into()],
9131 question: None,
9132 asks: 1,
9133 resolved: false,
9134 chosen_title: None,
9135 choice: None,
9136 });
9137 state.save().unwrap();
9138 let store = ask::Questions::open();
9139 let mut q = ask::Question::new(
9140 state.id.clone(),
9141 crate::github_text::ASK_NODE.to_owned(),
9142 crate::github_text::ASK_SEAT.to_owned(),
9143 "s".into(),
9144 String::new(),
9145 vec!["use fallback".into()],
9146 );
9147 store.put(&mut q).unwrap();
9148 let mut t = task();
9149 t.runs.push(state.id.clone());
9150 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
9151 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9152 store.put(&mut q).unwrap();
9153 assert_eq!(land_resume_state(&t), LandResume::Ready);
9154 assert!(!store.get(&q.id).unwrap().status.open());
9155 state.github_text.as_mut().unwrap().resolved = true;
9157 state.save().unwrap();
9158 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
9159 }
9160
9161 #[test]
9162 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
9163 crate::run::pin_test_home();
9168 let mut state = run_state(RunStatus::Landing);
9169 state.id = "20260101-000000-exp1".to_owned();
9170 state.parked = true;
9171 state.config.graph.answer_timeout = 60;
9172 state.save().unwrap();
9173
9174 let store = ask::Questions::open();
9175 let mut q = approval_question(&state.id);
9176 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9177 store.put(&mut q).unwrap();
9178
9179 let mut t = task();
9180 t.runs.push(state.id.clone());
9181 assert_eq!(
9182 land_resume_state(&t),
9183 LandResume::Ready,
9184 "an expired question must not be waited on forever"
9185 );
9186
9187 let after = store.get(&q.id).unwrap();
9188 assert!(
9189 !after.status.open(),
9190 "the question is abandoned, not silently ignored"
9191 );
9192 assert!(
9193 after.resolution().is_none(),
9194 "an abandoned question is not read as a decision"
9195 );
9196 }
9197
9198 #[test]
9199 fn reclaim_settles_a_running_task_against_its_last_run() {
9200 let mut t = task();
9201 t.start("20260904-000000-4043".to_owned());
9202 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
9203 assert_eq!(
9204 t.status,
9205 TaskStatus::Done,
9206 "a run that actually finished must not stay `running` forever"
9207 );
9208 }
9209
9210 #[test]
9211 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
9212 let mut t = task();
9213 t.start("20260904-000000-4043".to_owned());
9214 t.last_error = Some("earlier trouble".to_owned());
9215 let mut state = run_state(RunStatus::Judging);
9216 state.parked = true;
9217 reclaim(&mut t, Some(state), 1, "en");
9218 assert_eq!(t.status, TaskStatus::Parked);
9219 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
9220 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9221 assert!(t.park_reason.is_some());
9222 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
9223 assert!(!t.fresh_start);
9224 assert!(t.status.runnable());
9225 }
9226
9227 #[test]
9228 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
9229 let mut t = task();
9230 t.start("20260904-000000-4043".to_owned());
9231 t.last_error = Some("earlier trouble".to_owned());
9232 settle(
9233 &mut t,
9234 Verdict {
9235 status: RunStatus::Judging,
9236 left_pr: false,
9237 quota_hit: false,
9238 parked: true,
9239 no_viable_candidates: false,
9240 },
9241 "parked after `judging`",
9242 1,
9243 );
9244 assert_eq!(t.status, TaskStatus::Parked);
9245 assert_eq!(t.attempts, 0);
9246 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9247 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
9248 assert_eq!(t.schema, crate::queue::SCHEMA);
9249 assert!(
9250 crate::notices::task_held(&t).is_none(),
9251 "a park is not news"
9252 );
9253 t.start("20260904-000000-4043".to_owned());
9254 assert_eq!(t.park_reason, None);
9255 }
9256
9257 #[test]
9258 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
9259 let mut t = task();
9263 t.start("20260904-000000-4043".to_owned());
9264 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
9265 assert_eq!(t.status, TaskStatus::Failed);
9266 assert!(t.status.runnable());
9267 }
9268
9269 #[test]
9270 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
9271 let mut t = task();
9272 t.start("20260904-000000-4043".to_owned());
9273 assert_eq!(t.attempts, 1);
9274 let mut run = run_state(RunStatus::Implementing);
9275 run.id = "20260904-000000-4043".to_owned();
9276 run.begin_pass();
9277 let cause = crate::run::ParkCause::Upgrade {
9278 from: "0.121.0".to_owned(),
9279 to: Some("0.121.1".to_owned()),
9280 };
9281 run.end_pass_parked(cause.clone());
9282 run.parked = true;
9283 let verdict = Verdict {
9284 status: run.status,
9285 left_pr: false,
9286 quota_hit: false,
9287 parked: true,
9288 no_viable_candidates: false,
9289 };
9290 let mut orphan = t.clone();
9293 reclaim(&mut orphan, Some(run.clone()), 2, "en");
9294 assert_eq!(orphan.status, TaskStatus::Parked);
9295 assert_eq!(orphan.attempts, 0);
9296 assert_eq!(
9297 orphan.upgrade_resume_key(),
9298 Some((run.passes[0].started_at, orphan.id.as_str()))
9299 );
9300 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
9301 assert_eq!(t.attempts, 0);
9302 assert_eq!(t.passes.len(), 1);
9303 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9304 assert_eq!(t.passes[0].parked, Some(cause));
9305 assert_eq!(t.passes[0].pass_index, 0);
9306 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
9307 assert!(t.upgrade_resume_key().is_some());
9308
9309 t.start("20260904-000000-4043".to_owned());
9312 run.begin_pass();
9313 run.status = RunStatus::Failed;
9314 let verdict = Verdict {
9315 status: RunStatus::Failed,
9316 left_pr: false,
9317 quota_hit: false,
9318 parked: false,
9319 no_viable_candidates: false,
9320 };
9321 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
9322 assert_eq!(t.passes.len(), 2);
9323 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9324 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
9325 assert_eq!(t.passes[1].pass_index, 1);
9326 assert_eq!(t.attempts, 1);
9327 }
9328
9329 #[test]
9330 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
9331 let mut t = task();
9332 t.start("20260904-000000-4043".to_owned());
9333 reclaim(&mut t, None, 2, "en");
9334 assert_eq!(t.status, TaskStatus::Held);
9335 assert!(
9336 t.last_error
9337 .as_deref()
9338 .is_some_and(|e| e.contains("running")),
9339 "the operator needs to know why this task was held"
9340 );
9341 }
9342
9343 #[test]
9344 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
9345 let dir = tempfile::tempdir().unwrap();
9346 let queue = Queue::at(dir.path().to_path_buf());
9347
9348 let mut orphaned = task();
9350 orphaned.id = "20260904-000000-orph".to_owned();
9351 orphaned.status = TaskStatus::Running;
9352 orphaned.attempts = 1;
9353 queue.put(&mut orphaned).unwrap();
9354
9355 let mut alive = task();
9356 alive.id = "20260904-000000-live".to_owned();
9357 alive.status = TaskStatus::Running;
9358 alive.attempts = 1;
9359 queue.put(&mut alive).unwrap();
9360 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
9361
9362 let mut queued = task();
9363 queued.id = "20260904-000000-wait".to_owned();
9364 queue.put(&mut queued).unwrap();
9365
9366 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9367 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
9368
9369 assert_eq!(
9370 queue.get(&orphaned.id).unwrap().status,
9371 TaskStatus::Held,
9372 "nothing was driving it and there was no run to recover"
9373 );
9374 assert_eq!(
9375 queue.get(&alive.id).unwrap().status,
9376 TaskStatus::Running,
9377 "a live claim must protect the task it belongs to"
9378 );
9379 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9380 }
9381
9382 fn read_run_under(home: &Path, id: &str) -> RunState {
9388 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9389 serde_json::from_str(&body).unwrap()
9390 }
9391
9392 #[test]
9393 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9394 let dir = tempfile::tempdir().unwrap();
9395 let home = dir.path().to_path_buf();
9396 let now = Timestamp::now();
9397 let overrun_seat = || crate::run::ActiveSeat {
9398 node: "implement".to_owned(),
9399 started_at: now - jiff::SignedDuration::new(21_000, 0),
9400 timeout_secs: 3_600,
9401 attempt: 0,
9402 task: None,
9403 command: None,
9404 index: None,
9405 total: None,
9406 };
9407
9408 let mut dead = run_state(RunStatus::Implementing);
9409 dead.id = "20260101-000000-dead".to_owned();
9410 dead.active.insert("impl-A".to_owned(), overrun_seat());
9411 dead.driver_pid = Some(4242);
9414 dead.save_under(&home).unwrap();
9415
9416 let mut alive = run_state(RunStatus::Implementing);
9419 alive.id = "20260101-000000-aliv".to_owned();
9420 alive.active.insert("impl-A".to_owned(), overrun_seat());
9421 alive.save_under(&home).unwrap();
9422 let mut status = Status::new();
9423 status.current = vec![Current {
9424 worker: None,
9425 task: "20260101-000000-task".to_owned(),
9426 run: alive.id.clone(),
9427 }];
9428 write_status_to(&home.join("daemon.json"), &status).unwrap();
9429
9430 let questions = Questions::at(home.join("questions"));
9434 let mut q = ask::Question::new(
9435 dead.id.clone(),
9436 "implement".to_owned(),
9437 "impl-A".to_owned(),
9438 "Which storage backend?".to_owned(),
9439 String::new(),
9440 vec!["SQLite".to_owned(), "Redis".to_owned()],
9441 );
9442 questions.put(&mut q).unwrap();
9443
9444 let abandoned = reclaim_abandoned_runs_with(
9445 &home,
9446 now,
9447 |pid| if pid == 4242 { Some(false) } else { None },
9448 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9449 );
9450 assert_eq!(abandoned, vec![dead.id.clone()]);
9451
9452 let reloaded = read_run_under(&home, &dead.id);
9453 assert_eq!(reloaded.status, RunStatus::Failed);
9454 assert!(reloaded.active.is_empty());
9455 assert!(
9456 !questions.get(&q.id).unwrap().status.open(),
9457 "the failed run's own open question must be settled in the same pass"
9458 );
9459
9460 let still_alive = read_run_under(&home, &alive.id);
9461 assert_eq!(
9462 still_alive.status,
9463 RunStatus::Implementing,
9464 "a live daemon's claim protects it"
9465 );
9466 assert!(!still_alive.active.is_empty());
9467 }
9468
9469 #[test]
9479 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9480 let dir = tempfile::tempdir().unwrap();
9481 let home = dir.path().to_path_buf();
9482 let now = Timestamp::now();
9483
9484 let mut manual = run_state(RunStatus::Reviewing);
9485 manual.id = "20260101-000000-manl".to_owned();
9486 manual.active.insert(
9487 "review-1".to_owned(),
9488 crate::run::ActiveSeat {
9489 node: "review".to_owned(),
9490 started_at: now - jiff::SignedDuration::new(21_000, 0),
9491 timeout_secs: 3_600,
9492 attempt: 0,
9493 task: None,
9494 command: None,
9495 index: None,
9496 total: None,
9497 },
9498 );
9499 manual.driver_pid = Some(4242);
9503 manual.driver_started_at = Some("1790000000".to_owned());
9504 manual.save_under(&home).unwrap();
9505
9506 let abandoned = reclaim_abandoned_runs_with(
9507 &home,
9508 now,
9509 |pid| if pid == 4242 { Some(true) } else { None },
9510 |pid| {
9511 if pid == 4242 {
9512 Some("1790000000".to_owned())
9513 } else {
9514 None
9515 }
9516 },
9517 );
9518 assert!(
9519 abandoned.is_empty(),
9520 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9521 );
9522
9523 let reloaded = read_run_under(&home, &manual.id);
9524 assert_eq!(reloaded.status, RunStatus::Reviewing);
9525 assert!(!reloaded.active.is_empty());
9526 }
9527
9528 #[test]
9529 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9530 let dir = tempfile::tempdir().unwrap();
9531 let queue = Queue::at(dir.path().to_path_buf());
9532 let mut only = task();
9533 queue.put(&mut only).unwrap();
9534
9535 let _elsewhere = queue.claim(&only.id).unwrap();
9536 let candidates = runnable(&queue);
9537 assert_eq!(candidates.len(), 1, "the task is still runnable");
9538 assert!(
9539 queue.claim(&candidates[0].id).is_err(),
9540 "the loop cannot take a claim somebody else holds"
9541 );
9542
9543 let after = queue.get(&only.id).unwrap();
9544 assert_eq!(after.status, TaskStatus::Queued);
9545 assert_eq!(
9546 after.attempts, 0,
9547 "losing the race is not an attempt at the task"
9548 );
9549 assert_eq!(after.last_error, None);
9550 }
9551
9552 #[test]
9553 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9554 let dir = tempfile::tempdir().unwrap();
9555 let path = dir.path().join("daemon.json");
9556
9557 let mut status = Status::new();
9558 status.idle = false;
9559 status.completed = 7;
9560 status.current = vec![Current {
9561 worker: None,
9562 task: "20260902-000000-t111".to_owned(),
9563 run: "20260902-000001-r111".to_owned(),
9564 }];
9565 write_status_to(&path, &status).unwrap();
9566 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9567 assert_eq!(first.schema, SCHEMA);
9568 assert_eq!(first.pid, std::process::id());
9569 assert!(!first.idle);
9570 assert_eq!(first.completed, 7);
9571 assert_eq!(first.current, status.current);
9572 assert!(
9573 !path.with_extension("json.tmp").exists(),
9574 "the temp file is renamed, not left behind"
9575 );
9576
9577 std::thread::sleep(Duration::from_millis(5));
9578 status.updated_at = Timestamp::now();
9579 status.polls = 3;
9580 write_status_to(&path, &status).unwrap();
9581 let second: Status =
9582 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9583 assert!(
9584 second.updated_at > first.updated_at,
9585 "a reader can only detect staleness if the heartbeat moves"
9586 );
9587 assert_eq!(
9588 second.started_at, first.started_at,
9589 "the start time is not a heartbeat"
9590 );
9591 assert_eq!(second.polls, 3);
9592 }
9593
9594 #[test]
9595 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9596 let dir = tempfile::tempdir().unwrap();
9597
9598 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9599
9600 let mut status = Status::new();
9601 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9602 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9603 let stale = read_status(dir.path()).unwrap();
9604 assert!(
9605 !stale.running(Timestamp::now()),
9606 "a minute without a heartbeat is a dead daemon, not a busy one"
9607 );
9608 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9609
9610 status.updated_at = Timestamp::now();
9611 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9612 let fresh = read_status(dir.path()).unwrap();
9613 assert!(fresh.running(Timestamp::now()));
9614 }
9615
9616 #[test]
9617 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9618 let dir = tempfile::tempdir().unwrap();
9619 let now = Timestamp::now();
9620 let mine = "20260903-080619-01c2";
9621
9622 assert!(
9623 !is_working_on(dir.path(), mine, now),
9624 "no status file means nobody is working on anything"
9625 );
9626
9627 let mut status = Status::new();
9628 status.current = vec![Current {
9629 worker: None,
9630 task: "20260903-080340-0167".to_owned(),
9631 run: mine.to_owned(),
9632 }];
9633 status.updated_at = now;
9634 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9635 assert!(is_working_on(dir.path(), mine, now));
9636 assert!(
9637 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9638 "a daemon busy with one run is not working on another"
9639 );
9640
9641 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9644 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9645 assert!(
9646 !is_working_on(dir.path(), mine, now),
9647 "a stale heartbeat is a dead daemon, so its run is a leftover"
9648 );
9649 }
9650
9651 #[test]
9652 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9653 let dir = tempfile::tempdir().unwrap();
9654 let now = Timestamp::now();
9655
9656 assert!(
9657 !is_working_on_short(dir.path(), "01c2", now),
9658 "no status file means nobody is working on anything"
9659 );
9660
9661 let mut status = Status::new();
9662 status.current = vec![Current {
9663 worker: None,
9664 task: "20260903-080340-0167".to_owned(),
9665 run: "20260903-080619-01c2".to_owned(),
9666 }];
9667 status.updated_at = now;
9668 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9669 assert!(
9670 is_working_on_short(dir.path(), "01c2", now),
9671 "the run's short id is the last block of its full id"
9672 );
9673 assert!(
9674 !is_working_on_short(dir.path(), "3cbf", now),
9675 "a daemon busy with one worktree bay is not working on another"
9676 );
9677 }
9678
9679 #[test]
9680 fn a_newer_status_file_still_yields_a_reading() {
9681 let dir = tempfile::tempdir().unwrap();
9682 std::fs::write(
9685 dir.path().join("daemon.json"),
9686 serde_json::json!({
9687 "schema": 2,
9688 "updated_at": Timestamp::now().to_string(),
9689 "idle": true,
9690 "surprise": { "nested": [1, 2, 3] },
9691 })
9692 .to_string(),
9693 )
9694 .unwrap();
9695
9696 let reading = read_status(dir.path()).expect("a forward-compatible read");
9697 assert!(reading.running(Timestamp::now()));
9698 assert!(reading.idle);
9699 assert!(reading.current.is_empty());
9700 }
9701
9702 #[test]
9703 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9704 let dir = tempfile::tempdir().unwrap();
9710 std::fs::write(
9711 dir.path().join("daemon.json"),
9712 serde_json::json!({
9713 "schema": 1,
9714 "pid": 4242,
9715 "updated_at": Timestamp::now().to_string(),
9716 "idle": false,
9717 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9718 "completed": 3,
9719 "polls": 9,
9720 })
9721 .to_string(),
9722 )
9723 .unwrap();
9724
9725 let reading = read_status(dir.path()).expect("an older shape must still parse");
9726 assert!(reading.running(Timestamp::now()));
9727 assert_eq!(
9728 reading.current,
9729 vec![Current {
9730 worker: None,
9731 task: "20260902-140501-aaaa".to_owned(),
9732 run: "20260902-140502-bbbb".to_owned(),
9733 }]
9734 );
9735 }
9736
9737 #[test]
9738 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9739 let dir = tempfile::tempdir().unwrap();
9740 std::fs::write(
9741 dir.path().join("daemon.json"),
9742 serde_json::json!({
9743 "schema": 1,
9744 "updated_at": Timestamp::now().to_string(),
9745 "idle": true,
9746 "current": null,
9747 })
9748 .to_string(),
9749 )
9750 .unwrap();
9751 let with_null = read_status(dir.path()).expect("null must still parse");
9752 assert!(with_null.current.is_empty());
9753
9754 std::fs::write(
9755 dir.path().join("daemon.json"),
9756 serde_json::json!({
9757 "schema": 1,
9758 "updated_at": Timestamp::now().to_string(),
9759 "idle": true,
9760 })
9761 .to_string(),
9762 )
9763 .unwrap();
9764 let absent = read_status(dir.path()).expect("a missing field must still parse");
9765 assert!(absent.current.is_empty());
9766 }
9767
9768 #[test]
9769 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9770 let fallback = Path::new("/default");
9771 let mut blank = task();
9772 blank.repo = PathBuf::new();
9773 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9774 let mut dot = task();
9775 dot.repo = PathBuf::from(".");
9776 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9777 assert_eq!(
9778 repo_for(&task(), fallback),
9779 PathBuf::from("/repo"),
9780 "a task that names a repository keeps it"
9781 );
9782 }
9783
9784 #[test]
9785 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9786 let dir = tempfile::tempdir().unwrap();
9787 let path = dir.path().join("explicit.toml");
9788 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9789 let opts = Opts {
9790 config: Some(path),
9791 merge: Some("none".into()),
9792 ..Default::default()
9793 };
9794 let mut t = task();
9795 t.set_implementers(
9796 2,
9797 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9798 .unwrap(),
9799 )
9800 .unwrap();
9801 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9802 assert_eq!(cfg.graph.implementers, 2);
9803 assert_eq!(cfg.graph.judges, 1);
9804 assert_eq!(cfg.merge.mode, MergeMode::None);
9805 t.set_implementers(1, &cfg).unwrap();
9806 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9807 apply_task_implementers(&mut cfg, &t);
9808 assert_eq!(cfg.graph.implementers, 1);
9809 assert_eq!(cfg.graph.judges, 1);
9810 assert_eq!(cfg.merge.mode, MergeMode::None);
9811 }
9812
9813 #[test]
9814 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9815 let dir = tempfile::tempdir().unwrap();
9816 let repo = dir.path().join("repo");
9817 std::fs::create_dir(&repo).unwrap();
9818 std::fs::write(
9819 repo.join("magi.toml"),
9820 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9821 )
9822 .unwrap();
9823 let daemon_config = dir.path().join("daemon.toml");
9824 std::fs::write(
9825 &daemon_config,
9826 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9827 )
9828 .unwrap();
9829 let opts = Opts {
9830 config: Some(daemon_config),
9831 merge: Some("pr".into()),
9832 ..Default::default()
9833 };
9834 let q = Queue::at(dir.path().join("queue"));
9835 let validation: Config =
9836 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9837 .unwrap();
9838 for record in ["{}", r#"{"candidates":2}"#] {
9840 let overrides = serde_json::from_str(record).unwrap();
9841 let filing = crate::direct::Filing {
9842 instruction: "work".into(),
9843 title: "direct".into(),
9844 repo: repo.clone(),
9845 source: crate::queue::Source::Human,
9846 solo: false,
9847 overrides,
9848 review_of: None,
9849 sources: Default::default(),
9850 };
9851 let filed =
9852 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9853 let crate::direct::Filed::Standalone {
9854 mut task,
9855 claim: _claim,
9856 } = filed
9857 else {
9858 panic!("local filing")
9859 };
9860 let before = prepare_for(&repo, &opts, &task).unwrap();
9861 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9862 assert_eq!(
9863 before.merge.mode,
9864 MergeMode::None,
9865 "daemon --merge must not leak"
9866 );
9867 for n in [2, 1, 2] {
9868 task.set_implementers(n, &validation).unwrap();
9869 q.put(&mut task).unwrap();
9870 task = q.get(&task.id).unwrap();
9871 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9872 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9873 apply_task_implementers(&mut cfg, &task);
9874 assert_eq!(cfg.graph.implementers, usize::from(n));
9875 assert_eq!(cfg.graph.judges, 2);
9876 assert_eq!(cfg.merge.mode, MergeMode::None);
9877 }
9878 }
9879 }
9880
9881 #[test]
9882 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9883 let cfg: Config =
9884 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9885 .unwrap();
9886 let mut t = task();
9887 t.review_of = Some("review-target".into());
9888 t.hold_manual(Some("retry".into()));
9889 let mut previous = RunState::new(
9890 PathBuf::from("/repo"),
9891 "main".into(),
9892 "abc1234".into(),
9893 "review".into(),
9894 cfg.clone(),
9895 );
9896 previous.config.graph.implementers = 2;
9897 for count in [1, 3] {
9898 t.set_implementers(count, &cfg).unwrap();
9899 t.release();
9900 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9901 for proposed in [
9902 Starter::Resume(previous.id.clone()),
9903 Starter::Review("recovery-target".into()),
9904 ] {
9905 assert_eq!(
9906 starter_with_implementers(&t, Some(&previous), proposed),
9907 Starter::Review("review-target".into())
9908 );
9909 }
9910 let mut configured = cfg.clone();
9911 apply_task_implementers(&mut configured, &t);
9912 assert_eq!(configured.graph.implementers, usize::from(count));
9913 assert_eq!(previous.config.graph.implementers, 2);
9914 }
9915 }
9916
9917 #[test]
9918 fn task_implementers_dispatch_and_resume_agree() {
9919 let mut t = task();
9920 let cfg: crate::config::Config =
9921 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9922 .unwrap();
9923 let mut run = RunState::new(
9924 PathBuf::from("/repo"),
9925 "main".into(),
9926 "abc1234".into(),
9927 "work".into(),
9928 cfg.clone(),
9929 );
9930 run.config.graph.implementers = 2;
9931 for n in [2, 1, 3, 2] {
9932 t.set_implementers(n, &cfg).unwrap();
9933 let mut dispatched = cfg.clone();
9934 apply_task_implementers(&mut dispatched, &t);
9935 assert_eq!(dispatched.graph.implementers, usize::from(n));
9936 assert_eq!(implementers_match(&t, &run), n == 2);
9937 for proposed in [
9938 Starter::Resume(run.id.clone()),
9939 Starter::Review("old-branch".into()),
9940 ] {
9941 let expected = if n == 2 {
9942 proposed.clone()
9943 } else {
9944 Starter::Start
9945 };
9946 assert_eq!(
9947 starter_with_implementers(&t, Some(&run), proposed),
9948 expected
9949 );
9950 }
9951 assert_eq!(
9952 run.config.graph.implementers, 2,
9953 "history must not be rewritten"
9954 );
9955 }
9956 t.solo = false;
9957 t.overrides = None;
9958 assert!(implementers_match(&t, &run));
9959 }
9960
9961 #[test]
9962 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
9963 let mut solo_cfg = Config::default();
9969 solo_cfg.graph.implementers = 3;
9970 let mut solo_task = task();
9971 solo_task.solo = true;
9972 apply_task_implementers(&mut solo_cfg, &solo_task);
9973 assert_eq!(solo_cfg.graph.implementers, 1);
9974
9975 let mut plain_cfg = Config::default();
9976 plain_cfg.graph.implementers = 3;
9977 let plain_task = task();
9978 assert!(!plain_task.solo);
9979 apply_task_implementers(&mut plain_cfg, &plain_task);
9980 assert_eq!(
9981 plain_cfg.graph.implementers, 3,
9982 "a task that did not ask to run alone keeps the config's candidates"
9983 );
9984 }
9985
9986 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
9987 QuotaLoss {
9988 cause: crate::agent::QuotaCause::UsageLimit,
9989 agent: String::new(),
9990 seat: seat.into(),
9991 node: "judge".into(),
9992 at: at.parse().unwrap(),
9993 reset: reset.map(str::to_string),
9994 }
9995 }
9996
9997 #[test]
9998 fn authentication_losses_never_arm_a_cooldown_even_with_reset_hints() {
9999 let now = Timestamp::now();
10000 for reset in [None, Some("in 1h")] {
10001 let auth = crate::agent::Quota {
10002 cause: crate::agent::QuotaCause::Authentication,
10003 agent: "alpha".into(),
10004 reset: reset.map(str::to_owned),
10005 };
10006 let loss = QuotaLoss::from_report("advisor-1".into(), "advise", Some(&auth));
10007 assert_eq!(cooldown_until(&[loss], now), None);
10008 }
10009 }
10010
10011 #[test]
10012 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
10013 let old: Vec<QuotaLoss> = (1..=4)
10014 .map(|i| {
10015 loss(
10016 &format!("judge-{i}"),
10017 "2026-09-23T05:23:00Z",
10018 Some("2:40pm (Asia/Tokyo)"),
10019 )
10020 })
10021 .collect();
10022 let fresh = losses_this_attempt(&old, &old);
10023 assert!(fresh.is_empty());
10024 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
10025 }
10027
10028 #[test]
10029 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
10030 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
10031 let now = Timestamp::now();
10032 let mut after = old.clone();
10033 after.push(loss("judge-2", &now.to_string(), None));
10034 let fresh = losses_this_attempt(&old, &after);
10035 assert_eq!(fresh, vec![after[1].clone()]);
10036 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
10037 assert_eq!(
10038 until,
10039 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
10040 );
10041 }
10042
10043 #[test]
10044 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
10045 let before = vec![
10048 loss("judge-1", "2026-09-23T05:23:00Z", None),
10049 loss("judge-2", "2026-09-23T05:24:00Z", None),
10050 ];
10051 let after = vec![
10052 loss("judge-2", "2026-09-23T05:24:00Z", None),
10053 loss("judge-1", "2026-09-24T01:00:00Z", None),
10054 ];
10055 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
10056 }
10057
10058 #[test]
10059 fn merge_overrides_are_parsed_or_refused() {
10060 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
10061 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
10062 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
10063 assert!(merge_mode("squash").is_err());
10064 }
10065
10066 #[test]
10067 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
10068 let now = Timestamp::now();
10069 let fallback = Duration::from_secs(300);
10070 let cap = Duration::from_secs(1800);
10071
10072 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
10074
10075 let soon = now + jiff::SignedDuration::from_secs(600);
10077 assert_eq!(
10078 quota_wait(Some(soon), now, fallback, cap),
10079 Duration::from_secs(600)
10080 );
10081
10082 let past = now - jiff::SignedDuration::from_secs(60);
10085 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
10086
10087 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
10090 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
10091 }
10092
10093 #[test]
10094 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
10095 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10096
10097 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
10098 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
10099
10100 let already_past =
10104 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
10105 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
10106
10107 assert!(
10108 parse_reset_hint("session limit reached", now, now).is_none(),
10109 "free text with no recognised shape is not guessed at"
10110 );
10111 assert!(
10112 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
10113 "an unresolvable zone name is not guessed at either"
10114 );
10115 }
10116
10117 #[test]
10118 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
10119 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10120
10121 let at = parse_reset_hint(
10122 "You've hit your usage limit. Visit \
10123 https://chatgpt.com/codex/settings/usage to purchase more \
10124 credits or try again at Sep 19th, 2026 5:10 PM.",
10125 now,
10126 now,
10127 )
10128 .expect("the codex reset wording is a recognised shape");
10129 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
10130
10131 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
10136 .expect("an explicit year needs no rollover");
10137 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
10138
10139 assert!(
10140 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
10141 "a two-digit year is not the documented shape and is not guessed at"
10142 );
10143 assert!(
10144 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
10145 "a four-letter month name is not the documented three-letter abbreviation"
10146 );
10147 assert!(
10148 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
10149 "an explicit zone on the dated shape is a format nobody has \
10150 documented, and is refused rather than guessed at as UTC"
10151 );
10152 }
10153
10154 #[test]
10155 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
10156 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
10157 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
10158
10159 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
10160 assert_eq!(at.as_second() - recorded.as_second(), 3769);
10161
10162 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
10163 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
10164
10165 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
10166 assert!(
10167 parse_reset_hint(bad, now, recorded).is_none(),
10168 "{bad:?} must not be guessed at"
10169 );
10170 }
10171 }
10172
10173 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
10177 let config = dir.join("magi.toml");
10178 std::fs::write(
10179 &config,
10180 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
10181 )
10182 .unwrap();
10183 let opts = Opts {
10184 poll: Duration::from_secs(30),
10185 config: Some(config),
10186 repo: dir.join("repo"),
10190 ..Opts::default()
10191 };
10192 let home = dir.join("home");
10201 let stamps = home.join("service-ticks");
10204 std::fs::create_dir_all(&stamps).unwrap();
10205 for service in ["waiter", "deputies", "release-watch"] {
10206 std::fs::write(
10207 stamps.join(service),
10208 Timestamp::now().as_second().to_string(),
10209 )
10210 .unwrap();
10211 }
10212 for operation in [
10213 "conductor",
10214 "maintenance",
10215 "housekeeping",
10216 "disk-pressure",
10217 "cache",
10218 "fetch",
10219 ] {
10220 std::fs::write(
10221 stamps.join(format!("daemon-{operation}")),
10222 serde_json::to_vec(&Timestamp::now()).unwrap(),
10223 )
10224 .unwrap();
10225 }
10226 let worktrees = dir.join("wt");
10227 (
10228 opts,
10229 Queue::at(dir.join("queue")),
10230 home.join("daemon.json"),
10231 home,
10232 worktrees,
10233 )
10234 }
10235
10236 #[test]
10237 fn a_stop_is_idempotent_and_once_set_stays_set() {
10238 let stop = Stop::new();
10239 assert!(!stop.stopped());
10240
10241 stop.stop();
10242 assert!(stop.stopped());
10243 stop.stop();
10244 assert!(stop.stopped(), "a second stop is not a toggle");
10245
10246 let shared = stop.clone();
10247 assert!(
10248 shared.stopped(),
10249 "a clone is the same stop; that is how the loop and its caller share one"
10250 );
10251 }
10252
10253 #[tokio::test]
10254 async fn once_drain_rechecks_answers_after_queue_services_release_their_claims() {
10255 for operation in ["conductor", "maintenance", "housekeeping"] {
10256 let dir = tempfile::tempdir().unwrap();
10257 let (mut opts, queue, _, home, worktrees) = idle_loop(dir.path());
10258 opts.once = true;
10259 opts.poll = Duration::from_millis(1);
10260 let mut task = task();
10261 task.runs.push("fixture-run".into());
10262 queue.put(&mut task).unwrap();
10263 let claim = queue.claim(&task.id).unwrap();
10264
10265 let directory = home.join("workers/jobs/abcdef");
10268 std::fs::create_dir_all(&directory).unwrap();
10269 let lease = std::fs::File::create(directory.join("live.lock")).unwrap();
10270 crate::pr_head::try_lock_file(&lease).unwrap();
10271 let mut job = crate::worker::Job {
10272 id: "abcdef".into(),
10273 sequence: 1,
10274 kind: "daemon-service".into(),
10275 key: format!("service:daemon-{operation}"),
10276 payload: serde_json::Value::Null,
10277 generation: "fixture".into(),
10278 version: None,
10279 status: crate::worker::Status::Running,
10280 result: None,
10281 pid: Some(std::process::id()),
10282 created_at: Timestamp::now().as_second() as u64,
10283 finished_at: None,
10284 };
10285 let record = directory.join("job.json");
10286 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10287
10288 let status = Arc::new(Mutex::new(Status::new()));
10289 let stop = Stop::new();
10290 let mut drain = Box::pin(poll(
10291 &opts,
10292 &queue,
10293 &status,
10294 &home,
10295 &worktrees,
10296 &stop,
10297 DispatchLimits {
10298 max_concurrent: 1,
10299 pause_for_interrupts: false,
10300 recovery_cfg: None,
10301 },
10302 ));
10303 std::future::poll_fn(|cx| {
10304 use std::future::Future;
10305 assert!(
10306 drain.as_mut().poll(cx).is_pending(),
10307 "--once must wait for {operation} to release its queue claim"
10308 );
10309 std::task::Poll::Ready(())
10310 })
10311 .await;
10312 assert!(lock(&status).polls > 0, "the dispatch scan was exercised");
10313
10314 let questions = Questions::at(home.join("questions"));
10317 let mut q = action_question("fixture-run", ask::ChoiceAction::Done);
10318 questions.put(&mut q).unwrap();
10319 drop(claim);
10320 let expired =
10324 serde_json::to_vec(&"2000-01-01T00:00:00Z".parse::<Timestamp>().unwrap()).unwrap();
10325 for service in [operation, "housekeeping"] {
10326 std::fs::write(
10327 home.join("service-ticks").join(format!("daemon-{service}")),
10328 &expired,
10329 )
10330 .unwrap();
10331 }
10332 job.status = crate::worker::Status::Finished;
10333 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10334 drop(lease);
10335 tokio::time::timeout(Duration::from_secs(2), drain)
10336 .await
10337 .expect("the settled service must allow the drain to finish")
10338 .unwrap();
10339 let after = queue.get(&task.id).unwrap();
10340 assert_eq!(after.status, TaskStatus::Done, "answer after {operation}");
10341 assert!(after.action_applied(&q.id));
10342 for service in [operation, "housekeeping"] {
10343 assert_eq!(
10344 std::fs::read(home.join("service-ticks").join(format!("daemon-{service}")))
10345 .unwrap(),
10346 expired,
10347 "the final once rescan must not restart {service}"
10348 );
10349 }
10350 }
10351 }
10352
10353 #[test]
10354 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
10355 let stop = Stop::new();
10356 stop.busy.store(1, Ordering::SeqCst);
10357 assert!(
10358 !stop.finishing(),
10359 "a busy loop nobody has asked to stop is just running"
10360 );
10361
10362 stop.stop();
10363 assert!(
10364 stop.finishing(),
10365 "a stop asked for mid-run has not landed until the run is settled"
10366 );
10367
10368 stop.busy.store(0, Ordering::SeqCst);
10369 assert!(
10370 !stop.finishing(),
10371 "once the run is settled the stop has landed and there is nothing to finish"
10372 );
10373 }
10374
10375 #[test]
10376 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
10377 let stop = Stop::new();
10378 stop.busy.store(2, Ordering::SeqCst);
10379 stop.stop();
10380 assert!(stop.finishing(), "two runs still in flight");
10381
10382 stop.busy.store(1, Ordering::SeqCst);
10383 assert!(
10384 stop.finishing(),
10385 "one run finished, but a sibling is still working"
10386 );
10387
10388 stop.busy.store(0, Ordering::SeqCst);
10389 assert!(
10390 !stop.finishing(),
10391 "the last run out is what actually lands the stop"
10392 );
10393 }
10394
10395 #[tokio::test]
10396 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
10397 let dir = tempfile::tempdir().unwrap();
10398 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10399 let stop = Stop::new();
10400 stop.stop();
10401
10402 let began = std::time::Instant::now();
10403 tokio::time::timeout(
10404 Duration::from_secs(2),
10405 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10406 )
10407 .await
10408 .expect("a stopped loop must return, not sit out its poll interval")
10409 .expect("the loop's own setup and teardown must not fail");
10410 assert!(
10411 began.elapsed() < opts.poll,
10412 "returned only after {:?}, which is a poll interval, not a stop",
10413 began.elapsed()
10414 );
10415 }
10416
10417 #[tokio::test]
10418 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
10419 let dir = tempfile::tempdir().unwrap();
10420 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10421 let stop = Stop::new();
10422
10423 let asker = {
10426 let stop = stop.clone();
10427 tokio::spawn(async move {
10428 tokio::time::sleep(Duration::from_millis(20)).await;
10429 stop.stop();
10430 })
10431 };
10432
10433 let began = std::time::Instant::now();
10434 tokio::time::timeout(
10435 Duration::from_secs(2),
10436 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10437 )
10438 .await
10439 .expect("a stop asked for while idle must wake the wait")
10440 .expect("the loop's own setup and teardown must not fail");
10441 asker.await.unwrap();
10442 assert!(
10443 began.elapsed() < opts.poll,
10444 "returned only after {:?}, so the stop waited on the sleep",
10445 began.elapsed()
10446 );
10447 }
10448
10449 #[tokio::test]
10450 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
10451 let dir = tempfile::tempdir().unwrap();
10452 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10453 let stop = Stop::new();
10454 stop.stop();
10455
10456 tokio::time::timeout(
10457 Duration::from_secs(2),
10458 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10459 )
10460 .await
10461 .expect("a stopped loop must return")
10462 .expect("the loop's own setup and teardown must not fail");
10463
10464 assert!(
10465 home.is_dir(),
10466 "the loop did publish a status file, so its removal is the teardown and not an absence"
10467 );
10468 assert!(
10469 !status_file.exists(),
10470 "a stopped loop clears its status file"
10471 );
10472 assert!(
10473 read_status(&home).is_none(),
10474 "a reader must see no daemon at all, not a heartbeat that merely stopped"
10475 );
10476 }
10477
10478 #[tokio::test]
10479 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
10480 let dir = tempfile::tempdir().unwrap();
10481 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10482
10483 let mut settled = RunState::new(
10484 dir.path().join("repo"),
10485 "main".to_owned(),
10486 "abc1234".to_owned(),
10487 "fixture".to_owned(),
10488 Config::default(),
10489 );
10490 settled.status = RunStatus::Ready;
10491 let run_dir = home.join("runs").join(&settled.id);
10492 std::fs::create_dir_all(&run_dir).unwrap();
10493 std::fs::write(
10494 run_dir.join("run.json"),
10495 serde_json::to_string_pretty(&settled).unwrap(),
10496 )
10497 .unwrap();
10498 let questions = Questions::at(home.join("questions"));
10499 let mut question = ask::Question::new(
10500 settled.id.clone(),
10501 "review".to_owned(),
10502 "reviewer-1".to_owned(),
10503 "Continue?".to_owned(),
10504 String::new(),
10505 Vec::new(),
10506 );
10507 questions.put(&mut question).unwrap();
10508
10509 run_service_worker(
10510 &home,
10511 serde_json::to_value(ServiceWorker {
10512 opts,
10513 queue: queue.root().to_path_buf(),
10514 worktrees_root: worktrees,
10515 operation: "housekeeping".to_owned(),
10516 })
10517 .unwrap(),
10518 )
10519 .await
10520 .unwrap();
10521
10522 assert_eq!(
10523 questions.get(&question.id).unwrap().status,
10524 ask::QuestionStatus::Abandoned,
10525 "the independent housekeeping worker settles ended-run questions"
10526 );
10527 }
10528
10529 #[test]
10530 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10531 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10532
10533 assert!(
10534 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10535 "never checked before: due at once"
10536 );
10537
10538 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10539 assert!(
10540 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10541 "well inside the interval: not due yet"
10542 );
10543
10544 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10545 assert!(
10546 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10547 "exactly at the edge: not yet due, same convention as `clean::due`"
10548 );
10549
10550 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10551 assert!(
10552 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10553 "past the interval: due again"
10554 );
10555 }
10556
10557 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10562 let config = dir.join("magi.toml");
10563 std::fs::write(
10569 &config,
10570 format!(
10571 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10572 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10573 cache_dir.display()
10574 ),
10575 )
10576 .unwrap();
10577 Opts {
10578 config: Some(config),
10579 repo: dir.join("repo"),
10580 ..Opts::default()
10581 }
10582 }
10583
10584 #[tokio::test]
10585 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10586 let dir = tempfile::tempdir().unwrap();
10587 let home = dir.path().join("home");
10588 let cache_dir = dir.path().join("cache");
10589 std::fs::create_dir_all(&cache_dir).unwrap();
10590 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10591 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10592
10593 let running = Stop::new();
10596 let mut last_checked = None;
10597 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10598 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10599 .await;
10600 assert_eq!(
10601 crate::disk::dir_size(&cache_dir),
10602 0,
10603 "over the cap on the first check ever: pruned at once, no idle queue required"
10604 );
10605 assert_eq!(last_checked, Some(t0));
10606
10607 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10609 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10610 maybe_prune_cache_between_runs(
10611 &opts.repo,
10612 &opts,
10613 &home,
10614 &running,
10615 &mut last_checked,
10616 too_soon,
10617 )
10618 .await;
10619 assert_eq!(
10620 crate::disk::dir_size(&cache_dir),
10621 10,
10622 "too soon since the last check: left alone rather than rescanned every call"
10623 );
10624 assert_eq!(
10625 last_checked,
10626 Some(t0),
10627 "an idle check does not reset the clock"
10628 );
10629
10630 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10632 maybe_prune_cache_between_runs(
10633 &opts.repo,
10634 &opts,
10635 &home,
10636 &running,
10637 &mut last_checked,
10638 due_again,
10639 )
10640 .await;
10641 assert_eq!(
10642 crate::disk::dir_size(&cache_dir),
10643 0,
10644 "due again: pruned back under the cap"
10645 );
10646 }
10647
10648 #[tokio::test]
10656 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
10657 let dir = tempfile::tempdir().unwrap();
10658 let home = dir.path().join("home");
10659 let cache_dir = dir.path().join("cache");
10660 std::fs::create_dir_all(&cache_dir).unwrap();
10661 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10662 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10663
10664 let stop = Stop::new();
10665 stop.stop();
10666 assert!(
10667 !stop.finishing(),
10668 "no run is in flight at a between-runs boundary, so nothing else \
10669 would tell the operator this stop had not taken effect yet"
10670 );
10671
10672 let mut last_checked = None;
10673 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10674 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10675 .await;
10676 assert_eq!(
10677 crate::disk::dir_size(&cache_dir),
10678 10,
10679 "over its cap, and due for the first check ever, but a stop outranks \
10680 it: the cap is a standing policy the next start measures again"
10681 );
10682 assert_eq!(
10683 last_checked, None,
10684 "a check that never happened must not claim the interval"
10685 );
10686 }
10687
10688 #[tokio::test]
10691 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10692 let dir = tempfile::tempdir().unwrap();
10693 let cache_dir = dir.path().join("cache");
10694 std::fs::create_dir_all(&cache_dir).unwrap();
10695 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10696
10697 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10698 opts.poll = Duration::from_millis(20);
10699 opts.max_attempts = 1_000;
10700
10701 let queue = Queue::at(dir.path().join("queue"));
10702 for i in 0..128 {
10703 let mut task = Task::new(
10704 format!("queued work {i}"),
10705 format!("queued work {i}"),
10706 opts.repo.clone(),
10707 Source::Human,
10708 );
10709 queue.put(&mut task).unwrap();
10710 }
10711
10712 let home = dir.path().join("home");
10713 let worktrees = dir.path().join("wt");
10714 run_service_worker(
10715 &home,
10716 serde_json::to_value(ServiceWorker {
10717 opts,
10718 queue: queue.root().to_path_buf(),
10719 worktrees_root: worktrees,
10720 operation: "cache".to_owned(),
10721 })
10722 .unwrap(),
10723 )
10724 .await
10725 .unwrap();
10726 let after = queue.list();
10727 assert_eq!(after.len(), 128);
10728 assert!(
10729 after
10730 .iter()
10731 .all(|task| task.status.runnable() && task.attempts == 0)
10732 );
10733
10734 assert_eq!(
10735 crate::disk::dir_size(&cache_dir),
10736 0,
10737 "an oversized cache must not be left to grow unboundedly just because the \
10738 queue kept the loop busy the whole time"
10739 );
10740 }
10741
10742 #[test]
10743 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10744 let dir = tempfile::tempdir().unwrap();
10745 let queue = Queue::at(dir.path().join("queue"));
10746 let questions = Questions::at(dir.path().join("questions"));
10747 let mut task = task();
10748 queue.put(&mut task).unwrap();
10749
10750 let mut task_question = ask::Question::new(
10751 task.id.clone(),
10752 crate::conduct::NODE.to_owned(),
10753 "conduct".to_owned(),
10754 "Which backend?".to_owned(),
10755 String::new(),
10756 Vec::new(),
10757 );
10758 questions.put(&mut task_question).unwrap();
10759 task.block(vec![task_question.id.clone()], None);
10760 queue.put(&mut task).unwrap();
10761
10762 let mut run_question = ask::Question::new(
10763 "20260101-000000-run1".to_owned(),
10764 "review".to_owned(),
10765 "reviewer-1".to_owned(),
10766 "Run question".to_owned(),
10767 String::new(),
10768 Vec::new(),
10769 );
10770 questions.put(&mut run_question).unwrap();
10771
10772 let mut coincidental = ask::Question::new(
10777 task.id.clone(),
10778 "review".to_owned(),
10779 "reviewer-1".to_owned(),
10780 "Unrelated review question".to_owned(),
10781 String::new(),
10782 Vec::new(),
10783 );
10784 questions.put(&mut coincidental).unwrap();
10785
10786 reconcile_task_questions(&queue, &questions);
10787 assert!(questions.get(&task_question.id).unwrap().status.open());
10788 assert!(questions.get(&run_question.id).unwrap().status.open());
10789 assert!(questions.get(&coincidental.id).unwrap().status.open());
10790
10791 task.release();
10792 queue.put(&mut task).unwrap();
10793 reconcile_task_questions(&queue, &questions);
10794 assert_eq!(
10795 questions.get(&task_question.id).unwrap().status,
10796 ask::QuestionStatus::Abandoned
10797 );
10798 assert!(
10799 questions.get(&run_question.id).unwrap().status.open(),
10800 "run questions remain the run janitor's responsibility"
10801 );
10802 assert!(
10803 questions.get(&coincidental.id).unwrap().status.open(),
10804 "a non-conductor question must not be abandoned just because its \
10805 run id coincides with a task id"
10806 );
10807 }
10808
10809 #[test]
10810 fn a_freshly_started_running_task_is_never_stalled() {
10811 let dir = tempfile::tempdir().unwrap();
10812 let mut t = task();
10813 t.start("run-1".to_owned());
10814 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10817 }
10818
10819 #[test]
10820 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10821 let dir = tempfile::tempdir().unwrap();
10822 let mut t = task();
10823 t.start("run-1".to_owned());
10824 t.updated_at = Timestamp::now()
10825 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10826 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10827 assert_eq!(
10828 stalled_tasks(
10829 &Queue::at(dir.path().join("q")),
10830 dir.path(),
10831 Timestamp::now()
10832 )
10833 .len(),
10834 0,
10835 "the task was never written to this queue"
10836 );
10837 }
10838
10839 #[test]
10840 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10841 let dir = tempfile::tempdir().unwrap();
10842 let mut t = task();
10843 t.id = "20260903-080340-0167".to_owned();
10844 t.start("20260903-080619-01c2".to_owned());
10845 t.updated_at = Timestamp::now()
10846 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10847
10848 let mut status = Status::new();
10849 status.current = vec![Current {
10850 worker: None,
10851 task: t.id.clone(),
10852 run: "20260903-080619-01c2".to_owned(),
10853 }];
10854 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10855
10856 assert!(
10857 !is_stalled(&t, dir.path(), Timestamp::now()),
10858 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10859 );
10860 }
10861
10862 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10866 let path = queue.path_of(id);
10867 let body = std::fs::read_to_string(&path).unwrap();
10868 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10869 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10870 v["updated_at"] = serde_json::Value::String(old.to_string());
10871 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10872 }
10873
10874 #[test]
10875 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10876 let dir = tempfile::tempdir().unwrap();
10889 let queue = Queue::at(dir.path().join("queue"));
10890 let home = dir.path().join("home");
10891
10892 let mut t = task();
10893 t.id = "20260101-000001-lock".to_owned();
10894 t.start("run-1".to_owned());
10895 queue.put(&mut t).unwrap();
10896 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10897 std::fs::write(
10898 dir.path().join("queue").join(format!("{}.lock", t.id)),
10899 "not a pid",
10900 )
10901 .unwrap();
10902
10903 let now = Timestamp::now();
10904 assert!(
10905 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10906 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10907 and reclaim must leave the task alone"
10908 );
10909 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10910
10911 let stalled = stalled_tasks(&queue, &home, now);
10912 assert_eq!(
10913 stalled.len(),
10914 1,
10915 "reclaim's inability to claim it yet must not hide it from the conductor"
10916 );
10917 assert_eq!(stalled[0].id, t.id);
10918 }
10919
10920 #[test]
10921 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10922 let dir = tempfile::tempdir().unwrap();
10923 crate::run::set_home(dir.path().join("run-home"));
10924 let queue = Queue::at(dir.path().join("queue"));
10925 let home = dir.path().join("home");
10926 let questions = Questions::at(dir.path().join("questions"));
10927
10928 let mut t = task();
10929 t.id = "20260101-000003-dead".to_owned();
10930 t.start("missing-run".to_owned());
10931 queue.put(&mut t).unwrap();
10932 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10933
10934 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10937 assert_eq!(
10938 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
10939 [&t.id]
10940 );
10941 assert_eq!(
10942 reclaim_orphaned_running(&queue, 2, &Default::default()),
10943 [t.id.clone()]
10944 );
10945 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
10946
10947 crate::conduct::apply(
10950 &queue,
10951 &questions,
10952 &crate::conduct::Verdict {
10953 decisions: vec![crate::conduct::Decision {
10954 id: t.id.clone(),
10955 recovery: Some(crate::conduct::Recovery::Requeue),
10956 ..crate::conduct::Decision::default()
10957 }],
10958 },
10959 )
10960 .unwrap();
10961 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
10962 }
10963
10964 #[test]
10965 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
10966 let dir = tempfile::tempdir().unwrap();
10967 let queue = Queue::at(dir.path().join("queue"));
10968 let home = dir.path().join("home");
10969
10970 let mut fresh = task();
10971 fresh.id = "20260101-000001-aaaa".to_owned();
10972 fresh.start("run-1".to_owned());
10973 queue.put(&mut fresh).unwrap();
10974
10975 let mut old = task();
10976 old.id = "20260101-000002-bbbb".to_owned();
10977 old.start("run-2".to_owned());
10978 queue.put(&mut old).unwrap();
10979 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
10980
10981 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
10982 assert_eq!(stalled.len(), 1);
10983 assert_eq!(stalled[0].id, old.id);
10984 }
10985
10986 #[test]
10987 fn queued_and_finished_task_views_partition_by_status() {
10988 let dir = tempfile::tempdir().unwrap();
10989 let queue = Queue::at(dir.path().join("queue"));
10990
10991 let mut queued = task();
10992 queued.id = "20260101-000001-aaaa".to_owned();
10993 queue.put(&mut queued).unwrap();
10994
10995 let mut failed = task();
10996 failed.id = "20260101-000002-bbbb".to_owned();
10997 failed.start("run-1".to_owned());
10998 failed.fail("gate red", 5);
10999 queue.put(&mut failed).unwrap();
11000
11001 let mut held = task();
11002 held.id = "20260101-000003-cccc".to_owned();
11003 held.hold_machine(None);
11004 queue.put(&mut held).unwrap();
11005
11006 let mut running = task();
11007 running.id = "20260101-000004-dddd".to_owned();
11008 running.start("run-2".to_owned());
11009 queue.put(&mut running).unwrap();
11010
11011 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
11012 assert_eq!(queued_ids, [queued.id.clone()]);
11013
11014 let mut finished_ids: Vec<String> =
11015 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
11016 finished_ids.sort_unstable();
11017 let mut want = vec![failed.id.clone(), held.id.clone()];
11018 want.sort_unstable();
11019 assert_eq!(finished_ids, want);
11020 }
11021
11022 #[test]
11023 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
11024 for short in [false, true] {
11025 let dir = tempfile::tempdir().unwrap();
11026 let queue = Queue::at(dir.path().join("queue"));
11027 let questions = ask::Questions::at(dir.path().join("questions"));
11028 let mut running = task();
11029 running.id = "20261013-192213-feb2".into();
11030 running.start("existing-run".into());
11031 queue.put(&mut running).unwrap();
11032 let mut middle = task();
11033 middle.id = "20261013-193611-ca5b".into();
11034 middle.block(
11035 vec![if short {
11036 "feb2".into()
11037 } else {
11038 running.id.clone()
11039 }],
11040 None,
11041 );
11042 queue.put(&mut middle).unwrap();
11043 let mut waiting = task();
11044 waiting.id = "20261013-202849-ee37".into();
11045 waiting.block(
11046 vec![if short {
11047 "ca5b".into()
11048 } else {
11049 middle.id.clone()
11050 }],
11051 Some("wait for implementation".into()),
11052 );
11053 queue.put(&mut waiting).unwrap();
11054 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
11055 let inv = crate::blockers::Inventory::load(&queue, &questions);
11056 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
11057 assert!(inv.stuck().is_empty());
11058 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11059 assert!(report.quarantined.is_empty());
11060 assert!(report.asked.is_empty());
11061 resolve_blockers(&queue, &questions);
11062 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11063 assert!(questions.list().is_empty());
11064 running.succeed();
11067 queue.put(&mut running).unwrap();
11068 resolve_blockers(&queue, &questions);
11069 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
11070 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11071 middle = queue.get(&middle.id).unwrap();
11072 middle.succeed();
11073 queue.put(&mut middle).unwrap();
11074 resolve_blockers(&queue, &questions);
11075 let after = queue.get(&waiting.id).unwrap();
11076 assert_eq!(after.status, TaskStatus::Queued);
11077 assert!(after.blocked_by.is_empty());
11078 assert_eq!(after.instruction, waiting.instruction);
11079 }
11080 }
11081
11082 #[test]
11083 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
11084 for future_schema in [false, true] {
11085 let name = "20261013-193611-ca5b";
11086 let mut future = task();
11087 future.id = name.into();
11088 future.schema = crate::queue::SCHEMA + 1;
11089 let contents = if future_schema {
11090 serde_json::to_string(&future).unwrap()
11091 } else {
11092 "corrupt".into()
11093 };
11094 let dir = tempfile::tempdir().unwrap();
11095 let queue = Queue::at(dir.path().join("queue"));
11096 let questions = ask::Questions::at(dir.path().join("questions"));
11097 let mut waiting = task();
11098 waiting.id = "20261013-202849-ee37".into();
11099 waiting.block(vec!["ca5b".into()], None);
11100 queue.put(&mut waiting).unwrap();
11101 std::fs::write(queue.path_of(name), contents).unwrap();
11102 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
11103 assert!(
11104 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
11105 );
11106 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11107 assert!(report.asked.is_empty());
11108 resolve_blockers(&queue, &questions);
11109 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11110 assert!(questions.list().is_empty());
11111 }
11112 let dir = tempfile::tempdir().unwrap();
11113 let queue = Queue::at(dir.path().join("queue"));
11114 let questions = ask::Questions::at(dir.path().join("questions"));
11115 let mut waiting = task();
11116 waiting.id = "20261013-202849-ee37".into();
11117 waiting.block(vec!["ca5b".into()], None);
11118 queue.put(&mut waiting).unwrap();
11119 let mut done = task();
11120 done.id = "20261013-193611-ca5b".into();
11121 done.succeed();
11122 queue.put(&mut done).unwrap();
11123 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
11126 assert!(queue.get("ca5b").is_err());
11127 assert_eq!(
11128 crate::queue::dependency(&queue, &questions, "ca5b"),
11129 crate::queue::Dependency::Ambiguous
11130 );
11131 resolve_blockers(&queue, &questions);
11132 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11133 assert!(report.asked.is_empty());
11134 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11135 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
11136 }
11137
11138 #[test]
11139 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
11140 for held in [false, true] {
11141 for question in [false, true] {
11142 for reference in ["ca5b", "20261013-193611-ca5b"] {
11143 let dir = tempfile::tempdir().unwrap();
11144 let queue = Queue::at(dir.path().join("queue"));
11145 let questions = ask::Questions::at(dir.path().join("questions"));
11146 let mut prerequisite = task();
11147 prerequisite.id = "20261013-193611-ca5b".into();
11148 prerequisite.start("existing-run".into());
11149 queue.put(&mut prerequisite).unwrap();
11150 let mut waiting = task();
11151 waiting.id = "20261013-202849-ee37".into();
11152 if held {
11153 waiting.hold_machine(Some("waiting for separate consent".into()));
11154 }
11155 queue.put(&mut waiting).unwrap();
11156 let reason = "Deleting saved data requires separate owner consent";
11157 crate::conduct::apply(
11158 &queue,
11159 &questions,
11160 &crate::conduct::Verdict {
11161 decisions: vec![crate::conduct::Decision {
11162 id: waiting.id.clone(),
11163 blocked_by: vec![reference.into()],
11164 question: question.then(|| "May the saved data be deleted?".into()),
11165 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
11166 reason: Some(reason.into()),
11167 ..crate::conduct::Decision::default()
11168 }],
11169 },
11170 )
11171 .unwrap();
11172 let before = queue.get(&waiting.id).unwrap();
11173 if question {
11174 assert_eq!(before.status, TaskStatus::Blocked);
11175 let q = questions.get(&before.blocked_by[0]).unwrap();
11176 assert_eq!(q.summary, "May the saved data be deleted?");
11177 assert!(q.status.open());
11178 } else {
11179 assert_eq!(before.status, TaskStatus::Held);
11180 let recorded_reason = before.hold_reason.as_deref().unwrap();
11181 assert!(recorded_reason.starts_with(reason));
11182 if held {
11183 assert!(recorded_reason.contains("waiting for separate consent"));
11184 }
11185 }
11186 prerequisite.succeed();
11187 queue.put(&mut prerequisite).unwrap();
11188 resolve_blockers(&queue, &questions);
11189 let after = queue.get(&waiting.id).unwrap();
11190 assert_eq!(after.status, before.status);
11191 assert_eq!(after.blocked_by, before.blocked_by);
11192 assert_eq!(after.hold_reason, before.hold_reason);
11193 assert_eq!(after.instruction, waiting.instruction);
11194 if question {
11195 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
11196 }
11197 assert!(
11198 queue.next_runnable().is_none(),
11199 "dependency completion is not consent"
11200 );
11201 }
11202 }
11203 }
11204 }
11205
11206 #[test]
11207 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
11208 let dir = tempfile::tempdir().unwrap();
11209 let queue = Queue::at(dir.path().join("queue"));
11210 let questions = ask::Questions::at(dir.path().join("questions"));
11211
11212 let mut dep = task();
11213 dep.id = "20260101-000001-dep0".to_owned();
11214 dep.succeed();
11215 queue.put(&mut dep).unwrap();
11216
11217 let mut still_going = task();
11218 still_going.id = "20260101-000002-dep1".to_owned();
11219 queue.put(&mut still_going).unwrap();
11220
11221 let mut blocked = task();
11222 blocked.id = "20260101-000003-main".to_owned();
11223 blocked.block(
11224 vec![dep.id.clone(), still_going.id.clone()],
11225 Some("waits on both".to_owned()),
11226 );
11227 queue.put(&mut blocked).unwrap();
11228
11229 resolve_blockers(&queue, &questions);
11230
11231 let after = queue.get(&blocked.id).unwrap();
11232 assert_eq!(
11233 after.status,
11234 TaskStatus::Blocked,
11235 "one dependency is still outstanding"
11236 );
11237 assert_eq!(after.blocked_by, [still_going.id.clone()]);
11238 }
11239
11240 #[test]
11241 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
11242 let dir = tempfile::tempdir().unwrap();
11243 let queue = Queue::at(dir.path().join("queue"));
11244 let questions = ask::Questions::at(dir.path().join("questions"));
11245
11246 let mut q = crate::ask::Question::new(
11247 "20260101-000001-main".to_owned(),
11248 crate::conduct::NODE.to_owned(),
11249 "conduct".to_owned(),
11250 "Which backend?".to_owned(),
11251 String::new(),
11252 Vec::new(),
11253 );
11254 questions.put(&mut q).unwrap();
11255 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
11256 .unwrap();
11257 questions.put(&mut q).unwrap();
11258
11259 let mut blocked = task();
11260 blocked.id = "20260101-000001-main".to_owned();
11261 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
11262 queue.put(&mut blocked).unwrap();
11263
11264 resolve_blockers(&queue, &questions);
11265
11266 let after = queue.get(&blocked.id).unwrap();
11267 assert_eq!(
11268 after.status,
11269 TaskStatus::Queued,
11270 "the only blocker resolved"
11271 );
11272 assert_eq!(after.answers.len(), 1);
11273 assert_eq!(after.answers[0].question, "Which backend?");
11274 assert_eq!(after.answers[0].answer, "SQLite");
11275
11276 let instruction = instruction_for(&after);
11278 assert!(instruction.contains("Which backend?"));
11279 assert!(instruction.contains("SQLite"));
11280 }
11281
11282 #[test]
11283 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
11284 let dir = tempfile::tempdir().unwrap();
11285 let queue = Queue::at(dir.path().join("queue"));
11286 let questions = ask::Questions::at(dir.path().join("questions"));
11287
11288 let mut q = crate::ask::Question::new(
11289 "20260101-000001-main".to_owned(),
11290 crate::conduct::NODE.to_owned(),
11291 "conduct".to_owned(),
11292 "Is the setup done?".to_owned(),
11293 String::new(),
11294 Vec::new(),
11295 );
11296 q.abandon("no answer within 60s of asking");
11297 questions.put(&mut q).unwrap();
11298
11299 let mut blocked = task();
11300 blocked.id = "20260101-000001-main".to_owned();
11301 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
11302 queue.put(&mut blocked).unwrap();
11303
11304 resolve_blockers(&queue, &questions);
11305
11306 let after = queue.get(&blocked.id).unwrap();
11307 assert_eq!(
11308 after.status,
11309 TaskStatus::Held,
11310 "never left blocked on nothing"
11311 );
11312 assert!(!after.operator_held(), "a machine hold, for triage");
11313 assert!(
11314 after
11315 .hold_reason
11316 .as_deref()
11317 .unwrap_or_default()
11318 .contains("went unanswered")
11319 );
11320 }
11321
11322 #[test]
11323 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
11324 let dir = tempfile::tempdir().unwrap();
11330 let queue = Queue::at(dir.path().join("queue"));
11331 let questions = ask::Questions::at(dir.path().join("questions"));
11332
11333 let mut q = crate::ask::Question::new(
11334 "20260101-000001-main".to_owned(),
11335 crate::conduct::NODE.to_owned(),
11336 "conduct".to_owned(),
11337 "How should this be handled?".to_owned(),
11338 String::new(),
11339 Vec::new(),
11340 );
11341 questions.put(&mut q).unwrap();
11342 q.answer(crate::ask::Answer::Text(
11343 "leave it held, a human will look at it later".to_owned(),
11344 ))
11345 .unwrap();
11346 questions.put(&mut q).unwrap();
11347
11348 let mut held = task();
11349 held.id = "20260101-000001-main".to_owned();
11350 held.hold_machine(Some("out of attempts".to_owned()));
11351 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
11352 queue.put(&mut held).unwrap();
11353
11354 resolve_blockers(&queue, &questions);
11355
11356 let after = queue.get(&held.id).unwrap();
11357 assert_eq!(after.status, TaskStatus::Held);
11358 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
11359 assert_eq!(
11360 after.answers[0].answer,
11361 "leave it held, a human will look at it later"
11362 );
11363 }
11364
11365 #[test]
11366 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
11367 let dir = tempfile::tempdir().unwrap();
11368 let queue = Queue::at(dir.path().join("queue"));
11369 let questions = ask::Questions::at(dir.path().join("questions"));
11370 let mut dep = task();
11371 dep.id = "20260101-000001-gone".to_owned();
11372 queue.put(&mut dep).unwrap();
11373 let mut blocked = task();
11374 blocked.id = "20260101-000003-main".to_owned();
11375 blocked.block(vec![dep.id.clone()], None);
11376 queue.put(&mut blocked).unwrap();
11377
11378 let claim = queue.claim(&blocked.id).unwrap();
11380 queue.remove(&dep.id, false, &questions).unwrap();
11381 drop(claim);
11382 resolve_blockers(&queue, &questions);
11383
11384 let after = queue.get(&blocked.id).unwrap();
11385 assert_eq!(after.status, TaskStatus::Queued);
11386 assert!(after.blocked_by.is_empty());
11387 }
11388
11389 #[test]
11390 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
11391 let dir = tempfile::tempdir().unwrap();
11397 let queue = Queue::at(dir.path().join("queue"));
11398 let questions = ask::Questions::at(dir.path().join("questions"));
11399
11400 let mut still_going = task();
11401 still_going.id = "20260101-000002-dep1".to_owned();
11402 queue.put(&mut still_going).unwrap();
11403
11404 let mut blocked = task();
11405 blocked.id = "20260101-000003-main".to_owned();
11406 blocked.block(
11407 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
11408 Some("waits on both".to_owned()),
11409 );
11410 queue.put(&mut blocked).unwrap();
11411
11412 resolve_blockers(&queue, &questions);
11413
11414 let after = queue.get(&blocked.id).unwrap();
11415 assert_eq!(
11416 after.status,
11417 TaskStatus::Held,
11418 "a missing dependency must not leave the task blocked forever"
11419 );
11420 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
11421 assert!(after.blocked_by.is_empty());
11422 let reason = after.hold_reason.as_deref().unwrap_or_default();
11423 assert!(
11424 reason.contains("20260101-000001-gone"),
11425 "the missing id must be named so an operator can tell what happened: {reason}"
11426 );
11427 assert!(
11428 reason.contains(&still_going.id),
11429 "the still-valid dependency must not silently vanish from the record: {reason}"
11430 );
11431 }
11432
11433 #[test]
11434 fn instruction_for_is_unchanged_without_any_answers() {
11435 let t = task();
11436 assert_eq!(instruction_for(&t), t.instruction);
11437 }
11438
11439 #[test]
11440 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
11441 let dir = tempfile::tempdir().unwrap();
11442 let q = Queue::at(dir.path().join("queue"));
11443 let src = dir.path().join("shot.png");
11444 std::fs::write(&src, "x").unwrap();
11445 let mut t = task();
11446 q.attach(&mut t, &[src]).unwrap();
11447 let paths = task_attachments(&q, &t).unwrap();
11448 assert_eq!(paths.len(), 1);
11449 assert!(paths[0].is_absolute() && paths[0].is_file());
11450 std::fs::remove_file(&paths[0]).unwrap();
11451 let err = task_attachments(&q, &t).unwrap_err().to_string();
11452 assert!(err.contains("shot.png"), "{err}");
11453 }
11454
11455 #[tokio::test]
11456 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
11457 let dir = tempfile::tempdir().unwrap();
11458 let queue_path = dir.path().join("queue");
11459 let queue = Queue::at(queue_path.clone());
11460 let mut t = task();
11461 t.source = Source::Agent {
11462 run: "originating-chat".into(),
11463 node: crate::queue::CHAT_NODE.into(),
11464 };
11465 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
11466 stage: "config-fetch",
11467 target: "origin/main".into(),
11468 });
11469 record_config_failure(&mut t, &fetch);
11470 t.hold_machine(Some(format!("{fetch:#}")));
11471 let fetch_key = t.chat_notice_due().unwrap();
11472 t.chat_notices.push(fetch_key.clone());
11473 queue.put(&mut t).unwrap();
11474
11475 let queue = Queue::at(queue_path);
11476 let mut t = queue.get(&t.id).unwrap();
11477 let a = dir.path().join("a.toml");
11478 let b = dir.path().join("b.toml");
11479 for (path, text, new_cause) in [
11480 (&a, "[graph", true),
11481 (&a, "[verify", false),
11482 (&a, "[graph]\nimplementers = 'invalid-type'", true),
11483 (&b, "[graph]\nimplementers = 'invalid-type'", true),
11484 ] {
11485 t.release_operator("test", None);
11486 std::fs::write(path, text).unwrap();
11487 let error = Config::discover_fetched(dir.path(), Some(path))
11488 .await
11489 .unwrap_err();
11490 record_config_failure(&mut t, &error);
11491 t.hold_machine(Some(format!("{error:#}")));
11492 let due = t.chat_notice_due();
11493 assert_eq!(due.is_some(), new_cause, "{error:#}");
11494 if let Some(key) = due {
11495 assert_ne!(key, fetch_key);
11496 t.chat_notices.push(key);
11497 }
11498 }
11499 t.succeed();
11500 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11501 }
11502
11503 #[test]
11504 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11505 let dir = tempfile::tempdir().unwrap();
11506 let queue_path = dir.path().join("queue");
11507 let queue = Queue::at(queue_path.clone());
11508 let mut t = task();
11509 t.source = crate::queue::Source::Agent {
11510 run: "originating-chat".into(),
11511 node: crate::queue::CHAT_NODE.into(),
11512 };
11513 let a = dir.path().join("a.png");
11514 let b = dir.path().join("b.png");
11515 std::fs::write(&a, "a").unwrap();
11516 std::fs::write(&b, "b").unwrap();
11517 queue.attach(&mut t, &[a, b]).unwrap();
11518 let paths = task_attachments(&queue, &t).unwrap();
11519 for path in &paths {
11520 std::fs::remove_file(path).unwrap();
11521 }
11522 let missing = task_attachments(&queue, &t).unwrap_err();
11523 t.attempts += 1;
11524 let reason = missing.to_string();
11525 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11526 let first = t.chat_notice_due().expect("first attachment notice");
11527 t.chat_notices.push(first.clone());
11528 queue.put(&mut t).unwrap();
11529
11530 let queue = Queue::at(queue_path.clone());
11531 let mut t = queue.get(&t.id).unwrap();
11532 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11533 t.release_operator("test", None);
11534 t.attempts += 1;
11535 let missing = task_attachments(&queue, &t).unwrap_err();
11536 t.fail_startup_target(
11537 "startup-attachments",
11538 missing.name,
11539 "same missing attachment with regenerated wording",
11540 1,
11541 );
11542 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11543
11544 std::fs::write(&paths[0], "restored").unwrap();
11545 t.release_operator("test", None);
11546 t.attempts += 1;
11547 let missing = task_attachments(&queue, &t).unwrap_err();
11548 let reason = missing.to_string();
11549 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11550 let second = t
11551 .chat_notice_due()
11552 .expect("different attachment must notify");
11553 assert_ne!(first, second);
11554 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11555 }
11556
11557 #[test]
11558 fn resumed_instruction_is_unchanged_without_any_answers() {
11559 let t = task();
11560 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11561 }
11562
11563 #[test]
11564 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11565 let mut t = task();
11566 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11567 let old = t.instruction.clone();
11571
11572 let refreshed = resumed_instruction(&old, &t);
11573 assert!(refreshed.starts_with(&old), "the original text is kept");
11574 assert!(refreshed.contains("Which backend?"));
11575 assert!(refreshed.contains("SQLite"));
11576 }
11577
11578 #[test]
11579 fn resumed_instruction_keeps_an_original_answers_heading() {
11580 let mut t = task();
11581 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11582 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11583
11584 let refreshed = resumed_instruction(&t.instruction, &t);
11585
11586 assert!(
11587 refreshed.starts_with(&t.instruction),
11588 "an answers heading in the original instruction is not the appended block"
11589 );
11590 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11591 assert!(refreshed.contains("Which backend?"));
11592 assert!(refreshed.contains("SQLite"));
11593
11594 let repeated = resumed_instruction(&refreshed, &t);
11595 assert_eq!(
11596 repeated, refreshed,
11597 "only the final appended block is refreshed"
11598 );
11599 }
11600
11601 #[test]
11602 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11603 let mut t = task();
11604 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11605
11606 let once = resumed_instruction(&t.instruction, &t);
11610 let twice = resumed_instruction(&once, &t);
11611 assert_eq!(once, twice);
11612 assert_eq!(once.matches("Which backend?").count(), 1);
11613
11614 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
11616 let refreshed = resumed_instruction(&once, &t);
11617 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
11618 assert!(refreshed.contains("Which backend?"));
11619 assert!(refreshed.contains("Which cache?"));
11620 }
11621
11622 #[test]
11623 fn prepare_instruction_covers_all_three_starters() {
11624 let mut t = task();
11625 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11626
11627 assert_eq!(
11630 prepare_instruction(&Starter::Start, None, &t),
11631 Some(instruction_for(&t))
11632 );
11633
11634 let old = t.instruction.clone();
11637 assert_eq!(
11638 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
11639 Some(resumed_instruction(&old, &t))
11640 );
11641
11642 assert_eq!(
11646 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
11647 None
11648 );
11649 }
11650
11651 #[test]
11652 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
11653 assert_eq!(
11654 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
11655 Starter::Review("magi/eba2/A".to_owned())
11656 );
11657 }
11658
11659 #[test]
11660 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
11661 assert_eq!(
11662 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
11663 Starter::Start,
11664 "a vanished review branch must not fall back to resuming the old run either"
11665 );
11666 }
11667
11668 #[test]
11669 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11670 let mut t = task();
11671 t.start("old-run".to_owned());
11672 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11673 t.release();
11674 let branch = t.review_branch.take();
11675 assert_eq!(
11676 choose_starter(branch.as_deref(), true, Some("old-run")),
11677 Starter::Review("magi/eba2/A".to_owned()),
11678 "a review wins over resuming the old run"
11679 );
11680 }
11681
11682 #[test]
11683 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11684 assert_eq!(
11685 choose_starter(None, false, Some("some-run")),
11686 Starter::Resume("some-run".to_owned())
11687 );
11688 assert_eq!(choose_starter(None, false, None), Starter::Start);
11689 }
11690
11691 #[test]
11692 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11693 let mut released = task();
11694 released.start("stalled-run".to_owned());
11695 released.requeue();
11696 let unfinished = (!released.fresh_start)
11697 .then(|| Some("stalled-run".to_owned()))
11698 .flatten();
11699 assert_eq!(
11700 choose_starter(None, false, unfinished.as_deref()),
11701 Starter::Start,
11702 "release keeps run history but must not resume it"
11703 );
11704 assert_eq!(released.runs, ["stalled-run"]);
11705 }
11706
11707 fn stopped_public_run(status: RunStatus) -> RunState {
11708 let mut state = run_state(status);
11709 state.driver_exited = true;
11710 state.config.graph.land = true;
11711 state.config.graph.review_rounds = 1;
11712 state.reviews = vec![review_round(1)];
11713 state.merge = Some(crate::run::MergeOutcome {
11714 mode: MergeMode::Pr,
11715 ok: false,
11716 detail: "landing stopped".into(),
11717 empty: false,
11718 });
11719 state.pr = Some(crate::run::PrRecord {
11720 url: "https://example.test/pull/1".into(),
11721 number: 1,
11722 state: "open".into(),
11723 checks: "red".into(),
11724 round: 1,
11725 rounds: 1,
11726 red_at_merge: Vec::new(),
11727 });
11728 state
11729 }
11730
11731 #[test]
11732 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11733 for status in [RunStatus::Blocked, RunStatus::Gating] {
11734 let home = tempfile::tempdir().unwrap();
11735 let mut state = stopped_public_run(status);
11736 state.save_under(home.path()).unwrap();
11737 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11738 assert!(!exhausted_review_budget(&state));
11739 let mut task = held_task_with(&state.id);
11740 task.release();
11741 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11742 assert_eq!(
11743 choose_starter(None, false, unfinished.as_deref()),
11744 Starter::Resume(state.id.clone())
11745 );
11746 let question = action_question(&state.id, resume_action(&state.id));
11747 assert_eq!(
11748 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11749 ActionDecision::Resume(state.id.clone())
11750 );
11751 assert_eq!(
11753 crate::resume::validate(home.path(), &state.id).unwrap().id,
11754 state.id
11755 );
11756 let queue = Queue::at(home.path().join("queue"));
11757 let mut controlled = Task::new(
11758 "resume land".into(),
11759 state.instruction.clone(),
11760 state.repo.clone(),
11761 Source::Agent {
11762 run: "talk".into(),
11763 node: crate::queue::CHAT_NODE.into(),
11764 },
11765 );
11766 controlled.runs.push(state.id.clone());
11767 controlled.hold_machine(Some("land stopped".into()));
11768 queue.put(&mut controlled).unwrap();
11769 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
11770 assert_eq!(resumed.status, TaskStatus::Queued);
11771 assert_eq!(
11772 resumed.resume_override.unwrap().pinned_run,
11773 Some(state.id.clone())
11774 );
11775 assert_eq!(resumed.attempts, 0);
11776 }
11777 }
11778
11779 #[test]
11780 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11781 for review_rounds in [0, 1] {
11782 let home = tempfile::tempdir().unwrap();
11783 let mut state = run_state(RunStatus::Blocked);
11784 state.config.graph.review_rounds = review_rounds;
11785 if review_rounds > 0 {
11786 let mut review = review_round(1);
11787 review.clean = true;
11788 state.reviews = vec![review];
11789 }
11790 state.gate_ran = true;
11791 state.gate = vec![crate::run::CommandOutcome {
11792 command: "verify".into(),
11793 code: Some(1),
11794 output_tail: "gate failed after repair budget exhausted".into(),
11795 duration_ms: 0,
11796 resource_blocked: false,
11797 }];
11798 state.save_under(home.path()).unwrap();
11799 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11800 assert!(exhausted_review_budget(&state));
11801 let mut task = held_task_with(&state.id);
11802 task.release();
11803 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11804 let question = action_question(&state.id, resume_action(&state.id));
11805 assert_ne!(
11806 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11807 ActionDecision::Resume(state.id.clone())
11808 );
11809 let error = crate::resume::validate(home.path(), &state.id)
11810 .unwrap_err()
11811 .to_string();
11812 assert!(!error.contains("review budget exhausted"), "{error}");
11813 assert!(
11814 error.contains(if review_rounds == 0 {
11815 "review is disabled"
11816 } else {
11817 "gate already ran"
11818 }),
11819 "{error}"
11820 );
11821
11822 state.gate_ran = false;
11824 state.gate.clear();
11825 assert!(!exhausted_review_budget(&state));
11826 assert_eq!(
11827 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11828 Some(state.id)
11829 );
11830 }
11831 }
11832
11833 #[test]
11834 fn resource_blocked_verification_resumes_the_same_run() {
11835 for (kind, stage) in [
11836 ("cache", "clean"),
11837 ("coordination", "clean"),
11838 ("cache", "review-e2e"),
11839 ("coordination", "no-reviews"),
11840 ("coordination", "handoff"),
11841 ] {
11842 let home = tempfile::tempdir().unwrap();
11843 let mut state = run_state(RunStatus::Blocked);
11844 state.config.graph.review_rounds = 1;
11845 let mut review = review_round(1);
11846 review.clean = true;
11847 state.reviews = vec![review];
11848 if stage == "review-e2e" {
11849 let review = &mut state.reviews[0];
11850 review.clean = false;
11851 review.blocking = 1;
11852 review.e2e = vec![crate::run::CommandOutcome {
11853 command: "verify".into(),
11854 code: None,
11855 output_tail: "resource wait expired".into(),
11856 duration_ms: 0,
11857 resource_blocked: true,
11858 }];
11859 } else if stage == "no-reviews" {
11860 state.config.graph.review_rounds = 0;
11861 state.reviews.clear();
11862 } else if stage == "handoff" {
11863 let review = &mut state.reviews[0];
11866 review.clean = false;
11867 review.blocking = 1;
11868 review.e2e = vec![crate::run::CommandOutcome {
11869 command: "verify".into(),
11870 code: Some(0),
11871 output_tail: String::new(),
11872 duration_ms: 0,
11873 resource_blocked: false,
11874 }];
11875 }
11876 state.gate_ran = false;
11877 state.resource_block = Some(crate::run::ResourceBlock {
11878 cause: crate::run::ResourceBlockCause::Other,
11879 node: if stage == "review-e2e" {
11880 "review"
11881 } else {
11882 "gate"
11883 }
11884 .into(),
11885 kind: kind.into(),
11886 detail: "resource wait expired".into(),
11887 at: jiff::Timestamp::now(),
11888 });
11889 state.save_under(home.path()).unwrap();
11890 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11891 assert!(!exhausted_review_budget(&state));
11892 let mut task = held_task_with(&state.id);
11893 task.release();
11894 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11895 assert_eq!(
11896 choose_starter(None, false, unfinished.as_deref()),
11897 Starter::Resume(state.id.clone())
11898 );
11899 let question = action_question(&state.id, resume_action(&state.id));
11900 assert_eq!(
11901 decide_action(&task, &question, &PHRASES_EN, |_| Ok(state.clone())),
11902 ActionDecision::Resume(state.id.clone())
11903 );
11904 assert_eq!(
11905 crate::resume::validate(home.path(), &state.id).unwrap().id,
11906 state.id
11907 );
11908 let queue = Queue::at(home.path().join("queue"));
11909 let mut controlled = Task::new(
11910 "resume verification".into(),
11911 state.instruction.clone(),
11912 state.repo.clone(),
11913 Source::Agent {
11914 run: "talk".into(),
11915 node: crate::queue::CHAT_NODE.into(),
11916 },
11917 );
11918 controlled.runs.push(state.id.clone());
11919 controlled.hold_machine(Some("verification resource unavailable".into()));
11920 queue.put(&mut controlled).unwrap();
11921 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
11922 assert_eq!(resumed.status, TaskStatus::Queued);
11923 assert_eq!(
11924 resumed.resume_override.unwrap().pinned_run,
11925 Some(state.id.clone())
11926 );
11927 assert_eq!(resumed.attempts, 0);
11928
11929 state.config.graph.review_rounds = 1;
11931 let mut unresolved = review_round(1);
11932 unresolved.blocking = 1;
11933 state.reviews = vec![unresolved];
11934 state.resource_block = None;
11935 state.save_under(home.path()).unwrap();
11936 assert!(exhausted_review_budget(&state));
11937 assert!(crate::resume::validate(home.path(), &state.id).is_err());
11938 }
11939 }
11940
11941 #[tokio::test]
11942 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
11943 if !crate::standalone_test(
11944 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
11945 ) {
11946 return;
11947 }
11948 let home = tempfile::tempdir().unwrap();
11949 let mut owner = stopped_public_run(RunStatus::Blocked);
11950 owner.save_under(home.path()).unwrap();
11951 let history = vec![owner.id.clone()];
11952 let repo = owner.repo.clone();
11953 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
11954 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
11955 let initial =
11956 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
11957 assert!(
11958 initial.is_pending(),
11959 "a busy record must not hold or resume the task"
11960 );
11961 owner.retired_to = Some("20260101-000001-bbbb".into());
11962 owner.save_under(home.path()).unwrap();
11963 drop(lock);
11964 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
11965 .await
11966 .expect("ownership lock released")
11967 .expect("read settled owner");
11968 assert!(
11969 result.is_none(),
11970 "the retired owner must not be resumed from a stale read"
11971 );
11972 }
11973
11974 #[tokio::test]
11975 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
11976 if !crate::standalone_test(
11977 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
11978 ) {
11979 return;
11980 }
11981 let home = tempfile::tempdir().unwrap();
11982 let mut owner = stopped_public_run(RunStatus::Blocked);
11983 let repo = tempfile::tempdir().unwrap();
11984 crate::git::git(repo.path(), &["init"]).await.unwrap();
11985 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
11986 owner.id = "20260101-000000-aaaa".into();
11987 owner.save_under(home.path()).unwrap();
11988 let mut successor = run_state(RunStatus::Blocked);
11989 successor.id = "20260101-000001-bbbb".into();
11990 successor.save_under(home.path()).unwrap();
11991 let runs = vec![owner.id.clone(), successor.id.clone()];
11992 assert_eq!(
11993 public_resume_owner(home.path(), &runs, &owner.repo)
11994 .await
11995 .unwrap()
11996 .unwrap()
11997 .id,
11998 owner.id
11999 );
12000 let subdir = owner.repo.join("nested");
12001 std::fs::create_dir(&subdir).unwrap();
12002 assert_eq!(
12003 public_resume_owner(home.path(), &runs, &subdir)
12004 .await
12005 .unwrap()
12006 .unwrap()
12007 .id,
12008 owner.id
12009 );
12010 #[cfg(unix)]
12011 {
12012 let alias = home.path().join("repo-alias");
12013 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
12014 assert_eq!(
12015 public_resume_owner(home.path(), &runs, &alias.join("nested"))
12016 .await
12017 .unwrap()
12018 .unwrap()
12019 .id,
12020 owner.id
12021 );
12022 }
12023 let different = tempfile::tempdir().unwrap();
12024 crate::git::git(different.path(), &["init"]).await.unwrap();
12025 assert!(
12026 public_resume_owner(home.path(), &runs, different.path())
12027 .await
12028 .is_err()
12029 );
12030 std::fs::remove_file(
12031 home.path()
12032 .join("runs")
12033 .join(&successor.id)
12034 .join("run.json"),
12035 )
12036 .unwrap();
12037 assert!(
12038 public_resume_owner(home.path(), &runs, &owner.repo)
12039 .await
12040 .is_err()
12041 );
12042 }
12043
12044 #[test]
12045 fn an_ordinary_release_keeps_a_resumable_run_available() {
12046 let mut released = task();
12047 released.start("stalled-run".to_owned());
12048 released.release();
12049 let unfinished = (!released.fresh_start)
12050 .then(|| Some("stalled-run".to_owned()))
12051 .flatten();
12052 assert_eq!(
12053 choose_starter(None, false, unfinished.as_deref()),
12054 Starter::Resume("stalled-run".to_owned()),
12055 "manual release must preserve the normal resume path"
12056 );
12057 }
12058
12059 #[test]
12060 fn resume_refusals_explain_admission_without_changing_task_or_run() {
12061 crate::run::pin_test_home();
12062 use crate::handover::{ResumeRefusal, resume_refusal_under};
12063 let home = tempfile::tempdir().unwrap();
12064 let mut exhausted = run_state(RunStatus::Blocked);
12065 exhausted.driver_exited = true;
12066 exhausted.config.graph.review_rounds = 3;
12067 exhausted.reviews = (1..=3).map(review_round).collect();
12068 for refusal in [
12069 ResumeRefusal::ReviewBudget { used: 3, limit: 3 },
12070 ResumeRefusal::Status(RunStatus::Ready),
12071 ResumeRefusal::Released("20261022-000000-aaaa".into()),
12072 ResumeRefusal::Retired("retirement-successor".into()),
12073 ResumeRefusal::OwnershipTransferred("20261022-000000-bbbb".into()),
12074 ] {
12075 let mut state = exhausted.clone();
12076 let expected = match refusal {
12077 ResumeRefusal::Status(_) => {
12078 state.status = RunStatus::Ready;
12079 "status `ready`"
12080 }
12081 ResumeRefusal::Released(ref target) => {
12082 state.released_to = Some(target.clone());
12083 "execution was released"
12084 }
12085 ResumeRefusal::Retired(ref target) => {
12086 state.retired_to = Some(target.clone());
12087 "execution was retired"
12088 }
12089 ResumeRefusal::OwnershipTransferred(ref target) => {
12090 state.pr_taken_over_by = Some(target.clone());
12091 "PR ownership was transferred"
12092 }
12093 _ => "review budget exhausted (3/3 rounds used)",
12094 };
12095 assert_eq!(
12096 resume_refusal_under(&state, Some(home.path())),
12097 Some(refusal.clone())
12098 );
12099 assert_eq!(
12100 refusal.successor(),
12101 state
12102 .released_to
12103 .as_deref()
12104 .or(state.retired_to.as_deref())
12105 .or(state.pr_taken_over_by.as_deref())
12106 );
12107 state.save_under(home.path()).unwrap();
12108 let queue = Queue::at(home.path().join("queue"));
12109 let mut task = Task::new(
12110 "retained implementation".into(),
12111 state.instruction.clone(),
12112 state.repo.clone(),
12113 Source::Agent {
12114 run: "talk".into(),
12115 node: crate::queue::CHAT_NODE.into(),
12116 },
12117 );
12118 task.runs.push(state.id.clone());
12119 task.hold_machine(Some("stopped".into()));
12120 queue.put(&mut task).unwrap();
12121 let before_task = serde_json::to_value(queue.get(&task.id).unwrap()).unwrap();
12122 let before_run = serde_json::to_value(&state).unwrap();
12123 let error = crate::chat_control::resume(&queue, home.path(), &state.id)
12124 .unwrap_err()
12125 .to_string();
12126 assert!(error.contains(expected), "{error}");
12127 assert!(!error.contains("only a stalled or blocked"), "{error}");
12128 if matches!(refusal, ResumeRefusal::Status(_)) {
12129 assert!(
12130 error.contains("is final; there is nothing left to resume"),
12131 "{error}"
12132 );
12133 }
12134 if matches!(refusal, ResumeRefusal::ReviewBudget { .. }) {
12135 assert!(error.contains("magi review <saved-branch> --task <task-id> --merge none"));
12136 assert!(error.contains("review-only successor with a new review budget"));
12137 assert!(error.contains("does not resume this run"));
12138 }
12139 for target in [
12140 &state.released_to,
12141 &state.retired_to,
12142 &state.pr_taken_over_by,
12143 ]
12144 .into_iter()
12145 .flatten()
12146 {
12147 let short = if state.retired_to.as_ref() == Some(target) {
12148 target.as_str()
12149 } else {
12150 crate::run::short_of(target)
12151 };
12152 assert!(
12153 error.contains(&format!("run {short}; inspect `magi show {short}`")),
12154 "{error}"
12155 );
12156 if short != target {
12157 assert!(!error.contains(target), "full successor id leaked: {error}");
12158 }
12159 }
12160 assert_eq!(
12161 serde_json::to_value(queue.get(&task.id).unwrap()).unwrap(),
12162 before_task
12163 );
12164 assert_eq!(
12165 serde_json::to_value(RunState::load_under(&state.id, home.path()).unwrap())
12166 .unwrap(),
12167 before_run
12168 );
12169 let question = action_question(&state.id, resume_action(&state.id));
12170 for phrases in [&PHRASES_EN, &PHRASES_JA] {
12171 let ActionDecision::Refuse(detail) =
12172 decide_action(&task, &question, phrases, |_| Ok(state.clone()))
12173 else {
12174 panic!("refusal expected");
12175 };
12176 assert!(detail.contains(expected), "{detail}");
12177 }
12178 }
12179 exhausted.status = RunStatus::Ready;
12181 exhausted.released_to = Some("released".into());
12182 exhausted.retired_to = Some("retired".into());
12183 exhausted.pr_taken_over_by = Some("owner".into());
12184 let reason = resume_refusal_under(&exhausted, Some(home.path())).unwrap();
12185 assert_eq!(reason, ResumeRefusal::Status(RunStatus::Ready));
12186 let detail = reason.diagnostic(&exhausted);
12187 for target in ["released", "retired", "owner"] {
12188 assert!(detail.contains(&format!("magi show {target}")));
12189 }
12190 }
12191
12192 #[test]
12193 fn resume_checkpoint_admission_uses_the_supplied_question_store() {
12194 let home = tempfile::tempdir().unwrap();
12195 let mut state = run_state(RunStatus::Blocked);
12196 state.driver_exited = true;
12197 state.config.graph.review_rounds = 1;
12198 state.reviews = vec![review_round(1)];
12199 assert!(crate::handover::resume_refusal_under(&state, Some(home.path())).is_some());
12200 for checkpoint in ["publication", "approval", "armed", "landing", "question"] {
12201 let mut saved = state.clone();
12202 match checkpoint {
12203 "publication" => saved.pr_publication_pending = true,
12204 "approval" => {
12205 saved.land_approval = Some(crate::run::LandApproval {
12206 question: "approval".into(),
12207 head: "head".into(),
12208 })
12209 }
12210 "armed" => saved.land_armed_head = Some("head".into()),
12211 "landing" => saved.status = RunStatus::Landing,
12212 _ => {
12213 let mut question = ask::Question::new(
12214 saved.id.clone(),
12215 "review".into(),
12216 "review-1".into(),
12217 "continue?".into(),
12218 String::new(),
12219 vec!["continue".into()],
12220 );
12221 ask::Questions::at(home.path().join("questions"))
12222 .put(&mut question)
12223 .unwrap();
12224 }
12225 }
12226 saved.save_under(home.path()).unwrap();
12227 assert!(
12228 crate::handover::resume_refusal_under(&saved, Some(home.path())).is_none(),
12229 "{checkpoint}"
12230 );
12231 assert!(
12232 crate::resume::validate(home.path(), &saved.id).is_ok(),
12233 "{checkpoint}"
12234 );
12235 }
12236 }
12237
12238 #[test]
12239 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
12240 let mut state = run_state(RunStatus::Blocked);
12241 state.config.graph.review_rounds = 3;
12242 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
12243 assert!(exhausted_review_budget(&state));
12244
12245 state.reviews.pop();
12247 assert!(!exhausted_review_budget(&state));
12248
12249 let mut stalled = run_state(RunStatus::Stalled);
12252 stalled.config.graph.review_rounds = 1;
12253 stalled.reviews = vec![review_round(1)];
12254 assert!(!exhausted_review_budget(&stalled));
12255 }
12256
12257 fn review_round(round: usize) -> crate::run::ReviewRound {
12258 crate::run::ReviewRound {
12259 round,
12260 head: "deadbeef".to_owned(),
12261 verified_head: None,
12262 verified_at: None,
12263 reviews: Vec::new(),
12264 e2e: Vec::new(),
12265 verify_retried: false,
12266 e2e_deferred: false,
12267 e2e_defer_reason: None,
12268 fix: None,
12269 blocking: 0,
12270 answered: 1,
12271 expected: 1,
12272 clean: false,
12273 progressed: true,
12274 vote_split: false,
12275 reconsideration: Vec::new(),
12276 verdict: None,
12277 }
12278 }
12279
12280 #[test]
12281 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
12282 let mut run = RunState::new(
12283 PathBuf::from("/repo"),
12284 "main".to_owned(),
12285 "abc1234def".to_owned(),
12286 "add retries".to_owned(),
12287 Config::default(),
12288 );
12289 run.status = RunStatus::Judging;
12290 run.parked = true;
12291 let mut task = Task::new(
12292 "add retries".to_owned(),
12293 "add retries".to_owned(),
12294 PathBuf::from("/repo"),
12295 crate::queue::Source::Human,
12296 );
12297 task.status = TaskStatus::Failed;
12298 task.runs = vec![run.id.clone()];
12299 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
12300 assert!(with(&task, &run), "parked after judging is the case");
12301 let mut parked_task = task.clone();
12302 parked_task.status = TaskStatus::Parked;
12303 assert!(with(&parked_task, &run), "the Parked status is the case");
12304
12305 let mut not_parked = run.clone();
12306 not_parked.parked = false;
12307 not_parked.status = RunStatus::Stalled;
12308 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
12309
12310 let mut fresh = task.clone();
12311 fresh.fresh_start = true;
12312 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
12313
12314 let mut review = task.clone();
12315 review.review_branch = Some("magi/x/A".to_owned());
12316 assert!(!with(&review, &run), "review is ranked before resume");
12317
12318 let mut held = task.clone();
12319 held.status = TaskStatus::Held;
12320 assert!(!with(&held, &run), "a hold stays visible to the conductor");
12321
12322 let mut released = run.clone();
12323 released.released_to = Some("20260901-000000-new1".to_owned());
12324 assert!(!with(&task, &released), "nothing left to resume into");
12325
12326 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
12327 "unreadable"
12328 )));
12329 }
12330
12331 #[test]
12332 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
12333 let mut released = RunState::new(
12334 PathBuf::from("/repo"),
12335 "main".to_owned(),
12336 "abc1234def".to_owned(),
12337 "add retries".to_owned(),
12338 Config::default(),
12339 );
12340 released.status = RunStatus::Blocked;
12341 assert_eq!(
12342 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12343 Some(released.id.clone())
12344 );
12345 released.released_to = Some("20260901-000000-new1".to_owned());
12346 assert_eq!(
12347 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12348 None,
12349 "there is nothing left to resume it into"
12350 );
12351 }
12352
12353 #[test]
12354 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
12355 let mut exhausted = RunState::new(
12364 PathBuf::from("/repo"),
12365 "main".to_owned(),
12366 "abc1234def".to_owned(),
12367 "add retries".to_owned(),
12368 Config::default(),
12369 );
12370 exhausted.status = RunStatus::Blocked;
12371 exhausted.config.graph.review_rounds = 1;
12372 exhausted.reviews = vec![review_round(1)];
12373
12374 assert_eq!(
12375 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
12376 None,
12377 "an exhausted `Blocked` run must not be offered as resumable"
12378 );
12379
12380 let mut has_budget_left = RunState::new(
12383 PathBuf::from("/repo"),
12384 "main".to_owned(),
12385 "abc1234def".to_owned(),
12386 "add retries".to_owned(),
12387 Config::default(),
12388 );
12389 has_budget_left.status = RunStatus::Blocked;
12390 has_budget_left.config.graph.review_rounds = 3;
12391 has_budget_left.reviews = vec![review_round(1)];
12392
12393 assert_eq!(
12394 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
12395 Ok(has_budget_left.clone())
12396 }),
12397 Some(has_budget_left.id.clone())
12398 );
12399 }
12400
12401 #[test]
12402 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
12403 let mut run = RunState::new(
12404 PathBuf::from("/repo"),
12405 "main".to_owned(),
12406 "abc1234def".to_owned(),
12407 "add retries".to_owned(),
12408 Config::default(),
12409 );
12410 run.status = RunStatus::Blocked;
12411 run.config.graph.review_rounds = 1;
12412 run.reviews = vec![review_round(1)];
12413 run.pr_publication_pending = true;
12414 assert_eq!(
12415 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
12416 Some(run.id.clone()),
12417 "a spent review budget must not turn a failed publication into a re-competition"
12418 );
12419 }
12420
12421 #[test]
12422 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
12423 let mut task = Task::new(
12424 "recover work".into(),
12425 "recover work".into(),
12426 PathBuf::from("/repo"),
12427 crate::queue::Source::Human,
12428 );
12429 let mut history = std::collections::BTreeMap::new();
12430 for index in 0..8 {
12431 let mut run = run_state(RunStatus::Blocked);
12432 run.id = format!("20260101-000000-{index:04}");
12433 run.base_commit = format!("upstream-{index}");
12434 if index > 0 {
12436 run.candidates.clear();
12437 }
12438 task.runs.extend([run.id.clone(), run.id.clone()]);
12439 history.insert(run.id.clone(), run);
12440 task.requeue();
12441 }
12442 let check = |task: &Task, starter: &Starter, limit| {
12443 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
12444 };
12445 assert!(
12446 check(&task, &Starter::Start, 8)
12447 .unwrap()
12448 .contains("8 unretired")
12449 );
12450 assert!(check(&task, &Starter::Start, 9).is_none());
12451 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
12452 task.release_operator("owner", None);
12453 assert_eq!(task.attempts, 0);
12454 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
12455 assert!(check(&restored, &Starter::Start, 8).is_some());
12456 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
12457 assert!(check(&restored, &Starter::Start, 0).is_none());
12458 }
12459
12460 #[test]
12461 fn retained_attempt_bound_includes_failed_startups_without_execution_history() {
12462 let tmp = tempfile::tempdir().unwrap();
12463 let mut task = Task::new(
12464 "recover selected PR".into(),
12465 "preserve implementation".into(),
12466 tmp.path().join("repo"),
12467 crate::queue::Source::Human,
12468 );
12469 let mut failed = run_state(RunStatus::Failed);
12470 failed.id = "20260101-000000-fail".into();
12471 failed.candidates.clear();
12472 failed.prep_complete = Some(false);
12473 failed.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12474 failed.startup = Some(crate::run::Startup {
12475 kind: "pr-recovery".into(),
12476 branch: Some("selected-public-head".into()),
12477 stage: "worktree-setup".into(),
12478 complete: false,
12479 failure: Some("setup modified a tracked file and failed".into()),
12480 });
12481 let worktree = failed.worktree_root().join("under-review");
12484 std::fs::create_dir_all(&worktree).unwrap();
12485 let retained_edit = worktree.join("implementation");
12486 std::fs::write(&retained_edit, "retained setup edit").unwrap();
12487 task.record_startup_failure(crate::run::StartupFailure {
12488 run: failed.id.clone(),
12489 stage: "worktree-setup".into(),
12490 });
12491 let task: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
12492 assert!(task.runs.is_empty() && task.passes.is_empty());
12493 assert_eq!(task.attempts, 0);
12494 let starter = Starter::Review("selected-public-head".into());
12495 assert!(
12496 retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone()))
12497 .unwrap()
12498 .contains("1 unretired")
12499 );
12500 assert!(retained_attempt_hold(&task, &starter, 2, |_| Ok(failed.clone())).is_none());
12501 assert!(
12502 retained_attempt_hold(&task, &starter, 2, |_| anyhow::bail!("unreadable"))
12503 .unwrap()
12504 .contains("Cannot verify")
12505 );
12506 let mut mixed = task.clone();
12508 mixed.runs.push(failed.id.clone());
12509 mixed
12510 .startup_failures
12511 .push(task.startup_failures[0].clone());
12512 assert!(retained_attempt_hold(&mixed, &starter, 2, |_| Ok(failed.clone())).is_none());
12513 failed.retired_to = Some("verified-recovery-owner".into());
12514 assert!(retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone())).is_none());
12515 assert!(
12516 retained_attempt_hold(&task, &Starter::Resume(failed.id.clone()), 1, |_| {
12517 panic!("resuming the existing owner must not allocate a new attempt")
12518 })
12519 .is_none()
12520 );
12521 assert_eq!(
12522 std::fs::read_to_string(retained_edit).unwrap(),
12523 "retained setup edit"
12524 );
12525 assert!(task.runs.is_empty() && task.passes.is_empty());
12526 assert_eq!(task.attempts, 0);
12527 }
12528
12529 #[test]
12530 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
12531 let mut task = Task::new(
12532 "recover work".into(),
12533 "recover work".into(),
12534 PathBuf::from("/repo"),
12535 crate::queue::Source::Human,
12536 );
12537 let mut run = run_state(RunStatus::Superseded);
12538 run.candidates.clear();
12539 task.runs.push(run.id.clone());
12540 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
12541 run.retired_to = Some("authority".into());
12542 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
12543 run.retired_to = None;
12544 run.released_to = Some("authority".into());
12545 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
12546 assert!(
12547 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
12548 .unwrap()
12549 .contains("Cannot verify")
12550 );
12551 assert!(
12553 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
12554 panic!("resume must not load historical attempts")
12555 })
12556 .is_none()
12557 );
12558 }
12559
12560 #[test]
12561 fn retired_resume_requires_a_valid_same_repository_authority() {
12562 let mut source = run_state(RunStatus::Blocked);
12563 let mut empty = run_state(RunStatus::Blocked);
12564 empty.retired_to = Some(source.id.clone());
12565 let runs = vec![source.id.clone(), empty.id.clone()];
12566 let resolve = |source: &RunState, empty: &RunState| {
12567 unfinished_run_with(&runs, "fixture", |id| {
12568 if id == source.id {
12569 Ok(source.clone())
12570 } else if id == empty.id {
12571 Ok(empty.clone())
12572 } else {
12573 bail!("unreadable authority")
12574 }
12575 })
12576 };
12577 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
12578 assert_eq!(
12579 choose_starter(None, false, resolve(&source, &empty).as_deref()),
12580 Starter::Resume(source.id.clone())
12581 );
12582 source.retired_to = Some(empty.id.clone());
12583 assert!(
12584 resolve(&source, &empty).is_none(),
12585 "cycles do not revive obsolete runs"
12586 );
12587 source.retired_to = None;
12588 source.repo = PathBuf::from("/another-repository");
12589 assert!(resolve(&source, &empty).is_none());
12590 source.repo = empty.repo.clone();
12591 source.released_to = Some("later".into());
12592 assert!(resolve(&source, &empty).is_none());
12593 empty.retired_to = Some("unlisted-run".into());
12594 assert!(resolve(&source, &empty).is_none());
12595 }
12596
12597 #[test]
12598 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
12599 let mut older_stalled = RunState::new(
12607 PathBuf::from("/repo"),
12608 "main".to_owned(),
12609 "abc1234def".to_owned(),
12610 "add retries".to_owned(),
12611 Config::default(),
12612 );
12613 older_stalled.status = RunStatus::Stalled;
12614
12615 let mut newest_exhausted = RunState::new(
12616 PathBuf::from("/repo"),
12617 "main".to_owned(),
12618 "abc1234def".to_owned(),
12619 "add retries".to_owned(),
12620 Config::default(),
12621 );
12622 newest_exhausted.status = RunStatus::Blocked;
12623 newest_exhausted.config.graph.review_rounds = 1;
12624 newest_exhausted.reviews = vec![review_round(1)];
12625
12626 assert_eq!(
12627 unfinished_run_with(
12628 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
12629 "t",
12630 |_| Ok(newest_exhausted.clone())
12631 ),
12632 None,
12633 "the newest run is exhausted, so nothing here is worth resuming - \
12634 least of all the older, already-superseded run"
12635 );
12636 }
12637
12638 #[test]
12639 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
12640 assert_eq!(
12641 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
12642 Err(anyhow::anyhow!("fixture is absent"))
12643 }),
12644 None
12645 );
12646 }
12647
12648 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
12649 let mut q = ask::Question::new(
12650 run.to_owned(),
12651 "implement".to_owned(),
12652 "impl-A".to_owned(),
12653 "continue?".to_owned(),
12654 String::new(),
12655 vec!["resume で続行する".to_owned(), "other".to_owned()],
12656 );
12657 q.actions.insert("resume で続行する".to_owned(), action);
12658 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
12659 .unwrap();
12660 q
12661 }
12662
12663 fn held_task_with(run: &str) -> Task {
12664 let mut t = task();
12665 t.runs = vec![run.to_owned()];
12666 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
12667 t
12668 }
12669
12670 fn resume_action(run: &str) -> ask::ChoiceAction {
12671 ask::ChoiceAction::Resume { run: run.into() }
12672 }
12673
12674 #[test]
12675 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
12676 for parked in [false, true] {
12677 let mut t = task();
12678 t.start("A".into());
12679 t.start("B".into());
12680 assert_eq!(t.runs, vec!["A", "B"]);
12681 t.start("A".into());
12682 let mut resumed = run_state(RunStatus::Blocked);
12683 resumed.id = "A".into();
12684 resumed.parked = parked;
12685 resumed.driver_exited = true;
12686 t.hold_machine(Some("resumed run stopped".into()));
12687 let q = action_question("A", resume_action("A"));
12688 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12689 assert_eq!(
12690 decide_action(&t, &q, &PHRASES_EN, |id| {
12691 assert_eq!(id, "A");
12692 Ok(resumed.clone())
12693 }),
12694 ActionDecision::Resume("A".into())
12695 );
12696 assert_eq!(
12697 action_standing(&t, &action_question("B", resume_action("B"))),
12698 ActionStanding::Stale
12699 );
12700 t.release_operator("cli", None);
12701 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
12702 assert_eq!(id, "A");
12703 Ok(resumed.clone())
12704 });
12705 assert_eq!(
12706 choose_starter(None, false, unfinished.as_deref()),
12707 Starter::Resume("A".into())
12708 );
12709 assert_eq!(t.runs, vec!["B", "A"]);
12710 }
12711 }
12712
12713 #[test]
12714 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
12715 let mut state = crate::land::tests::repair_state(true, true);
12716 state.status = RunStatus::Landing;
12717 state.parked = true;
12718 state.driver_exited = true;
12719 let mut q = ask::Question::new(
12720 state.id.clone(),
12721 land::APPROVAL_NODE.into(),
12722 "land".into(),
12723 "Merge?".into(),
12724 String::new(),
12725 vec![land::APPROVE.into(), land::HOLD.into()],
12726 );
12727 q.id = state.id.clone();
12728 state.land_approval = Some(crate::run::LandApproval {
12729 question: q.id.clone(),
12730 head: "current-head".into(),
12731 });
12732 assert!(land::upgrade_repair_choice(&mut q, &state));
12733 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12734 let mut task = held_task_with(&state.id);
12735 task.status = TaskStatus::Parked;
12736 assert_eq!(
12737 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12738 ActionDecision::Resume(state.id.clone())
12739 );
12740 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12741 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
12742 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
12743 assert_eq!(
12744 decide_action(&task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12745 ActionDecision::Resume(state.id.clone()),
12746 "a repair answer newer than the manual hold resumes the existing owner"
12747 );
12748 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12749 for answered_at in [q.answered_at, None] {
12750 let mut older = q.clone();
12751 older.answered_at = answered_at;
12752 let refused = decide_action(&task, &older, &PHRASES_EN, |_| {
12753 panic!("a later manual hold must prevent dispatch")
12754 });
12755 let ActionDecision::Refuse(reason) = refused else {
12756 panic!("expected an actionable refusal, got {refused:?}");
12757 };
12758 assert!(reason.contains("later operator decision"));
12759 assert!(reason.contains(&format!("magi task release {}", task.id)));
12760 }
12761 task.operator_decided_at = None;
12762 let mut superseded = state.clone();
12763 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
12764 assert_eq!(
12765 decide_action(&task, &q, &PHRASES_EN, |_| Ok(superseded)),
12766 ActionDecision::Stale
12767 );
12768 for case in 1..4 {
12769 let mut changed = state.clone();
12770 match case {
12771 1 => changed.status = RunStatus::Blocked,
12772 2 => changed.released_to = Some("new-owner".into()),
12773 _ => changed.land_approval.as_mut().unwrap().head.clear(),
12774 }
12775 assert!(matches!(
12776 decide_action(&task, &q, &PHRASES_EN, |_| Ok(changed)),
12777 ActionDecision::Refuse(_)
12778 ));
12779 }
12780 assert!(matches!(
12781 decide_action(&task, &q, &PHRASES_EN, |_| anyhow::bail!("missing owner")),
12782 ActionDecision::Refuse(_)
12783 ));
12784 task.runs.push("replacement".into());
12785 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12786 assert_eq!(
12787 decide_action(&task, &q, &PHRASES_EN, |_| panic!(
12788 "old owner must not load"
12789 )),
12790 ActionDecision::Stale
12791 );
12792 }
12793
12794 #[test]
12795 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
12796 let mut state = crate::land::tests::repair_state(true, true);
12797 state.status = RunStatus::Landing;
12798 state.parked = true;
12799 state.driver_exited = true;
12800 let mut q = ask::Question::new(
12801 state.id.clone(),
12802 land::APPROVAL_NODE.into(),
12803 "land".into(),
12804 "Merge?".into(),
12805 String::new(),
12806 vec![land::APPROVE.into(), land::HOLD.into()],
12807 );
12808 q.id = state.id.clone();
12809 state.land_approval = Some(crate::run::LandApproval {
12810 question: q.id.clone(),
12811 head: "current-head".into(),
12812 });
12813 state.save().unwrap();
12814 land::upgrade_repair_choice(&mut q, &state);
12815 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12816 Questions::open().put(&mut q).unwrap();
12817 let tmp = tempfile::tempdir().unwrap();
12818 let queue = Queue::at(tmp.path().join("queue"));
12819 let mut task = held_task_with(&state.id);
12820 task.status = TaskStatus::Parked;
12821 queue.put(&mut task).unwrap();
12822 let empty = Arc::new(tokio::sync::Semaphore::new(0));
12823 let free = Arc::new(tokio::sync::Semaphore::new(1));
12824 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
12825 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
12826 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
12827 assert!(claim.permit.is_some());
12828 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
12829 drop(claim);
12830 Questions::open()
12831 .update(&q.id, |q| {
12832 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
12833 Ok(())
12834 })
12835 .unwrap();
12836 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
12837 assert!(claim.permit.is_none());
12838 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
12839 }
12840
12841 #[test]
12842 fn decide_action_resumes_only_the_latest_resumable_run() {
12843 let t = held_task_with("r1");
12844 let q = action_question("r1", resume_action("r1"));
12845 let load = |s: RunState| move |_: &str| Ok(s);
12846 assert_eq!(
12847 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Blocked))),
12848 ActionDecision::Resume("r1".into())
12849 );
12850 let q_other = action_question("r1", resume_action("r0"));
12852 assert!(matches!(
12853 decide_action(
12854 &t,
12855 &q_other,
12856 &PHRASES_EN,
12857 load(run_state(RunStatus::Blocked))
12858 ),
12859 ActionDecision::Refuse(_)
12860 ));
12861 assert!(matches!(
12863 decide_action(&t, &q, &PHRASES_EN, load(run_state(RunStatus::Ready))),
12864 ActionDecision::Refuse(_)
12865 ));
12866 let mut released = run_state(RunStatus::Blocked);
12868 released.released_to = Some("elsewhere".into());
12869 assert!(matches!(
12870 decide_action(&t, &q, &PHRASES_EN, load(released)),
12871 ActionDecision::Refuse(_)
12872 ));
12873 assert!(matches!(
12875 decide_action(&t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
12876 ActionDecision::Refuse(_)
12877 ));
12878 }
12879
12880 #[test]
12881 fn decide_action_ignores_a_question_about_an_earlier_run() {
12882 let mut t = held_task_with("r1");
12883 t.runs.push("r2".to_owned());
12884 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12885 assert_eq!(
12886 decide_action(
12887 &t,
12888 &action_question("r1", ask::ChoiceAction::Done),
12889 &PHRASES_EN,
12890 never
12891 ),
12892 ActionDecision::Stale
12893 );
12894 }
12895
12896 #[test]
12897 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
12898 let mut t = held_task_with("r1");
12899 let never = |_: &str| -> Result<RunState> { bail!("not read") };
12900 assert_eq!(
12901 decide_action(
12902 &t,
12903 &action_question("r1", ask::ChoiceAction::Requeue),
12904 &PHRASES_EN,
12905 never
12906 ),
12907 ActionDecision::Requeue
12908 );
12909 let done_q = action_question("r1", ask::ChoiceAction::Done);
12910 assert_eq!(
12911 decide_action(&t, &done_q, &PHRASES_EN, never),
12912 ActionDecision::Done
12913 );
12914 t.mark_action_applied(&done_q.id);
12915 assert_eq!(
12916 decide_action(&t, &done_q, &PHRASES_EN, never),
12917 ActionDecision::Skip
12918 );
12919
12920 let mut plain = action_question("r1", ask::ChoiceAction::Done);
12922 plain.actions.clear();
12923 assert_eq!(
12924 decide_action(&held_task_with("r1"), &plain, &PHRASES_EN, never),
12925 ActionDecision::Skip
12926 );
12927 let mut running = held_task_with("r1");
12929 running.status = TaskStatus::Running;
12930 assert_eq!(
12931 decide_action(
12932 &running,
12933 &action_question("r1", ask::ChoiceAction::Done),
12934 &PHRASES_EN,
12935 never
12936 ),
12937 ActionDecision::Skip
12938 );
12939 }
12940
12941 #[test]
12942 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
12943 crate::run::pin_test_home();
12944 for legacy in [false, true] {
12945 let dir = tempfile::tempdir().unwrap();
12946 let queue = Queue::at(dir.path().join("queue"));
12947 let questions = Questions::at(dir.path().join("questions"));
12948 let run = "20260101-000000-hold";
12949 let mut task = held_task_with(run);
12950 task.status = TaskStatus::Parked;
12951 let mut q = ask::Question::new(
12952 run.into(),
12953 land::APPROVAL_NODE.into(),
12954 "land".into(),
12955 "Repair?".into(),
12956 String::new(),
12957 vec![land::REPAIR.into()],
12958 );
12959 q.actions.insert(land::REPAIR.into(), resume_action(run));
12960 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12961 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12963 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
12964 questions.put(&mut q).unwrap();
12965 task.hold_manual(Some("Keep this PR paused".into()));
12966 if legacy {
12967 task.hold_source = None;
12968 }
12969 queue.put(&mut task).unwrap();
12970
12971 apply_task_choice_actions(&queue, &questions, dir.path());
12972 let after = queue.get(&task.id).unwrap();
12973 assert!(after.operator_held());
12974 assert_eq!(after.hold_source, task.hold_source);
12975 assert_eq!(after.operator_decided_at, task.operator_decided_at);
12976 assert_eq!(after.runs, task.runs);
12977 assert_eq!(after.instruction, task.instruction);
12978 assert!(after.resume_override.is_none());
12979 assert!(after.action_applied(&q.id));
12980 let reason = after.hold_reason.as_deref().unwrap();
12981 assert!(reason.starts_with("Keep this PR paused"));
12982 assert!(reason.contains("later operator decision"));
12983 assert!(reason.contains(&format!("magi task release {}", task.id)));
12984
12985 for recovery in [
12986 crate::conduct::Recovery::Review,
12987 crate::conduct::Recovery::Requeue,
12988 ] {
12989 crate::conduct::apply(
12990 &queue,
12991 &questions,
12992 &crate::conduct::Verdict {
12993 decisions: vec![crate::conduct::Decision {
12994 id: task.id.clone(),
12995 recovery: Some(recovery),
12996 ..Default::default()
12997 }],
12998 },
12999 )
13000 .unwrap();
13001 }
13002 apply_task_choice_actions(&queue, &questions, dir.path());
13005 let still_held = queue.get(&task.id).unwrap();
13006 assert!(still_held.operator_held());
13007 assert_eq!(still_held.hold_reason, after.hold_reason);
13008 assert_eq!(still_held.runs, task.runs);
13009 assert!(still_held.review_branch.is_none());
13010 assert!(!still_held.fresh_start);
13011 assert!(queue.next_runnable().is_none());
13012 }
13013 }
13014
13015 #[test]
13016 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
13017 for held in [false, true] {
13018 for retry in [false, true] {
13019 let dir = tempfile::tempdir().unwrap();
13020 let queue = Queue::at(dir.path().join("queue"));
13021 let questions = Questions::at(dir.path().join("questions"));
13022 let home = dir.path().join("home");
13023 let mut t = task();
13024 if held {
13025 t.hold_machine(Some("driver state unknown".into()));
13026 }
13027 queue.put(&mut t).unwrap();
13028 let verdict = serde_json::from_value(serde_json::json!({
13029 "decisions": [{
13030 "id": t.id,
13031 "question": "Driver state unknown; retry the checks or hold?",
13032 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
13033 "actions": {
13034 "daemon: recheck and retry": {"do": "requeue"},
13035 "daemon: hold the task": {"do": "hold"}
13036 }
13037 }]
13038 }))
13039 .unwrap();
13040 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
13041 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13042 let mut q = questions.list().pop().unwrap();
13043 let choice = if retry {
13044 "daemon: recheck and retry"
13045 } else {
13046 "daemon: hold the task"
13047 };
13048 assert_eq!(q.actions.len(), 2);
13049 q.answer(ask::Answer::Choice(choice.into())).unwrap();
13050 questions.put(&mut q).unwrap();
13051
13052 questions.beat(&q.id, ask::WaiterKind::Deputy);
13054 for _ in 0..2 {
13055 resolve_blockers(&queue, &questions);
13056 apply_task_choice_actions(&queue, &questions, &home);
13057 let waiting = queue.get(&t.id).unwrap();
13058 assert_eq!(waiting.status, TaskStatus::Blocked);
13059 assert_eq!(waiting.blocked_by, [q.id.clone()]);
13060 assert!(waiting.answers.is_empty());
13061 assert!(!waiting.action_applied(&q.id));
13062 assert!(runnable(&queue).is_empty());
13063 let free = Arc::new(tokio::sync::Semaphore::new(1));
13064 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
13065 }
13066 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13067
13068 resolve_blockers(&queue, &questions);
13070 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13071 apply_task_choice_actions(&queue, &questions, &home);
13072 let after = queue.get(&t.id).unwrap();
13073 assert_eq!(
13074 after.status,
13075 if retry {
13076 TaskStatus::Queued
13077 } else {
13078 TaskStatus::Held
13079 }
13080 );
13081 assert!(after.action_applied(&q.id));
13082 assert_eq!(after.operator_held(), !retry);
13083 assert_eq!(after.answers.len(), 1);
13084
13085 let mut changed = after;
13087 if retry {
13088 changed.hold_manual(Some("later hold".into()));
13089 } else {
13090 changed.release_operator("test", None);
13091 }
13092 let expected = changed.status;
13093 queue.put(&mut changed).unwrap();
13094 apply_task_choice_actions(&queue, &questions, &home);
13095 assert_eq!(queue.get(&t.id).unwrap().status, expected);
13096 }
13097 }
13098 }
13099
13100 #[test]
13101 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
13102 let dir = tempfile::tempdir().unwrap();
13103 let queue = Queue::at(dir.path().join("queue"));
13104 let questions = Questions::at(dir.path().join("questions"));
13105 let home = dir.path().join("home");
13106 let mut state = run_state(RunStatus::Blocked);
13107 state.id = "20260101-000000-act1".to_owned();
13108 state.save_under(&home).unwrap();
13109
13110 let mut t = held_task_with(&state.id);
13111 queue.put(&mut t).unwrap();
13112 let mut q = action_question(&state.id, resume_action(&state.id));
13113 questions.put(&mut q).unwrap();
13114
13115 apply_task_choice_actions(&queue, &questions, &home);
13116 let after = queue.get(&t.id).unwrap();
13117 assert_eq!(after.status, TaskStatus::Queued);
13118 assert!(!after.fresh_start);
13119 assert!(after.action_applied(&q.id));
13120 let pin = after.resume_override.clone().unwrap();
13121 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
13122 assert!(pin.forced);
13123
13124 let mut again = queue.get(&t.id).unwrap();
13126 again.hold_machine(Some("later".into()));
13127 queue.put(&mut again).unwrap();
13128 apply_task_choice_actions(&queue, &questions, &home);
13129 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13130 }
13131
13132 #[test]
13133 fn a_delivered_answer_is_still_acted_on_exactly_once() {
13134 let dir = tempfile::tempdir().unwrap();
13135 let queue = Queue::at(dir.path().join("queue"));
13136 let questions = Questions::at(dir.path().join("questions"));
13137 let home = dir.path().join("home");
13138 let mut state = run_state(RunStatus::Blocked);
13139 state.id = "20260101-000000-act2".to_owned();
13140 state.save_under(&home).unwrap();
13141
13142 let mut t = held_task_with(&state.id);
13143 queue.put(&mut t).unwrap();
13144 let mut q = action_question(&state.id, resume_action(&state.id));
13145 q.answer_delivered = true;
13146 questions.put(&mut q).unwrap();
13147
13148 apply_task_choice_actions(&queue, &questions, &home);
13151 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13152 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
13153
13154 let mut again = queue.get(&t.id).unwrap();
13156 again.hold_machine(Some("later".into()));
13157 queue.put(&mut again).unwrap();
13158 apply_task_choice_actions(&queue, &questions, &home);
13159 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13160 }
13161
13162 #[test]
13163 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
13164 let dir = tempfile::tempdir().unwrap();
13165 let queue = Queue::at(dir.path().join("queue"));
13166 let questions = Questions::at(dir.path().join("questions"));
13167 let home = dir.path().join("home");
13168 let mut state = run_state(RunStatus::Blocked);
13169 state.id = "20260101-000000-act3".to_owned();
13170 state.save_under(&home).unwrap();
13171 let mut t = held_task_with(&state.id);
13172 queue.put(&mut t).unwrap();
13173 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
13174 questions.put(&mut q).unwrap();
13175
13176 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
13177 apply_task_choice_actions(&queue, &questions, &home);
13178 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13179
13180 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13181 apply_task_choice_actions(&queue, &questions, &home);
13182 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13183 }
13184
13185 #[test]
13186 fn action_standing_tells_the_reasons_apart() {
13187 let mut t = held_task_with("r1");
13188 let q = action_question("r1", ask::ChoiceAction::Requeue);
13189 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
13190 let mut plain = q.clone();
13191 plain.actions.clear();
13192 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
13193
13194 t.mark_action_applied(&q.id);
13196 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
13197
13198 let mut t = held_task_with("r1");
13200 t.start("r2".to_owned());
13201 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
13202 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
13204 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13205 t.status = TaskStatus::Blocked;
13206 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13207
13208 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
13210 c.node = crate::conduct::NODE.to_owned();
13211 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
13212 assert!(!ActionStanding::Busy.daemon_owns());
13213 }
13214
13215 #[test]
13216 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
13217 let dir = tempfile::tempdir().unwrap();
13218 let queue = Queue::at(dir.path().join("queue"));
13219 let questions = Questions::at(dir.path().join("questions"));
13220 let home = dir.path().join("home");
13221 let mut state = run_state(RunStatus::Blocked);
13222 state.id = "20260101-000000-act4".to_owned();
13223 state.save_under(&home).unwrap();
13224 let mut t = held_task_with(&state.id);
13225 t.status = TaskStatus::Running;
13226 queue.put(&mut t).unwrap();
13227 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
13228 q.cwd = Some(dir.path().display().to_string());
13229 questions.put(&mut q).unwrap();
13230
13231 let running = queue.get(&t.id).unwrap();
13234 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
13235 assert!(matches!(
13236 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
13237 crate::waiter::Action::Deliver(_)
13238 ));
13239 apply_task_choice_actions(&queue, &questions, &home);
13241 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
13242 let mut back = queue.get(&t.id).unwrap();
13243 back.hold_machine(Some("later".into()));
13244 queue.put(&mut back).unwrap();
13245 apply_task_choice_actions(&queue, &questions, &home);
13246 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
13247 let mut again = queue.get(&t.id).unwrap();
13248 again.hold_machine(Some("again".into()));
13249 queue.put(&mut again).unwrap();
13250 apply_task_choice_actions(&queue, &questions, &home);
13251 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13252 }
13253}