1use std::path::{Path, PathBuf};
28use std::sync::Arc;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::{Mutex, MutexGuard};
31use std::time::Duration;
32
33use anyhow::{Context, Result, bail};
34use jiff::Timestamp;
35use serde::{Deserialize, Serialize};
36use tokio::sync::Notify;
37
38use crate::ask::{self, Questions};
39use crate::clean;
40use crate::conduct::Conductor;
41use crate::config::{Config, MergeMode};
42use crate::graph::Runner;
43use crate::land;
44use crate::notices::{self, Link, Notice};
45use crate::queue::{Queue, Task, TaskStatus};
46use crate::run::{Liveness, QuotaLoss, RunState, RunStatus};
47use crate::triage;
48
49pub const SCHEMA: u32 = 1;
51
52pub const HEARTBEAT: Duration = Duration::from_secs(5);
56
57pub const STALE_SECS: i64 = 30;
66
67pub const POLL: Duration = Duration::from_secs(5);
69
70pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
74
75pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
94
95#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(default)]
98pub struct Current {
99 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub worker: Option<String>,
102 pub task: String,
104 pub run: String,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct Status {
116 pub schema: u32,
118 pub pid: u32,
120 pub started_at: Timestamp,
122 pub updated_at: Timestamp,
124 pub idle: bool,
126 pub current: Vec<Current>,
132 pub completed: usize,
134 pub polls: u64,
136 #[serde(default)]
138 pub question_recovery: Option<crate::question_delivery::Recovery>,
139 #[serde(default)]
141 pub worker_protocol: Option<u32>,
142}
143
144impl Status {
145 #[must_use]
147 pub fn new() -> Self {
148 let now = Timestamp::now();
149 Self {
150 schema: SCHEMA,
151 pid: std::process::id(),
152 started_at: now,
153 updated_at: now,
154 idle: true,
155 current: Vec::new(),
156 completed: 0,
157 polls: 0,
158 question_recovery: None,
159 worker_protocol: Some(1),
160 }
161 }
162}
163
164impl Default for Status {
165 fn default() -> Self {
166 Self::new()
167 }
168}
169
170#[derive(Debug, Clone, Serialize, Deserialize)]
172#[serde(default)]
173pub struct Opts {
174 pub repo: PathBuf,
176 pub config: Option<PathBuf>,
178 pub poll: Duration,
180 pub max_attempts: usize,
182 pub once: bool,
184 pub merge: Option<String>,
186 pub worktrees_root: Option<PathBuf>,
195}
196
197impl Default for Opts {
198 fn default() -> Self {
199 Self {
200 repo: PathBuf::from("."),
201 config: None,
202 poll: POLL,
203 max_attempts: 2,
204 once: false,
205 merge: None,
206 worktrees_root: None,
207 }
208 }
209}
210
211impl Opts {
212 fn absolute(&self) -> Result<Self> {
213 let mut opts = self.clone();
214 opts.repo = std::path::absolute(&opts.repo)?;
215 opts.config = opts
216 .config
217 .as_deref()
218 .map(std::path::absolute)
219 .transpose()?;
220 opts.worktrees_root = opts
221 .worktrees_root
222 .as_deref()
223 .map(std::path::absolute)
224 .transpose()?;
225 Ok(opts)
226 }
227}
228
229fn max_concurrent(n: usize) -> usize {
234 n.max(1)
235}
236
237#[must_use]
239pub fn status_path() -> PathBuf {
240 crate::run::home().join("daemon.json")
241}
242
243pub fn write_status(status: &Status) -> Result<()> {
245 write_status_to(&status_path(), status)
246}
247
248pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
253 if let Some(parent) = path.parent() {
254 std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
255 }
256 let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
257 let tmp = path.with_extension("json.tmp");
258 std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
259 std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
260 Ok(())
261}
262
263pub fn clear_status() {
266 clear_status_at(&status_path());
267}
268
269fn clear_status_at(path: &Path) {
273 let _ = std::fs::remove_file(path);
274}
275
276#[derive(Debug, Clone, Default)]
288pub struct Stop {
289 stopped: Arc<AtomicBool>,
293 busy: Arc<std::sync::atomic::AtomicUsize>,
295 wake: Arc<Notify>,
299 pause: crate::graph::Pause,
302}
303
304impl Stop {
305 #[must_use]
307 pub fn new() -> Self {
308 Self::default()
309 }
310
311 pub fn stop(&self) {
314 self.stopped.store(true, Ordering::SeqCst);
315 self.wake.notify_one();
319 }
320
321 #[must_use]
323 pub fn stopped(&self) -> bool {
324 self.stopped.load(Ordering::SeqCst)
325 }
326
327 #[must_use]
330 pub fn finishing(&self) -> bool {
331 self.stopped() && self.busy_now()
332 }
333
334 pub fn park(&self, cause: crate::run::ParkCause) {
340 self.pause.park_for(cause);
341 self.stop();
342 }
343
344 #[must_use]
346 pub fn parking(&self) -> bool {
347 self.pause.parked()
348 }
349
350 #[must_use]
352 pub fn pause(&self) -> crate::graph::Pause {
353 self.pause.clone()
354 }
355
356 #[must_use]
358 pub fn busy_now(&self) -> bool {
359 self.busy.load(Ordering::SeqCst) > 0
360 }
361
362 async fn idle(&self, poll: Duration) {
364 tokio::select! {
365 () = tokio::time::sleep(poll) => {}
366 () = self.wake.notified() => {}
367 }
368 }
369}
370
371#[derive(Debug, Clone, Default, Deserialize)]
378#[serde(default)]
379pub struct Reading {
380 pub schema: u32,
382 pub pid: Option<u32>,
384 pub started_at: Option<Timestamp>,
386 pub updated_at: Option<Timestamp>,
388 pub idle: bool,
390 #[serde(deserialize_with = "de_current")]
402 pub current: Vec<Current>,
403 pub completed: u64,
405 pub polls: u64,
407 #[serde(default)]
409 pub question_recovery: Option<crate::question_delivery::Recovery>,
410 pub worker_protocol: Option<u32>,
412}
413
414fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
417where
418 D: serde::Deserializer<'de>,
419{
420 #[derive(Deserialize)]
421 #[serde(untagged)]
422 enum Shape {
423 Many(Vec<Current>),
424 One(Current),
425 }
426 Ok(
427 Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
428 Shape::Many(v) => v,
429 Shape::One(c) => vec![c],
430 }),
431 )
432}
433
434impl Reading {
435 #[must_use]
438 pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
439 self.updated_at
440 .map(|at| (now.as_second() - at.as_second()).max(0))
441 }
442
443 #[must_use]
447 pub fn running(&self, now: Timestamp) -> bool {
448 self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
449 }
450}
451
452#[must_use]
459pub fn read_status(home: &Path) -> Option<Reading> {
460 let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
461 serde_json::from_str(&body).ok()
462}
463
464#[must_use]
473pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
474 let mut current = read_status(home)
475 .filter(|reading| reading.running(now))
476 .map(|reading| reading.current)
477 .unwrap_or_default();
478 let workers = crate::worker::observe_active(home).and_then(|jobs| {
479 jobs.into_iter()
480 .filter(|job| job.kind == "task")
481 .map(|job| {
482 let request: TaskWorker = serde_json::from_value(job.payload)?;
483 let run = Queue::at(request.queue)
484 .get(&request.task)
485 .ok()
486 .and_then(|task| task.execution_runs().last().cloned())
487 .unwrap_or_default();
488 Ok(Current {
489 worker: Some(job.id),
490 task: request.task,
491 run,
492 })
493 })
494 .collect::<Result<Vec<_>>>()
495 });
496 match workers {
497 Ok(workers) => {
498 for worker in workers {
499 if !current.iter().any(|claim| {
500 claim.task == worker.task
501 && claim.run == worker.run
502 && claim.worker == worker.worker
503 }) {
504 current.push(worker);
505 }
506 }
507 }
508 Err(error) => {
509 tracing::error!("cannot determine active workers: {error:#}");
512 current.extend(
513 Queue::at(home.join("queue"))
514 .list()
515 .into_iter()
516 .map(|task| Current {
517 worker: None,
518 task: task.id.clone(),
519 run: task.execution_runs().last().cloned().unwrap_or_default(),
520 }),
521 );
522 current.extend(
523 crate::run::list_ids_in(&home.join("runs"))
524 .into_iter()
525 .map(|run| Current {
526 worker: None,
527 task: String::new(),
528 run,
529 }),
530 );
531 }
532 }
533 current
534}
535
536#[must_use]
538pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
539 current_work(home, now).iter().any(|c| c.run == run)
540}
541
542pub(crate) fn is_working_on_other(home: &Path, run: &str, now: Timestamp) -> Result<bool> {
547 let same_home = std::env::var_os("MAGI_WORKER_HOME")
550 .is_some_and(|worker_home| Path::new(&worker_home) == home);
551 let caller = if same_home {
552 crate::worker::current_job(home)?
553 } else {
554 None
555 }
556 .filter(|job| {
557 job.kind == "task"
558 && job.status == crate::worker::Status::Running
559 && job.pid == Some(std::process::id())
560 });
561 Ok(other_claims_run(
562 ¤t_work(home, now),
563 run,
564 caller.as_ref().map(|job| job.id.as_str()),
565 ))
566}
567
568fn other_claims_run(claims: &[Current], run: &str, caller: Option<&str>) -> bool {
569 claims
570 .iter()
571 .any(|claim| claim.run == run && !(caller.is_some() && claim.worker.as_deref() == caller))
572}
573
574#[must_use]
584pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
585 current_work(home, now)
586 .iter()
587 .any(|c| crate::run::short_of(&c.run) == short)
588}
589
590#[must_use]
592pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
593 current_work(home, now).iter().any(|c| c.task == task)
594}
595
596pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
636 sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
637}
638
639fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
643where
644 F: Fn(u32) -> bool,
645{
646 let this_process = std::process::id();
647 let mut swept: Vec<String> = std::fs::read_dir(queue.root())
648 .into_iter()
649 .flatten()
650 .flatten()
651 .map(|e| e.path())
652 .filter(|p| p.extension().is_some_and(|x| x == "lock"))
653 .filter(|p| {
654 match std::fs::read_to_string(p)
655 .ok()
656 .and_then(|body| body.trim().parse::<u32>().ok())
657 {
658 Some(pid) if pid == this_process => false,
662 Some(pid) => !pid_alive(pid),
663 None => p
664 .metadata()
665 .and_then(|m| m.modified())
666 .and_then(|t| t.elapsed().map_err(std::io::Error::other))
667 .is_ok_and(|age| age >= older_than),
668 }
669 })
670 .filter(|p| std::fs::remove_file(p).is_ok())
671 .filter_map(|p| {
672 p.file_stem()
673 .and_then(|s| s.to_str())
674 .map(std::borrow::ToOwned::to_owned)
675 })
676 .collect();
677 swept.sort_unstable();
678 swept
679}
680
681fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
686 task.status == TaskStatus::Running
687 && (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
688 && !is_working_on_task(home, &task.id, now)
689}
690
691fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
694 queue
695 .list()
696 .into_iter()
697 .filter(|t| is_stalled(t, home, now))
698 .collect()
699}
700
701fn queued_tasks(queue: &Queue) -> Vec<Task> {
707 queue
708 .list()
709 .into_iter()
710 .filter(|t| t.status == TaskStatus::Queued)
711 .collect()
712}
713
714fn finished_tasks(queue: &Queue) -> Vec<Task> {
717 queue
718 .list()
719 .into_iter()
720 .filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
721 .collect()
722}
723
724fn resolve_blockers(queue: &Queue, questions: &Questions) {
741 for listed in queue.list() {
742 if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
743 continue;
744 }
745 let Ok(_claim) = queue.claim(&listed.id) else {
746 continue;
747 };
748 let Ok(mut task) = queue.get(&listed.id) else {
749 continue;
750 };
751 if task.status != TaskStatus::Blocked {
752 continue;
753 }
754 let deleted = queue.apply_deleted_blockers(&mut task, questions);
758 if !deleted.is_empty() {
759 record(queue, &mut task);
760 for id in &deleted {
761 queue.note_dependency_deleted(&task, id);
762 }
763 if task.status != TaskStatus::Blocked {
764 continue;
765 }
766 }
767 let missing = crate::queue::missing_blockers(queue, questions, &task.blocked_by);
768 if !missing.is_empty() {
769 let language = language_of(&task, Path::new("."));
770 let mut cause = missing.clone();
771 cause.sort();
772 cause.dedup();
773 task.notice_cause(&format!("missing-blockers:{}", cause.join(",")), None, None);
774 task.hold_machine(Some(crate::queue::missing_blocker_hold_reason_in(
775 &task.blocked_by,
776 &missing,
777 &language,
778 )));
779 record(queue, &mut task);
780 continue;
781 }
782 if let Some(q) = task.blocked_by.iter().find_map(|id| {
785 let crate::queue::Dependency::Question(full) =
786 crate::queue::dependency(queue, questions, id)
787 else {
788 return None;
789 };
790 questions.get(&full).ok().filter(|q| {
791 q.node == crate::conduct::NODE && q.status == ask::QuestionStatus::Abandoned
792 })
793 }) {
794 let language = language_of(&task, Path::new("."));
795 task.notice_cause(&format!("unanswered-question:{}", q.id), None, None);
796 task.hold_machine(Some(unanswered_question_hold_reason(&q, &language)));
797 record(queue, &mut task);
798 continue;
799 }
800 let mut changed = false;
801 for id in task.blocked_by.clone() {
802 let resolved = crate::queue::dependency(queue, questions, &id);
803 if let crate::queue::Dependency::Task(full) = &resolved {
804 if queue
805 .get(full)
806 .is_ok_and(|dep| dep.status == TaskStatus::Done)
807 {
808 task.unblock(&id);
809 changed = true;
810 }
811 continue;
812 }
813 if let crate::queue::Dependency::Question(full) = resolved
814 && let Ok(q) = questions.get(&full)
815 && q.status == ask::QuestionStatus::Answered
816 {
817 if q.node == crate::conduct::NODE
821 && q.chosen_action().is_some()
822 && !task.action_applied(&q.id)
823 {
824 continue;
825 }
826 let answer = match &q.answer {
827 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
828 None => String::new(),
829 };
830 task.record_answer(q.summary.clone(), answer);
831 task.unblock(&id);
832 changed = true;
833 }
834 }
835 if changed {
836 record(queue, &mut task);
837 }
838 }
839}
840
841fn unanswered_question_hold_reason(q: &ask::Question, language: &str) -> String {
843 if crate::lang::is_japanese(language) {
844 format!(
845 "質問 {} 「{}」 に期限内の回答がなく、取り下げられました - `magi task triage` を参照",
846 q.short(),
847 q.summary
848 )
849 } else {
850 format!(
851 "question {} \"{}\" went unanswered and was abandoned - see `magi task triage`",
852 q.short(),
853 q.summary
854 )
855 }
856}
857
858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
861pub(crate) enum ActionStanding {
862 NoAction,
864 Pending,
866 Applied,
869 Stale,
871 Busy,
873}
874
875impl ActionStanding {
876 pub(crate) fn daemon_owns(self) -> bool {
878 matches!(self, Self::Pending | Self::Applied | Self::Stale)
879 }
880}
881
882pub(crate) fn action_standing(task: &Task, q: &ask::Question) -> ActionStanding {
886 if task.stale_question(q) {
887 ActionStanding::Stale
888 } else if q.chosen_action().is_none() {
889 ActionStanding::NoAction
890 } else if task.action_applied(&q.id) {
891 ActionStanding::Applied
892 } else if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
893 ActionStanding::Stale
894 } else if matches!(
895 task.status,
896 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
897 ) {
898 ActionStanding::Busy
899 } else {
900 ActionStanding::Pending
901 }
902}
903
904#[derive(Debug, Clone, PartialEq, Eq)]
906enum ActionDecision {
907 Skip,
909 Resume(String),
911 Requeue,
913 Hold,
915 Done,
917 Stale,
920 Refuse(String),
923}
924
925fn decide_action<F>(
934 home: &Path,
935 task: &Task,
936 q: &ask::Question,
937 p: &Phrases,
938 load: F,
939) -> ActionDecision
940where
941 F: FnOnce(&str) -> Result<RunState>,
942{
943 if task.stale_question(q) {
944 return ActionDecision::Stale;
945 }
946 let Some(action) = q.chosen_action() else {
947 return ActionDecision::Skip;
948 };
949 if task.action_applied(&q.id)
950 || matches!(
951 task.status,
952 TaskStatus::Running | TaskStatus::Blocked | TaskStatus::Done
953 )
954 {
955 return ActionDecision::Skip;
956 }
957 if q.node != crate::conduct::NODE && task.execution_runs().last() != Some(&q.run) {
960 return ActionDecision::Stale;
961 }
962 let repair = q.node == land::APPROVAL_NODE && q.resolution().as_deref() == Some(land::REPAIR);
963 let stopping = q.node == land::APPROVAL_NODE
964 && q.resolution()
965 .as_deref()
966 .is_some_and(crate::merge_decision::stops);
967 let decision_at = q.answered_at.unwrap_or(q.asked_at);
971 if task.operator_decided_at.is_some_and(|at| at > decision_at) {
972 return if repair || stopping {
973 ActionDecision::Refuse(format!(
974 "{} answer {} was superseded by a later operator decision; the task stays held. Inspect `magi show {}`; if this action is still wanted, explicitly release the existing owner with `magi task release {}`",
975 if repair { "repair" } else { "stop" },
976 q.id,
977 q.run,
978 task.id
979 ))
980 } else {
981 ActionDecision::Stale
982 };
983 }
984 match action {
985 a if a.maintenance() => ActionDecision::Skip,
986 ask::ChoiceAction::ResumeRun { .. }
987 | ask::ChoiceAction::ReleaseTask { .. }
988 | ask::ChoiceAction::RerunChecks { .. }
989 | ask::ChoiceAction::AbandonRun { .. } => unreachable!(),
990 ask::ChoiceAction::Requeue => ActionDecision::Requeue,
991 ask::ChoiceAction::Hold => ActionDecision::Hold,
992 ask::ChoiceAction::Done => ActionDecision::Done,
993 ask::ChoiceAction::Resume { run } => {
994 if task.execution_runs().last() != Some(run) {
995 return ActionDecision::Refuse((p.resume_not_latest)(
996 q.short(),
997 ask::short_id(run),
998 ));
999 }
1000 match load(run) {
1001 Ok(s) if stopping => {
1002 if s.merge_decision.as_ref().is_some_and(|d| d.closed)
1003 || !s
1004 .land_approval
1005 .as_ref()
1006 .is_some_and(|binding| binding.question == q.id)
1007 {
1008 ActionDecision::Stale
1009 } else if s.liveness_under(home, false) == Liveness::Live {
1010 ActionDecision::Skip
1011 } else if s.status == RunStatus::Landing
1012 && !s.released()
1013 && s.retired_to.is_none()
1014 {
1015 ActionDecision::Resume(run.clone())
1016 } else {
1017 ActionDecision::Refuse("Owner stop decision cannot resume its original landing execution; inspect the existing run. No replacement or merge was started.".into())
1018 }
1019 }
1020 Ok(s)
1021 if q.node == land::APPROVAL_NODE
1022 && q.resolution().as_deref() == Some(land::REPAIR) =>
1023 {
1024 if s.merge_repairs.iter().any(|r| r.question == q.id)
1025 || s.land_approval.as_ref().is_some_and(|a| a.question != q.id)
1026 {
1027 ActionDecision::Stale
1030 } else if s.liveness_under(home, false) == Liveness::Live {
1031 ActionDecision::Skip
1032 } else if s.status != RunStatus::Landing
1033 || s.released()
1034 || s.reviews.is_empty()
1035 || !implementers_match(task, &s)
1036 || !s
1037 .land_approval
1038 .as_ref()
1039 .is_some_and(|a| a.question == q.id && !a.head.is_empty())
1040 {
1041 ActionDecision::Refuse(format!(
1042 "repair question {} no longer names a supported current approval on run {}; inspect `magi show {}` and restore the existing owner's approval before retrying; do not file another task",
1043 q.id, run, run
1044 ))
1045 } else {
1046 ActionDecision::Resume(run.clone())
1047 }
1048 }
1049 Ok(s) if s.liveness_under(home, false) == Liveness::Live => {
1050 ActionDecision::Refuse("run already has a live driver".into())
1051 }
1052 Ok(s) => match crate::handover::resume_refusal_under(&s, Some(home)) {
1053 None => ActionDecision::Resume(run.clone()),
1054 Some(reason) => ActionDecision::Refuse((p.resume_cannot_progress)(
1055 q.short(),
1056 ask::short_id(run),
1057 &reason.diagnostic(&s),
1058 )),
1059 },
1060 Err(e) => ActionDecision::Refuse((p.resume_unreadable)(
1061 q.short(),
1062 ask::short_id(run),
1063 &format!("{e:#}"),
1064 )),
1065 }
1066 }
1067 }
1068}
1069
1070struct Phrases {
1084 graph_stopped: fn(&str, &str) -> String,
1086 quorum_lost: &'static str,
1087 quota_took_out: &'static str,
1089 run_ended: &'static str,
1091 waiting_for_answer: &'static str,
1093 recovered_running: &'static str,
1094 no_run_to_recover: &'static str,
1095 could_not_start: &'static str,
1096 resume_not_latest: fn(&str, &str) -> String,
1097 resume_cannot_progress: fn(&str, &str, &str) -> String,
1098 resume_unreadable: fn(&str, &str, &str) -> String,
1099 handover_refused: fn(&crate::handover::Refused) -> String,
1102 handover_hint: &'static str,
1104 reviewers_never_answered: fn(usize, usize) -> String,
1107}
1108
1109const PHRASES_EN: Phrases = Phrases {
1110 graph_stopped: |status, detail| {
1111 format!("the graph stopped at `{status}` without reaching a terminal status: {detail}")
1112 },
1113 quorum_lost: "the judging panel lost its quorum",
1114 quota_took_out: "; quota took out ",
1115 run_ended: "run ended ",
1116 waiting_for_answer: " - waiting for operator answer to question ",
1117 recovered_running: "recovered a `running` task whose daemon never recorded the outcome: ",
1118 no_run_to_recover: "task was `running` with no live daemon and no readable \
1119 run to recover; held for a human to check what happened",
1120 could_not_start: "could not start the run: ",
1121 resume_not_latest: |q, run| {
1122 format!("question {q} asked to resume run {run}, which is not this task's latest run")
1123 },
1124 resume_cannot_progress: |q, run, reason| {
1125 format!("question {q} asked to resume run {run}, which cannot make progress: {reason}")
1126 },
1127 resume_unreadable: |q, run, e| {
1128 format!("question {q} asked to resume run {run}, which could not be read: {e}")
1129 },
1130 handover_refused: |r| r.to_string(),
1131 handover_hint: " (clean up the other worktree, then release the task from the queue)",
1132 reviewers_never_answered: |missing, rounds| {
1133 format!(
1134 "{missing} reviewer seat(s) never answered after {rounds} rounds; \
1135 refusing to call it clean"
1136 )
1137 },
1138};
1139
1140const PHRASES_JA: Phrases = Phrases {
1141 graph_stopped: |status, detail| {
1142 format!("グラフが終端状態に達しないまま `{status}` で停止しました: {detail}")
1143 },
1144 quorum_lost: "審査パネルが定足数を失いました",
1145 quota_took_out: "。クォータで脱落: ",
1146 run_ended: "run 終了: ",
1147 waiting_for_answer: " - オペレーターの回答待ち: 質問 ",
1148 recovered_running: "daemon が結果を記録しないまま `running` だったタスクを回収しました: ",
1149 no_run_to_recover: "タスクは `running` でしたが、生きた daemon も回収できる run も見つかりません。\
1150 何が起きたか人が確認するため保留にしました",
1151 could_not_start: "run を開始できませんでした: ",
1152 resume_not_latest: |q, run| {
1153 format!(
1154 "質問 {q} は run {run} の再開を求めましたが、これはタスクの最新の run ではありません"
1155 )
1156 },
1157 resume_cannot_progress: |q, run, reason| {
1158 format!("質問 {q} は run {run} の再開を求めましたが、これは進行できません: {reason}")
1159 },
1160 resume_unreadable: |q, run, e| {
1161 format!("質問 {q} は run {run} の再開を求めましたが、読み込めませんでした: {e}")
1162 },
1163 handover_refused: |r| {
1164 use crate::handover::Refused;
1165 match r {
1166 Refused::Foreign { branch, path, why } => format!(
1167 "ブランチ `{branch}` は {path} にチェックアウトされており、magi は自動では\
1168 削除しません。不要なら `git worktree remove` でその worktree を削除して\
1169 から、やり直してください(詳細: {why})"
1170 ),
1171 Refused::Unsafe { branch, path, why } => format!(
1172 "ブランチ `{branch}` は {path} にチェックアウトされています。そこの作業を\
1173 コミットか破棄したうえで `git worktree remove` で worktree を削除する\
1174 か、run を破棄してよいと伝えてから、やり直してください(詳細: {why})"
1175 ),
1176 Refused::ReleaseFailed { branch, path, run } => format!(
1177 "ブランチ `{branch}` は {path} で run {run} がチェックアウトしており、その\
1178 worktree の解放に失敗したか、変更が見つかりました(未コミットの変更が\
1179 ある worktree は git が削除を拒否します)。worktree はそのまま残しました"
1180 ),
1181 }
1182 },
1183 handover_hint: "(他の worktree を片付けてから、タスクをキューから解放してください)",
1184 reviewers_never_answered: |missing, rounds| {
1185 format!(
1186 "{missing} 席のレビュアーが {rounds} ラウンドの間に一度も回答しなかったため、\
1187 クリーンとは認めません"
1188 )
1189 },
1190};
1191
1192fn phrases(language: &str) -> &'static Phrases {
1193 if crate::lang::is_japanese(language) {
1194 &PHRASES_JA
1195 } else {
1196 &PHRASES_EN
1197 }
1198}
1199
1200fn language_of(task: &Task, fallback: &Path) -> String {
1204 crate::lang::of_repo(&repo_for(task, fallback))
1205}
1206
1207pub(crate) fn task_of_question<'a>(tasks: &'a [Task], q: &ask::Question) -> Option<&'a Task> {
1211 if q.node == crate::conduct::NODE {
1212 return tasks.iter().find(|t| t.id == q.run);
1213 }
1214 tasks.iter().find(|t| t.runs.contains(&q.run))
1215}
1216
1217fn repair_choice_offers(questions: &Questions) {
1220 for q in questions.list().into_iter().filter(|q| {
1221 q.status.open()
1222 && q.node == crate::conduct::NODE
1223 && q.choices
1224 .iter()
1225 .any(|c| crate::choice_effect::describe(q, c).is_none())
1226 }) {
1227 let _ = questions.update(&q.id, |current| {
1228 if current.status.open() {
1229 crate::choice_effect::executable_conductor(current);
1230 if let Some(deputy) = &mut current.deputy {
1231 deputy.brief = crate::deputy::brief(
1232 ¤t.run,
1233 ¤t.detail,
1234 ¤t.choices,
1235 ¤t.actions,
1236 );
1237 }
1238 }
1239 Ok(())
1240 });
1241 }
1242}
1243
1244pub async fn apply_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1247 apply_maintenance_choices(queue, questions, home).await;
1248 apply_task_choice_actions(queue, questions, home);
1249}
1250
1251async fn apply_maintenance_choices(queue: &Queue, questions: &Questions, home: &Path) {
1254 for q in questions.list() {
1255 let Some(action) = q.chosen_action().filter(|a| a.maintenance()) else {
1256 continue;
1257 };
1258 if asker_may_still_read(questions, &q, home) || !ask::valid_asset_name(&q.id) {
1261 continue;
1262 }
1263 if queue.list().iter().any(|task| {
1264 (task.id == q.run
1265 || task.runs.contains(&q.run)
1266 || q.routed_task.as_deref() == Some(&task.id))
1267 && task.stale_question(&q)
1268 }) {
1269 continue;
1270 }
1271 let dir = home.join("choice-actions");
1272 if std::fs::create_dir_all(&dir).is_err() {
1273 continue;
1274 }
1275 let Ok(lock) = std::fs::OpenOptions::new()
1276 .create(true)
1277 .truncate(false)
1278 .write(true)
1279 .open(dir.join(format!("{}.lock", q.id)))
1280 else {
1281 continue;
1282 };
1283 if crate::pr_head::try_lock_file(&lock).is_err() {
1284 continue;
1285 }
1286 let fresh = !home
1287 .join("choice-actions")
1288 .join(format!("{}.json", q.id))
1289 .exists();
1290 let result = async {
1291 let receipt = dir.join(format!("{}.json", q.id));
1292 if receipt.exists() {
1293 let prior: serde_json::Value = serde_json::from_slice(&std::fs::read(&receipt)?)?;
1294 anyhow::ensure!(
1295 prior.get("status").and_then(|s| s.as_str()) == Some("applied"),
1296 "previous maintenance attempt did not confirm completion; inspect its receipt"
1297 );
1298 return Ok::<_, anyhow::Error>(());
1299 }
1300 let record = |status: &str| -> anyhow::Result<()> {
1303 let tmp = receipt.with_extension("tmp");
1304 std::fs::write(
1305 &tmp,
1306 serde_json::to_vec(&serde_json::json!({
1307 "action": action, "status": status,
1308 }))?,
1309 )?;
1310 std::fs::rename(tmp, &receipt)?;
1311 Ok(())
1312 };
1313 let applied = async {
1314 match action {
1315 ask::ChoiceAction::ResumeRun { run } => {
1316 crate::resume::validate(home, run)?;
1317 record("started")?;
1318 let decided = q.answered_at.unwrap_or(q.asked_at);
1319 if !crate::resume::pin_queued(queue, run, Some(decided), Some(&q.id))? {
1320 crate::resume::validate_unqueued(home, run)?;
1321 crate::resume::start(home, run.clone()).await?;
1322 }
1323 }
1324 ask::ChoiceAction::ReleaseTask {
1325 task,
1326 held_at,
1327 reason,
1328 } => {
1329 anyhow::ensure!(
1330 queue.resolve_id(task)? == *task,
1331 "task action requires an exact task ID"
1332 );
1333 let _claim = queue.claim(task)?;
1334 let mut task = queue.get(task)?;
1335 if task.action_applied(&q.id) {
1336 return Ok(());
1337 }
1338 if q.node == crate::conduct::NODE
1343 && q.run == task.id
1344 && task.status == TaskStatus::Blocked
1345 && task.blocked_from == Some(TaskStatus::Held)
1346 && task.blocked_by.as_slice() == [q.id.as_str()]
1347 {
1348 task.unblock(&q.id);
1349 }
1350 anyhow::ensure!(
1351 task.status == TaskStatus::Held
1352 && task.held_at == Some(*held_at)
1353 && task.hold_reason.as_deref() == reason.as_deref()
1354 && task
1355 .operator_decided_at
1356 .is_none_or(|at| at <= q.answered_at.unwrap_or(q.asked_at)),
1357 "task no longer has the hold this question names"
1358 );
1359 task.release_operator("choice", Some(&q.id));
1360 task.mark_action_applied(&q.id);
1361 queue.put(&mut task)?;
1362 }
1363 ask::ChoiceAction::AbandonRun { run } => {
1364 record("started")?;
1365 crate::driver::abandon_proven_dead(home, run)?;
1366 }
1367 ask::ChoiceAction::RerunChecks {
1368 repo,
1369 pr,
1370 head,
1371 runs,
1372 } => {
1373 rerun_checks_once(home, &q.id, repo, pr, head, runs, || record("started"))
1374 .await?;
1375 }
1376 _ => unreachable!(),
1377 }
1378 Ok::<_, anyhow::Error>(())
1379 }
1380 .await;
1381 match applied {
1382 Ok(()) => record("applied"),
1383 Err(e) => {
1384 record(&format!("refused or failed: {e:#}"))?;
1385 Err(e)
1386 }
1387 }
1388 }
1389 .await;
1390 if home
1395 .join("choice-actions")
1396 .join(format!("{}.json", q.id))
1397 .exists()
1398 && let Some(listed) = task_of_question(&queue.list(), &q)
1399 && !listed.action_applied(&q.id)
1400 && let Ok(_claim) = queue.claim(&listed.id)
1401 && let Ok(mut origin) = queue.get(&listed.id)
1402 {
1403 if !origin.operator_held()
1404 && origin.operator_decided_at.is_none_or(|at| at <= q.asked_at)
1405 && !matches!(origin.status, TaskStatus::Running | TaskStatus::Done)
1406 {
1407 if result.is_err() || matches!(action, ask::ChoiceAction::ResumeRun { .. }) {
1408 origin.hold_machine(Some(format!(
1409 "Maintenance choice {} settled; inspect its target before releasing this task",
1410 q.id
1411 )));
1412 } else {
1413 origin.unblock(&q.id);
1414 }
1415 }
1416 origin.mark_action_applied(&q.id);
1417 let _ = queue.put(&mut origin);
1418 }
1419 if let Err(e) = result
1420 && fresh
1421 {
1422 notices::raise_in(
1423 home,
1424 Notice::warn(
1425 &format!("action:{}", q.id),
1426 format!("Choice action {} was not completed: {e:#}", q.id),
1427 )
1428 .link(if q.run_names_task() {
1429 Link::Task { id: q.run.clone() }
1430 } else {
1431 Link::Run { id: q.run.clone() }
1432 }),
1433 );
1434 }
1435 }
1436}
1437
1438pub(crate) async fn rerun_checks_once(
1445 home: &Path,
1446 request: &str,
1447 repo: &Path,
1448 pr: &str,
1449 head: &str,
1450 runs: &[String],
1451 started: impl FnOnce() -> anyhow::Result<()>,
1452) -> anyhow::Result<()> {
1453 use crate::release_watch::{GhForge, ReleaseForge};
1454 anyhow::ensure!(
1455 !runs.is_empty()
1456 && runs.len() <= 20
1457 && runs
1458 .iter()
1459 .all(|r| !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit()))
1460 && !pr.starts_with('-'),
1461 "invalid rerun targets"
1462 );
1463 let snap = GhForge.snapshot(repo, pr).await?;
1464 anyhow::ensure!(
1465 snap.head == head && snap.state == crate::land::PrLifecycle::Open,
1466 "pull request head or lifecycle changed"
1467 );
1468 anyhow::ensure!(
1469 runs.iter().all(|r| snap
1470 .checks
1471 .iter()
1472 .any(|c| c.run.as_ref() == Some(r) && crate::release_watch::is_failed(c.verdict))),
1473 "named workflow is no longer failed on this pull request"
1474 );
1475 started()?;
1476 let dir = home.join("choice-actions");
1477 std::fs::create_dir_all(&dir)?;
1478 let rerun_lock = std::fs::OpenOptions::new()
1479 .create(true)
1480 .truncate(false)
1481 .write(true)
1482 .open(dir.join("reruns.lock"))?;
1483 crate::pr_head::try_lock_file(&rerun_lock)?;
1484 let ledger = dir.join("reruns.json");
1485 let mut recorded: std::collections::BTreeSet<String> = if ledger.exists() {
1486 serde_json::from_slice(&std::fs::read(&ledger)?)?
1487 } else {
1488 Default::default()
1489 };
1490 for run in runs {
1491 let key = serde_json::to_string(&(request, repo, pr, head, run))?;
1492 if !recorded.insert(key) {
1493 continue;
1494 }
1495 let tmp = ledger.with_extension("tmp");
1496 std::fs::write(&tmp, serde_json::to_vec(&recorded)?)?;
1497 std::fs::rename(tmp, &ledger)?;
1498 GhForge.rerun(repo, run).await?;
1499 }
1500 Ok(())
1501}
1502
1503pub(crate) fn apply_task_choice_actions(queue: &Queue, questions: &Questions, home: &Path) {
1512 let tasks = queue.list();
1513 for q in questions.list() {
1514 if q.resolution().as_deref() == Some(crate::merge_decision::DISMISS) {
1516 continue;
1517 }
1518 if q.chosen_action().is_none_or(|a| a.maintenance()) {
1519 continue;
1520 }
1521 let Some(listed) = task_of_question(&tasks, &q) else {
1522 if q.node == land::APPROVAL_NODE
1523 && q.resolution().as_deref().is_some_and(|choice| {
1524 choice == land::REPAIR || crate::merge_decision::stops(choice)
1525 })
1526 {
1527 notices::raise_in(
1528 home,
1529 Notice::warn(
1530 &format!("repair-owner:{}", q.id),
1531 format!(
1532 "Approval action is waiting: no task owns run {}. Inspect `magi show {}` and restore its original queue owner; do not file a competing task.",
1533 q.run, q.run
1534 ),
1535 ).link(Link::Run { id: q.run.clone() }),
1536 );
1537 }
1538 continue;
1539 };
1540 if listed.action_applied(&q.id) {
1541 continue;
1542 }
1543 let Ok(_claim) = queue.claim(&listed.id) else {
1544 continue;
1545 };
1546 let Ok(mut task) = queue.get(&listed.id) else {
1547 continue;
1548 };
1549 if q.node == crate::conduct::NODE && task.status == TaskStatus::Blocked {
1552 let blockers: Vec<_> = task
1553 .blocked_by
1554 .iter()
1555 .filter(|id| {
1556 crate::queue::dependency(queue, questions, id)
1557 == crate::queue::Dependency::Question(q.id.clone())
1558 })
1559 .cloned()
1560 .collect();
1561 if !blockers.is_empty() {
1562 for id in blockers {
1563 task.unblock(&id);
1564 }
1565 let answer = match &q.answer {
1566 Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
1567 None => String::new(),
1568 };
1569 task.record_answer(q.summary.clone(), answer);
1570 }
1571 }
1572 let language = language_of(&task, Path::new("."));
1573 let decision = decide_action(home, &task, &q, phrases(&language), |id| {
1574 RunState::load_under(id, home)
1575 });
1576 let ran = matches!(
1577 decision,
1578 ActionDecision::Resume(_)
1579 | ActionDecision::Requeue
1580 | ActionDecision::Hold
1581 | ActionDecision::Done
1582 );
1583 if ran && asker_may_still_read(questions, &q, home) {
1584 continue;
1593 }
1594 let done = matches!(decision, ActionDecision::Done);
1595 match decision {
1596 ActionDecision::Skip => continue,
1597 ActionDecision::Resume(run) => {
1598 task.release_operator("choice", Some(&q.id));
1599 task.resume_override = Some(crate::queue::OperatorResume {
1600 question_id: q.id.clone(),
1601 at: Timestamp::now(),
1602 conductor_rehold: None,
1603 forced: true,
1604 pinned_run: Some(run),
1605 });
1606 }
1607 ActionDecision::Stale => {}
1608 ActionDecision::Requeue => {
1609 task.release_operator("choice-requeue", Some(&q.id));
1610 task.requeue();
1611 }
1612 ActionDecision::Hold => task.hold_manual(Some(q.summary.clone())),
1613 ActionDecision::Done => task.succeed(),
1614 ActionDecision::Refuse(why) => {
1615 if task.operator_held() {
1616 task.hold_reason = Some(match task.hold_reason.take() {
1619 Some(reason) => format!("{reason}\n\n{why}"),
1620 None => why,
1621 });
1622 } else {
1623 task.notice_cause(&format!("resume-refused:{}", q.id), None, None);
1624 task.hold_machine(Some(why));
1625 }
1626 notices::raise(
1627 Notice::warn(
1628 &format!("action:{}", q.id),
1629 "An answer asked the daemon to resume a run that cannot be resumed; the task stays held.",
1630 )
1631 .link(Link::Task {
1632 id: task.id.clone(),
1633 }),
1634 );
1635 }
1636 }
1637 task.mark_action_applied(&q.id);
1638 record(queue, &mut task);
1641 if done && queue.get(&task.id).is_ok_and(|t| t.action_applied(&q.id)) {
1642 supersede_prior_runs(&task, home);
1643 }
1644 let _ = questions.update(&q.id, |r| {
1647 r.answer_delivered = true;
1648 Ok(())
1649 });
1650 }
1651}
1652
1653fn asker_may_still_read(questions: &Questions, q: &ask::Question, home: &Path) -> bool {
1657 let now = Timestamp::now();
1658 if questions.read_lease(&q.id).is_some_and(|l| l.fresh(now)) {
1659 return true;
1660 }
1661 q.node != crate::conduct::NODE
1662 && RunState::load_under(&q.run, home).is_ok_and(|s| {
1663 s.seats_active()
1664 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
1665 })
1666}
1667
1668fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
1679 let tasks = queue.list();
1680 let by_id: std::collections::BTreeMap<&str, &Task> =
1681 tasks.iter().map(|t| (t.id.as_str(), t)).collect();
1682 let referenced: Vec<&str> = tasks
1683 .iter()
1684 .flat_map(|task| task.blocked_by.iter().map(String::as_str))
1685 .collect();
1686
1687 for mut question in questions.list() {
1688 if !question.status.open() || question.node != crate::conduct::NODE {
1689 continue;
1690 }
1691 if referenced
1696 .iter()
1697 .any(|id| question.id.starts_with(id) || question.id.ends_with(id))
1698 {
1699 continue;
1700 }
1701 let Some(task) = by_id.get(question.run.as_str()) else {
1702 continue;
1703 };
1704 question.abandon(format!(
1705 "task {} no longer waits for this answer",
1706 task.short()
1707 ));
1708 if let Err(e) = questions.put(&mut question) {
1709 tracing::warn!(
1710 "could not retire question {} for task {}: {e:#}",
1711 question.short(),
1712 task.short()
1713 );
1714 }
1715 }
1716}
1717
1718#[derive(Debug, Clone, Copy)]
1725pub struct Verdict {
1726 pub status: RunStatus,
1728 pub left_pr: bool,
1730 pub quota_hit: bool,
1732 pub parked: bool,
1734 pub no_viable_candidates: bool,
1741}
1742
1743pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
1802 settle_in(task, verdict, detail, max_attempts, &PHRASES_EN)
1803}
1804
1805fn settle_in(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize, p: &Phrases) {
1807 if verdict.parked {
1813 task.park(detail);
1814 return;
1815 }
1816 match verdict.status {
1817 RunStatus::Merged | RunStatus::Ready
1818 if task.pr_recovery.as_ref().is_some_and(|s| {
1819 s.disposition != Some(crate::pr_recovery::Disposition::Review)
1820 }) =>
1821 {
1822 task.hold_manual(Some("Public delivery is not confirmed. Inspect the selected PR and run, then use recover-pr --disposition land with the intended source and validation configuration.".into()));
1823 }
1824 RunStatus::Merged | RunStatus::Ready => task.succeed(),
1825 RunStatus::AlreadyInBase => task.already_landed(detail),
1826 RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
1827 RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
1828 task.stall(detail)
1829 }
1830 RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
1831 RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
1832 RunStatus::Blocked => task.fail(detail, max_attempts),
1833 RunStatus::VerifiedNoop => task.handed_off(detail),
1834 other => task.fail((p.graph_stopped)(label(other), detail), max_attempts),
1835 }
1836}
1837
1838pub fn supersede_prior_runs(task: &Task, home: &Path) {
1887 let now = Timestamp::now();
1888 let last_run_succeeded = task
1895 .runs
1896 .last()
1897 .and_then(|id| RunState::load_under(id, home).ok())
1898 .is_some_and(|s| matches!(s.status, RunStatus::Merged | RunStatus::Ready));
1899 for id in task.superseded_attempts(last_run_succeeded) {
1900 let mut state = match RunState::load_under(id, home) {
1901 Ok(s) => s,
1902 Err(e) => {
1903 tracing::warn!("could not load run {id} to mark it superseded: {e:#}");
1904 continue;
1905 }
1906 };
1907 if !matches!(state.status, RunStatus::Blocked | RunStatus::Stalled) {
1908 continue;
1909 }
1910 let daemon_claims = is_working_on(home, id, now);
1911 if state.liveness(daemon_claims) == Liveness::Live {
1912 continue;
1913 }
1914 state.status = RunStatus::Superseded;
1915 if let Err(e) = state.save_under(home) {
1916 tracing::warn!("could not mark run {id} superseded: {e:#}");
1917 }
1918 }
1919}
1920
1921fn resweep_superseded_attempts(queue: &Queue, home: &Path) {
1943 for task in queue.list() {
1944 if task.status != TaskStatus::Done || task.runs.len() < 2 {
1945 continue;
1946 }
1947 supersede_prior_runs(&task, home);
1948 }
1949}
1950
1951fn settle_and_diagnose(
1958 task: &mut Task,
1959 verdict: Verdict,
1960 detail: &str,
1961 max_attempts: usize,
1962 state: &RunState,
1963) {
1964 let p = phrases(&state.config.graph.language);
1965 if let Some(d) = &state.merge_decision {
1968 use crate::merge_decision as md;
1969 if task
1970 .execution_runs()
1971 .last()
1972 .is_some_and(|id| id != &state.id)
1973 {
1974 return;
1975 }
1976 if (d.choice == md::WITHDRAW && d.closed) || d.choice == md::DISMISS {
1977 task.succeed();
1978 } else if let Some(agent) = md::selected_agent(&d.choice).filter(|_| d.closed) {
1979 if state.liveness(false) != Liveness::Dead {
1980 task.hold_manual(Some("Previous driver is still live; replacement held until it exits. Release the task after checking the driver.".into()));
1981 return;
1982 }
1983 let replacement = md::Replacement {
1984 question: d.question.clone(),
1985 run: state.id.clone(),
1986 pr: d.target.pr.clone(),
1987 agent: agent.to_owned(),
1988 context: format!(
1989 "Previous PR: {}. Previous run: {}. Owner requested a new implementation.\nReview evidence:\n{}",
1990 d.target.pr,
1991 state.id,
1992 serde_json::to_string(&state.reviews).unwrap_or_default()
1993 ),
1994 };
1995 task.requeue();
1996 task.pr_recovery = None;
1997 task.review_of = None;
1998 task.replacement = Some(replacement);
1999 } else {
2000 task.hold_manual(Some(if d.choice.eq_ignore_ascii_case(land::HOLD) {
2001 format!("Decision held by owner; no further repair or merge work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str()))
2002 } else { format!("PR closure was not confirmed. Task held; inspect the existing run before starting replacement work. {}", state.merge.as_ref().map_or("", |m| m.detail.as_str())) }));
2003 }
2004 task.mark_action_applied(&d.question);
2005 if let Some(pass) = state.passes.last() {
2006 task.record_pass_at(
2007 &state.id,
2008 pass.index,
2009 Some(pass.started_at),
2010 task.attempts,
2011 None,
2012 );
2013 }
2014 return;
2015 }
2016 if let Some(replacement) = &task.replacement
2017 && !state.parked
2018 && state.resource_block.is_none()
2019 && (state.viable().is_empty()
2020 || (state.status == RunStatus::Blocked && state.tally.is_none()))
2021 {
2022 task.hold_manual(Some(format!("Selected implementer `{}` was unavailable, failed or hit quota before producing a viable implementation. No other implementer was dispatched. Inspect the run, restore availability, then explicitly release this task.", replacement.agent)));
2023 return;
2024 }
2025 if !verdict.parked && verdict.quota_hit && state.status == RunStatus::Stalled {
2026 let mut agents: Vec<_> = state
2027 .quota
2028 .iter()
2029 .filter(|q| {
2030 q.cause == crate::agent::QuotaCause::Authentication
2031 && state
2032 .passes
2033 .last()
2034 .is_none_or(|pass| q.at >= pass.started_at)
2035 })
2036 .map(|q| q.agent.as_str())
2037 .collect();
2038 agents.sort_unstable();
2039 agents.dedup();
2040 if !agents.is_empty() {
2041 task.stall(detail);
2042 task.notice_cause("agent-authentication", None, None);
2043 task.hold_machine(Some(format!("[authentication] Agents {} cannot authenticate; re-authenticate, then resume the held task. No attempt was spent.", agents.join(", "))));
2044 return;
2045 }
2046 }
2047 let attempts_before = task.attempts;
2048 if let Some(block) = state.resource_block.as_ref().filter(|_| !verdict.parked) {
2055 task.notice_cause(
2056 &format!("resource:{}:{}", block.kind, block.node),
2057 None,
2058 None,
2059 );
2060 task.last_error = Some(detail.to_owned());
2061 task.diagnostic = None;
2062 task.attempts = task.attempts.saturating_sub(1);
2063 if block.kind == "agent authentication" {
2064 task.hold_machine(Some(format!(
2065 "[authentication] {}; no attempt was spent",
2066 block.detail
2067 )));
2068 } else if block.kind == "agent usage limit" {
2069 if let Some(agent) = &state.config.graph.selected_implementer {
2071 task.hold_machine(Some(format!("[agent usage limit] Selected implementer `{agent}` is unavailable. No other implementer was dispatched; resume after usage is available. No attempt was spent.")));
2072 } else {
2073 task.status = TaskStatus::Failed;
2074 task.last_error = Some(block.detail.clone());
2075 }
2076 } else if block.kind == "disk pressure" {
2077 task.hold_machine(Some(format!(
2078 "{}{}",
2079 crate::disk::RUNNING_PRESSURE,
2080 block.detail
2081 )));
2082 } else {
2083 task.hold_machine(Some(format!(
2084 "{}{}; no attempt was spent; resume to retry the affected operation",
2085 crate::cache::RESOURCE_HOLD,
2086 block.description()
2087 )));
2088 }
2089 if let Some(pass) = state.passes.last() {
2090 task.record_pass_at(
2091 &state.id,
2092 pass.index,
2093 Some(pass.started_at),
2094 attempts_before,
2095 None,
2096 );
2097 }
2098 return;
2099 }
2100 if let Some(hold) = state.land_unrelated.as_ref().filter(|hold| {
2101 state.parked
2102 && state.status == RunStatus::Landing
2103 && state
2104 .pr_pushed_head
2105 .as_ref()
2106 .is_none_or(|h| h == &hold.head)
2107 && task.execution_runs().last() == Some(&state.id)
2108 }) {
2109 let pass = state.passes.last().map_or(0, |p| p.index);
2112 if !task
2113 .passes
2114 .iter()
2115 .any(|p| p.run == state.id && p.pass_index == pass)
2116 {
2117 task.attempts = task.attempts.saturating_sub(1);
2118 }
2119 task.hold_machine(Some(format!(
2120 "[ci-unrelated] CI failure reproduced on base; owner question {}",
2121 hold.question
2122 )));
2123 task.last_error = Some(detail.to_owned());
2124 task.diagnostic = None;
2125 task.record_pass_at(
2126 &state.id,
2127 pass,
2128 state.passes.last().map(|p| p.started_at),
2129 attempts_before,
2130 Some(crate::run::ParkCause::Question),
2131 );
2132 return;
2133 }
2134 let unresolved = task.hold_notice_cause.clone();
2135 if state.base_sync_blocked() {
2136 task.hold_machine(Some(detail.to_owned()));
2138 task.last_error = Some(detail.to_owned());
2139 } else if !verdict.parked
2140 && task
2141 .pr_recovery
2142 .as_ref()
2143 .is_some_and(|s| s.disposition != Some(crate::pr_recovery::Disposition::Review))
2144 && matches!(
2145 verdict.status,
2146 RunStatus::Ready | RunStatus::Merged | RunStatus::AlreadyInBase
2147 )
2148 {
2149 if state.pr_recovery == task.pr_recovery && state.public_delivery_complete() {
2150 task.succeed();
2151 } else {
2152 task.hold_manual(Some("Selected PR delivery is incomplete: local review or publication is not a confirmed public merge. Inspect the run and PR; use recover-pr --disposition land with --source-head <reviewed-full-SHA> and the required verification configuration. No automatic new attempt will start.".into()));
2153 }
2154 } else if !verdict.parked
2155 && task.pr_recovery.is_some()
2156 && matches!(
2157 verdict.status,
2158 RunStatus::Blocked | RunStatus::Failed | RunStatus::Gating
2159 )
2160 && !verdict.quota_hit
2161 {
2162 task.hold_unrequested_retry(Some(format!("Selected recovery stopped: {detail}. Inspect the retained source and run; correct the cause before explicitly releasing or selecting again. No automatic replacement will start.")));
2163 } else {
2164 settle_in(task, verdict, detail, max_attempts, p);
2165 }
2166 if !verdict.parked
2167 && !verdict.quota_hit
2168 && matches!(
2169 verdict.status,
2170 RunStatus::Blocked | RunStatus::Failed | RunStatus::Stalled
2171 )
2172 {
2173 task.recovery_failed(state);
2174 }
2175 if state.prep_failure.is_some() {
2176 task.hold_notice_cause = unresolved;
2179 task.startup_notice_cause("startup-candidates");
2180 }
2181 if let Some(pass) = state.passes.last() {
2184 let parked = match &pass.end {
2185 Some(crate::run::PassEnd::Parked { cause }) => Some(cause.clone()),
2186 _ => None,
2187 };
2188 task.record_pass_at(
2189 &state.id,
2190 pass.index,
2191 Some(pass.started_at),
2192 attempts_before,
2193 parked,
2194 );
2195 }
2196 if task.status == TaskStatus::Held {
2197 if state.base_sync_blocked() {
2198 task.base_sync_notice(state);
2199 } else if state.prep_failure.is_none() {
2200 task.notice_cause(
2201 label(verdict.status),
2202 Some(&state.id),
2203 state.passes.last().map(|p| p.index),
2204 );
2205 }
2206 task.diagnostic = diagnostic(state);
2207 note_open_question(task, &state.id, p);
2208 }
2209}
2210
2211fn note_open_question(task: &mut Task, run: &str, p: &Phrases) {
2226 let Some(home) = crate::run::try_home() else {
2227 return;
2228 };
2229 let open = Questions::at(home.join("questions")).open_for(run);
2230 let Some(q) = open.first() else {
2231 return;
2232 };
2233 let mut decisions: Vec<_> = open.iter().map(|q| q.id.as_str()).collect();
2234 decisions.sort_unstable();
2235 task.notice_cause(
2236 &format!("questions:{}", decisions.join(",")),
2237 Some(run),
2238 None,
2239 );
2240 let base = task.hold_reason.clone().unwrap_or_default();
2241 task.hold_reason = Some(format!("{base}{}{}", p.waiting_for_answer, q.short()));
2242}
2243
2244fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize, language: &str) {
2255 match last_run {
2256 Some(state) => {
2257 let verdict = Verdict {
2258 status: state.status,
2259 left_pr: state.pr.is_some(),
2260 quota_hit: state.quota.iter().any(|loss| {
2261 state
2262 .passes
2263 .last()
2264 .is_none_or(|pass| loss.at >= pass.started_at)
2265 }),
2266 parked: state.parked,
2267 no_viable_candidates: state.viable().is_empty(),
2268 };
2269 let detail = format!(
2270 "{}{}",
2271 phrases(&state.config.graph.language).recovered_running,
2272 describe(&state)
2273 );
2274 settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
2275 }
2276 None => {
2277 let why = phrases(language).no_run_to_recover;
2280 task.last_error = Some(why.to_owned());
2281 task.notice_cause("missing-run", None, None);
2284 task.hold_machine(Some(why.to_owned()));
2285 }
2286 }
2287}
2288
2289fn reclaim_orphaned_running(
2311 queue: &Queue,
2312 max_attempts: usize,
2313 live_workers: &std::collections::BTreeSet<String>,
2314) -> Vec<String> {
2315 let mut reclaimed = Vec::new();
2316 for listed in queue.list() {
2317 if listed.status != TaskStatus::Running || live_workers.contains(&listed.id) {
2318 continue;
2319 }
2320 let Ok(_claim) = queue.claim(&listed.id) else {
2321 continue;
2322 };
2323 let Ok(mut task) = queue.get(&listed.id) else {
2327 continue;
2328 };
2329 if task.status != TaskStatus::Running {
2330 continue;
2331 }
2332 let last_run = task
2333 .execution_runs()
2334 .last()
2335 .and_then(|id| RunState::load(id).ok());
2336 if let Some(state) = &last_run
2346 && let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
2347 {
2348 tracing::warn!("abandon questions for {}: {e:#}", state.id);
2349 }
2350 let language = if last_run.is_none() {
2351 language_of(&task, Path::new("."))
2352 } else {
2353 String::new()
2354 };
2355 reclaim(&mut task, last_run, max_attempts, &language);
2356 if task.status == TaskStatus::Done {
2357 supersede_prior_runs(&task, &crate::run::home());
2358 }
2359 record(queue, &mut task);
2360 reclaimed.push(task.id.clone());
2361 }
2362 reclaimed
2363}
2364
2365fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
2397 reclaim_abandoned_runs_with(
2398 home,
2399 now,
2400 crate::proc::pid_status,
2401 crate::proc::process_started_at,
2402 )
2403}
2404
2405fn reclaim_abandoned_runs_with<F, G>(
2411 home: &Path,
2412 now: Timestamp,
2413 query: F,
2414 identity: G,
2415) -> Vec<String>
2416where
2417 F: Fn(u32) -> Option<bool>,
2418 G: Fn(u32) -> Option<String>,
2419{
2420 let mut abandoned = Vec::new();
2421 for entry in std::fs::read_dir(home.join("runs"))
2422 .into_iter()
2423 .flatten()
2424 .flatten()
2425 {
2426 let id = entry.file_name().to_string_lossy().into_owned();
2427 if !crate::run::is_run_id(&id) {
2428 continue;
2429 }
2430 let Ok(_owner) = crate::pr_head::owner_lock(home, &id) else {
2433 continue;
2434 };
2435 let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
2443 continue;
2444 };
2445 let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
2446 continue;
2447 };
2448 if state.driver_exited || state.status.done() || !state.active_all_overrun(now) {
2449 continue;
2450 }
2451 let daemon_claims = is_working_on(home, &id, now);
2463 if state.liveness_under_with(home, daemon_claims, &query, &identity)
2464 != crate::run::Liveness::Dead
2465 {
2466 continue;
2467 }
2468 state.abandon("daemon");
2469 if let Err(e) = state.save_under(home) {
2470 tracing::warn!("could not persist abandoned run {id}: {e:#}");
2471 continue;
2472 }
2473 if let Some(notice) = notices::run_ended(&state) {
2476 notices::raise_in(home, notice);
2477 }
2478 if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
2486 tracing::warn!("abandon questions for {id}: {e:#}");
2487 }
2488 abandoned.push(id);
2489 }
2490 abandoned
2491}
2492
2493#[derive(Debug, Clone, Serialize, Deserialize)]
2494struct TaskWorker {
2495 opts: Opts,
2496 queue: PathBuf,
2497 task: String,
2498 #[serde(default)]
2499 lane: PermitKind,
2500 #[serde(default)]
2501 standalone: bool,
2502}
2503
2504fn task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2505 Ok(crate::worker::active(home)?
2506 .into_iter()
2507 .filter(|job| job.kind == "task")
2508 .collect())
2509}
2510
2511fn owned_task_jobs(home: &Path) -> Result<Vec<crate::worker::Job>> {
2512 task_jobs(home)?
2513 .into_iter()
2514 .filter_map(
2515 |job| match serde_json::from_value::<TaskWorker>(job.payload.clone()) {
2516 Ok(request) if !request.standalone => Some(Ok(job)),
2517 Ok(_) => None,
2518 Err(error) => Some(Err(error.into())),
2519 },
2520 )
2521 .collect()
2522}
2523
2524#[must_use]
2526pub fn managed(home: &Path) -> bool {
2527 match crate::worker::active(home) {
2528 Err(error) => {
2529 tracing::error!("cannot determine scheduler ownership: {error:#}");
2530 true
2531 }
2532 Ok(jobs) if jobs.iter().any(|j| j.kind == "scheduler") => true,
2533 Ok(_) => {
2534 read_status(home).is_some_and(|r| r.worker_protocol == Some(1))
2535 && crate::worker::try_lease(home, "scheduler").map_or(true, |lease| lease.is_none())
2536 }
2537 }
2538}
2539
2540pub async fn spawn_detached(home: &Path, opts: &Opts) -> Result<crate::worker::Job> {
2542 anyhow::ensure!(
2543 crate::worker::control(home, "scheduler")?.is_none() || !managed(home),
2544 "the scheduler is stopping; wait for retirement before starting it again"
2545 );
2546 crate::worker::spawn(
2547 home,
2548 "scheduler",
2549 "scheduler",
2550 serde_json::to_value(opts.absolute()?)?,
2551 )
2552 .await
2553}
2554
2555fn park_request_path(home: &Path, job: &crate::worker::Job) -> PathBuf {
2556 home.join("worker-parks").join(format!("{}.json", job.id))
2557}
2558
2559fn request_task_park(
2560 home: &Path,
2561 job: &crate::worker::Job,
2562 cause: crate::run::ParkCause,
2563) -> Result<()> {
2564 let path = park_request_path(home, job);
2565 std::fs::create_dir_all(path.parent().context("worker park directory")?)?;
2566 std::fs::write(path, serde_json::to_vec(&cause)?)?;
2567 crate::worker::request(home, &job.key, crate::worker::Control::Park)
2568}
2569
2570pub fn request_control(home: &Path, control: crate::worker::Control) -> Result<()> {
2572 if control == crate::worker::Control::Park {
2573 for job in owned_task_jobs(home)? {
2574 request_task_park(home, &job, crate::run::ParkCause::Operator)?;
2575 }
2576 }
2577 crate::worker::request(home, "scheduler", control)
2578}
2579
2580pub async fn restart_detached(home: &Path) -> Result<()> {
2582 let active = crate::worker::active(home)?;
2583 let prior = active.into_iter().find(|job| job.kind == "scheduler");
2584 if prior.is_none() && !managed(home) {
2585 return Ok(());
2586 }
2587 let opts: Opts = match prior {
2588 Some(job) => serde_json::from_value(job.payload)?,
2589 None => serde_json::from_slice(&std::fs::read(home.join("scheduler-options.json"))?)?,
2590 };
2591 request_control(home, crate::worker::Control::Stop)?;
2592 loop {
2593 let reserved = crate::worker::active(home)?
2594 .iter()
2595 .any(|job| job.kind == "scheduler");
2596 if !reserved && crate::worker::try_lease(home, "scheduler")?.is_some() {
2597 break;
2598 }
2599 tokio::time::sleep(Duration::from_millis(100)).await;
2600 }
2601 spawn_detached(home, &opts).await?;
2602 Ok(())
2603}
2604
2605pub async fn run_scheduler_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2607 let opts: Opts = serde_json::from_value(payload)?;
2608 let _ = home;
2609 serve_until(opts, Stop::new()).await
2610}
2611
2612pub async fn run_task_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
2614 let request: TaskWorker = serde_json::from_value(payload)?;
2615 let queue = Queue::at(request.queue);
2616 let key = format!("task:{}", request.task);
2617 let job = task_jobs(home)?
2618 .into_iter()
2619 .find(|job| job.key == key)
2620 .context("task worker reservation is missing")?;
2621 let park_path = park_request_path(home, &job);
2622 let _claim = loop {
2625 match queue.claim(&request.task) {
2626 Ok(claim) => break claim,
2627 Err(e) if e.chain().any(|c| c.is::<std::io::Error>()) => {
2630 return Err(e.context("claim the task for its worker"));
2631 }
2632 Err(_) => {
2633 sweep_stale_claims(&queue, STALE_CLAIM);
2634 tokio::time::sleep(Duration::from_millis(50)).await;
2635 }
2636 }
2637 };
2638 let mut task = queue.get(&request.task)?;
2639 if !task.status.runnable()
2640 || (!request.standalone
2641 && request.lane
2642 != permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent))
2643 {
2644 return Ok(());
2645 }
2646 let stop = Stop::new();
2647 let watcher = {
2648 let home = home.to_path_buf();
2649 let stop = stop.clone();
2650 let park_path = park_path.clone();
2651 tokio::spawn(async move {
2652 loop {
2653 if crate::worker::control(&home, &key).ok().flatten().is_some() {
2654 let cause = std::fs::read(&park_path)
2655 .ok()
2656 .and_then(|body| serde_json::from_slice(&body).ok())
2657 .unwrap_or(crate::run::ParkCause::Operator);
2658 stop.park(cause);
2659 return;
2660 }
2661 tokio::time::sleep(Duration::from_millis(200)).await;
2662 }
2663 })
2664 };
2665 let quota = if request.standalone {
2666 run_claimed_until(&request.opts, &queue, &mut task, &stop).await
2667 } else {
2668 let status = Arc::new(Mutex::new(Status::new()));
2669 Box::pin(attempt(
2670 &request.opts,
2671 &queue,
2672 &status,
2673 &stop,
2674 crate::graph::Pause::new(),
2675 &mut task,
2676 ))
2677 .await
2678 };
2679 watcher.abort();
2680 let _ = std::fs::remove_file(park_path);
2681 persist_cooldown(home, "a).await?;
2682 Ok(())
2683}
2684
2685pub async fn spawn_claimed(
2688 home: &Path,
2689 opts: &Opts,
2690 queue: &Queue,
2691 task: &Task,
2692 claim: crate::queue::Claim,
2693) -> Result<crate::worker::Job> {
2694 let request = TaskWorker {
2695 opts: opts.absolute()?,
2696 queue: std::path::absolute(queue.root())?,
2697 task: task.id.clone(),
2698 lane: permit_kind(land_resume_state(task) == LandResume::Ready, task.urgent),
2699 standalone: true,
2700 };
2701 let job = crate::worker::spawn(
2702 home,
2703 "task",
2704 &format!("task:{}", task.id),
2705 serde_json::to_value(request)?,
2706 )
2707 .await?;
2708 drop(claim);
2709 Ok(job)
2710}
2711
2712pub async fn run_detached_claimed(
2715 opts: &Opts,
2716 queue: &Queue,
2717 task: &mut Task,
2718 claim: crate::queue::Claim,
2719) -> Result<()> {
2720 let home = crate::run::home();
2721 let job = spawn_claimed(&home, opts, queue, task, claim).await?;
2722 loop {
2723 crate::worker::active(&home)?;
2726 let current = crate::worker::get(&home, &job.id)?;
2727 if matches!(
2728 current.status,
2729 crate::worker::Status::Finished | crate::worker::Status::Failed
2730 ) {
2731 *task = queue.get(&task.id)?;
2732 anyhow::ensure!(
2733 current.status != crate::worker::Status::Failed,
2734 "task worker failed: {}",
2735 current.result.as_deref().unwrap_or("unknown failure")
2736 );
2737 return Ok(());
2738 }
2739 tokio::time::sleep(Duration::from_millis(200)).await;
2740 }
2741}
2742
2743async fn persist_cooldown(home: &Path, quota: &[QuotaLoss]) -> Result<()> {
2744 if let Some(until) = cooldown_until(quota, Timestamp::now()) {
2745 let _lease = loop {
2746 if let Some(lease) = crate::worker::try_lease(home, "task-quota-cooldown")? {
2747 break lease;
2748 }
2749 tokio::time::sleep(Duration::from_millis(10)).await;
2750 };
2751 if read_cooldown(home)?.is_none_or(|previous| previous < until) {
2752 let path = home.join("task-quota-cooldown.json");
2753 let temporary = path.with_extension("json.tmp");
2754 std::fs::write(&temporary, serde_json::to_vec(&until)?)?;
2755 std::fs::rename(temporary, path)?;
2756 }
2757 }
2758 Ok(())
2759}
2760
2761fn read_cooldown(home: &Path) -> Result<Option<Timestamp>> {
2762 match std::fs::read(home.join("task-quota-cooldown.json")) {
2763 Ok(body) => Ok(Some(serde_json::from_slice(&body)?)),
2764 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
2765 Err(error) => Err(error.into()),
2766 }
2767}
2768
2769pub async fn serve(opts: Opts) -> Result<()> {
2775 tracing::info!(
2776 "automatic update and restart requires magi web; standalone serve does not restart"
2777 );
2778 serve_until(opts, Stop::new()).await
2779}
2780
2781pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
2792 let opts = opts.absolute()?;
2793 let home = crate::run::home();
2794 let _scheduler = crate::worker::try_lease(&home, "scheduler")?
2795 .context("another scheduler already owns this home")?;
2796 if let Some(reading) = read_status(&home)
2797 && legacy_scheduler_running(&reading, Timestamp::now(), std::process::id())
2798 {
2799 bail!("a legacy scheduler is still running; stop it before starting independent dispatch");
2800 }
2801 if !crate::worker::active(&home)?
2802 .iter()
2803 .any(|job| job.kind == "scheduler")
2804 {
2805 crate::worker::clear_control(&home, "scheduler")?;
2806 }
2807 std::fs::write(
2808 home.join("scheduler-options.json"),
2809 serde_json::to_vec(&opts)?,
2810 )?;
2811 let control = {
2812 let home = home.clone();
2813 let stop = stop.clone();
2814 tokio::spawn(async move {
2815 loop {
2816 match crate::worker::control(&home, "scheduler").ok().flatten() {
2817 Some(crate::worker::Control::Park) => {
2818 let _ = request_control(&home, crate::worker::Control::Park);
2819 stop.park(crate::run::ParkCause::Operator);
2820 break;
2821 }
2822 Some(crate::worker::Control::Stop) => {
2823 stop.stop();
2824 break;
2825 }
2826 None => {}
2827 }
2828 tokio::time::sleep(Duration::from_millis(100)).await;
2829 }
2830 })
2831 };
2832 let signal = {
2833 let stop = stop.clone();
2834 tokio::spawn(async move {
2835 if tokio::signal::ctrl_c().await.is_ok() {
2836 stop.stop();
2837 tracing::info!("scheduler shutdown requested; independent workers continue");
2838 }
2839 })
2840 };
2841
2842 let worktrees_root = opts
2843 .worktrees_root
2844 .clone()
2845 .unwrap_or_else(crate::run::default_worktree_root);
2846 let outcome = drive(
2847 &opts,
2848 &Queue::open(),
2849 &status_path(),
2850 &crate::run::home(),
2851 &worktrees_root,
2852 &stop,
2853 )
2854 .await;
2855
2856 signal.abort();
2857 control.abort();
2858 outcome
2859}
2860
2861async fn drive(
2874 opts: &Opts,
2875 queue: &Queue,
2876 status_file: &Path,
2877 home: &Path,
2878 worktrees_root: &Path,
2879 stop: &Stop,
2880) -> Result<()> {
2881 let recovery_cfg = prepare(&opts.repo, opts).ok();
2889 let daemon_cfg = recovery_cfg
2890 .as_ref()
2891 .map(|c| c.daemon.clone())
2892 .unwrap_or_default();
2893 let reconciled = crate::driver::reconcile(home);
2894 if !reconciled.is_empty() {
2895 tracing::info!(runs = ?reconciled, "reconciled stopped run drivers at startup");
2896 }
2897 let mut initial_status = Status::new();
2898 let owned = owned_task_jobs(home)?;
2899 initial_status.current = current_work(home, Timestamp::now())
2900 .into_iter()
2901 .filter(|current| {
2902 owned
2903 .iter()
2904 .any(|job| job.key.strip_prefix("task:") == Some(current.task.as_str()))
2905 })
2906 .collect();
2907 initial_status.idle = initial_status.current.is_empty();
2908 initial_status.question_recovery = Some(crate::question_delivery::Recovery::new(
2909 recovery_cfg.as_ref(),
2910 daemon_cfg.max_deputies,
2911 ));
2912 let status = Arc::new(Mutex::new(initial_status));
2913 write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
2914 let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
2915
2916 let concurrency = max_concurrent(daemon_cfg.max_concurrent_runs);
2922
2923 tracing::info!(
2924 "magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once{})",
2925 queue.root().display(),
2926 opts.poll.as_secs(),
2927 opts.max_attempts,
2928 concurrency,
2929 if daemon_cfg.pause_for_interrupts {
2930 ", interrupts enabled"
2931 } else {
2932 ""
2933 }
2934 );
2935
2936 let outcome = tokio::select! {
2937 outcome = poll(opts, queue, &status, home, worktrees_root, stop, DispatchLimits {
2938 max_concurrent: concurrency,
2939 pause_for_interrupts: daemon_cfg.pause_for_interrupts,
2940 recovery_cfg,
2941 }) => outcome,
2942 () = async {
2943 while !stop.stopped() {
2944 tokio::time::sleep(Duration::from_millis(25)).await;
2945 }
2946 } => {
2947 if stop.parking() {
2948 request_control(home, crate::worker::Control::Park)?;
2949 }
2950 Ok(())
2951 }
2952 };
2953
2954 stop.busy.store(0, Ordering::SeqCst);
2955 beat.abort();
2956 clear_status_at(status_file);
2957 outcome
2958}
2959
2960pub(crate) fn release_watch_settings_from_config(
2963 home: &Path,
2964 repo: &Path,
2965 cfg: &Config,
2966) -> (Vec<PathBuf>, u64) {
2967 let mut paths: Vec<PathBuf> = crate::repos::scan(&cfg.repos.roots)
2968 .into_iter()
2969 .map(|r| r.path)
2970 .collect();
2971 let mut known: Vec<PathBuf> = Queue::at(home.join("queue"))
2974 .list()
2975 .into_iter()
2976 .map(|t| t.repo)
2977 .collect();
2978 let runs = home.join("runs");
2979 for id in crate::run::list_ids_in(&runs) {
2980 let repo = std::fs::read_to_string(runs.join(&id).join("run.json"))
2981 .ok()
2982 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
2983 .and_then(|v| v.get("repo")?.as_str().map(PathBuf::from));
2984 known.extend(repo);
2985 }
2986 for k in known {
2987 let k = k.canonicalize().unwrap_or(k);
2988 if k.join(".git").exists() && !paths.contains(&k) {
2989 paths.push(k);
2990 }
2991 }
2992 let own = repo.canonicalize().unwrap_or_else(|_| repo.to_path_buf());
2993 if !paths.contains(&own) {
2994 paths.push(own);
2995 }
2996 (paths, cfg.daemon.release_stall_minutes)
2997}
2998
2999const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
3001
3002async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
3008 loop {
3009 tokio::time::sleep(HEARTBEAT).await;
3010 let snapshot = {
3011 let mut guard = lock(&status);
3012 guard.updated_at = Timestamp::now();
3013 guard.clone()
3014 };
3015 if let Err(e) = write_status_to(&path, &snapshot) {
3016 tracing::warn!("could not refresh the daemon status file: {e:#}");
3019 }
3020 }
3021}
3022
3023#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3026enum LandResume {
3027 NotLanding,
3030 StillWaiting,
3035 Ready,
3039}
3040
3041fn land_resume_state(task: &Task) -> LandResume {
3045 let Some(run_id) = task.execution_runs().last() else {
3046 return LandResume::NotLanding;
3047 };
3048 let Ok(state) = RunState::load(run_id) else {
3049 return LandResume::NotLanding;
3050 };
3051 if !state.parked {
3052 return LandResume::NotLanding;
3053 }
3054 if state.status == RunStatus::Landing
3055 && state.land_approval.as_ref().is_some_and(|binding| {
3056 ask::Questions::open()
3057 .get(&binding.question)
3058 .is_ok_and(|q| {
3059 q.run == state.id
3060 && q.resolution()
3061 .as_deref()
3062 .is_some_and(crate::merge_decision::stops)
3063 })
3064 })
3065 {
3066 return LandResume::Ready;
3067 }
3068 if !implementers_match(task, &state) {
3069 return LandResume::NotLanding;
3070 }
3071 let node = if state.status == RunStatus::Landing {
3075 land::APPROVAL_NODE
3076 } else if state.github_text.as_ref().is_some_and(|g| !g.resolved) {
3077 crate::github_text::ASK_NODE
3078 } else {
3079 return LandResume::NotLanding;
3080 };
3081 let store = ask::Questions::open();
3082 let waiting = store
3083 .list()
3084 .into_iter()
3085 .filter(|q| &q.run == run_id && q.node == node)
3086 .max_by(|a, b| a.id.cmp(&b.id));
3087 let Some(mut q) = waiting else {
3088 return LandResume::Ready;
3089 };
3090 if !q.status.open() {
3091 if q.resolution().as_deref() == Some(land::REPAIR) {
3094 return if state.liveness(false) == Liveness::Live {
3095 LandResume::StillWaiting
3096 } else {
3097 LandResume::NotLanding
3098 };
3099 }
3100 return LandResume::Ready;
3101 }
3102 let timeout = Duration::from_secs(state.config.graph.answer_timeout);
3109 let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
3110 if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
3111 q.abandon(format!(
3112 "no answer within {}s of asking",
3113 timeout.as_secs().max(1)
3114 ));
3115 if store.put(&mut q).is_ok() {
3118 return LandResume::Ready;
3119 }
3120 }
3121 LandResume::StillWaiting
3122}
3123
3124const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
3132
3133const CACHE_CHECK_INTERVAL_SECS: u64 = 5 * 60;
3145
3146#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3159enum Interrupt {
3160 Idle,
3162 Parking {
3174 parked: Vec<String>,
3175 interrupt_task: String,
3176 },
3177 Running {
3185 parked: Vec<String>,
3186 interrupt_task: String,
3187 },
3188 Resuming { parked: Vec<String> },
3195}
3196
3197fn advance_interrupt(state: Interrupt, in_flight: &[String], runnable: &[Task]) -> Interrupt {
3218 match state {
3219 Interrupt::Idle => {
3220 if in_flight.len() != 1 {
3232 return Interrupt::Idle;
3233 }
3234 match runnable.iter().find(|t| t.interrupt) {
3235 Some(t) => Interrupt::Parking {
3236 parked: in_flight.to_vec(),
3237 interrupt_task: t.id.clone(),
3238 },
3239 None => Interrupt::Idle,
3240 }
3241 }
3242 Interrupt::Parking {
3243 parked,
3244 interrupt_task,
3245 } => {
3246 if in_flight.iter().any(|id| parked.contains(id)) {
3247 Interrupt::Parking {
3249 parked,
3250 interrupt_task,
3251 }
3252 } else if in_flight.contains(&interrupt_task) {
3253 Interrupt::Running {
3254 parked,
3255 interrupt_task,
3256 }
3257 } else if runnable.iter().any(|t| t.id == interrupt_task) {
3258 Interrupt::Parking {
3262 parked,
3263 interrupt_task,
3264 }
3265 } else {
3266 Interrupt::Resuming { parked }
3271 }
3272 }
3273 Interrupt::Running {
3274 parked,
3275 interrupt_task,
3276 } => {
3277 if in_flight.contains(&interrupt_task) {
3278 Interrupt::Running {
3279 parked,
3280 interrupt_task,
3281 }
3282 } else {
3283 Interrupt::Resuming { parked }
3289 }
3290 }
3291 Interrupt::Resuming { parked } => {
3292 if in_flight.iter().any(|id| parked.contains(id)) {
3293 Interrupt::Idle
3299 } else if runnable.iter().any(|t| parked.contains(&t.id)) {
3300 Interrupt::Resuming { parked }
3301 } else {
3302 Interrupt::Idle
3305 }
3306 }
3307 }
3308}
3309
3310fn advance_interrupt_tick(
3316 enabled: bool,
3317 state: Interrupt,
3318 in_flight: &[String],
3319 runnable: &[Task],
3320) -> Interrupt {
3321 if !enabled {
3322 return Interrupt::Idle;
3323 }
3324 advance_interrupt(state, in_flight, runnable)
3325}
3326
3327fn interrupt_gate(state: &Interrupt, in_flight: &[String], candidates: Vec<Task>) -> Vec<Task> {
3332 match state {
3333 Interrupt::Idle => candidates,
3334 Interrupt::Parking {
3335 parked,
3336 interrupt_task,
3337 } => {
3338 if in_flight.iter().any(|id| parked.contains(id)) {
3339 Vec::new()
3340 } else {
3341 candidates
3342 .into_iter()
3343 .filter(|t| &t.id == interrupt_task)
3344 .collect()
3345 }
3346 }
3347 Interrupt::Running { .. } => Vec::new(),
3348 Interrupt::Resuming { parked } => candidates
3356 .into_iter()
3357 .find(|t| parked.contains(&t.id))
3358 .into_iter()
3359 .collect(),
3360 }
3361}
3362
3363struct DispatchLimits {
3367 max_concurrent: usize,
3370 pause_for_interrupts: bool,
3372 recovery_cfg: Option<Config>,
3373}
3374
3375#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
3384enum PermitKind {
3385 None,
3390 Urgent,
3397 #[default]
3400 Ordinary,
3401}
3402
3403fn permit_kind(priority: bool, urgent: bool) -> PermitKind {
3406 if priority {
3407 PermitKind::None
3408 } else if urgent {
3409 PermitKind::Urgent
3410 } else {
3411 PermitKind::Ordinary
3412 }
3413}
3414
3415fn available_slots(requests: &[TaskWorker], limit: usize) -> (usize, usize) {
3416 let (ordinary, urgent) = requests.iter().filter(|request| !request.standalone).fold(
3417 (0usize, 0usize),
3418 |(ordinary, urgent), request| match request.lane {
3419 PermitKind::Ordinary => (ordinary + 1, urgent),
3420 PermitKind::Urgent => (ordinary, urgent + 1),
3421 PermitKind::None => (ordinary, urgent),
3422 },
3423 );
3424 (
3425 limit.saturating_sub(ordinary),
3426 1usize.saturating_sub(urgent),
3427 )
3428}
3429
3430struct DispatchClaim {
3431 task: Task,
3432 claim: crate::queue::Claim,
3433 permit: Option<tokio::sync::OwnedSemaphorePermit>,
3434}
3435
3436fn claim_dispatch(
3439 queue: &Queue,
3440 candidate: &Task,
3441 cooling_down: bool,
3442 ordinary: &Arc<tokio::sync::Semaphore>,
3443 urgent: &Arc<tokio::sync::Semaphore>,
3444) -> Option<DispatchClaim> {
3445 let Ok(claim) = queue.claim(&candidate.id) else {
3446 tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
3447 return None;
3448 };
3449 let task = match queue.get(&candidate.id) {
3450 Ok(t) if t.status.runnable() => t,
3451 Ok(_) => return None,
3452 Err(e) => {
3453 tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
3454 return None;
3455 }
3456 };
3457 let resume = land_resume_state(&task);
3458 if resume == LandResume::StillWaiting {
3459 return None;
3460 }
3461 let priority = resume == LandResume::Ready;
3462 if !priority && cooling_down {
3463 return None;
3464 }
3465 let permit = match permit_kind(priority, task.urgent) {
3468 PermitKind::None => None,
3469 PermitKind::Urgent => Some(Arc::clone(urgent).try_acquire_owned().ok()?),
3470 PermitKind::Ordinary => Some(Arc::clone(ordinary).try_acquire_owned().ok()?),
3471 };
3472 let stamp = Timestamp::now();
3475 let mut task = task;
3476 task.dispatched_at = Some(stamp);
3477 if let Err(e) = queue.modify(&task.id, |t| {
3478 t.dispatched_at = Some(stamp);
3479 true
3480 }) {
3481 tracing::warn!("could not record dispatch order of {}: {e:#}", task.short());
3482 }
3483 Some(DispatchClaim {
3484 task,
3485 claim,
3486 permit,
3487 })
3488}
3489
3490#[derive(Serialize, Deserialize)]
3509struct ServiceWorker {
3510 opts: Opts,
3511 queue: PathBuf,
3512 worktrees_root: PathBuf,
3513 operation: String,
3514}
3515
3516fn load_conductor(home: &Path) -> Result<Conductor> {
3517 match std::fs::read(home.join("conductor-state.json")) {
3518 Ok(body) => serde_json::from_slice(&body).context("read conductor state"),
3519 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Conductor::new()),
3520 Err(error) => Err(error.into()),
3521 }
3522}
3523
3524fn conductor_work(queue: &Queue, home: &Path) -> (Vec<Task>, Vec<Task>, Vec<Task>) {
3525 let live: std::collections::BTreeSet<_> = current_work(home, Timestamp::now())
3526 .into_iter()
3527 .map(|c| c.task)
3528 .collect();
3529 let stalled = stalled_tasks(queue, home, Timestamp::now());
3530 let stalled_ids: std::collections::BTreeSet<_> = stalled.iter().map(|t| t.id.clone()).collect();
3531 let finished = finished_tasks(queue)
3532 .into_iter()
3533 .filter(|t| !live.contains(&t.id) && !stalled_ids.contains(&t.id))
3534 .filter(|t| !awaiting_resume_with(t, |id| RunState::load_under(id, home)))
3535 .collect();
3536 let queued = queued_tasks(queue)
3537 .into_iter()
3538 .filter(|t| !live.contains(&t.id))
3539 .collect();
3540 (queued, stalled, finished)
3541}
3542
3543async fn start_service(
3544 home: &Path,
3545 opts: &Opts,
3546 queue: &Queue,
3547 worktrees_root: &Path,
3548 operation: &str,
3549 cfg: Option<&Config>,
3550 busy: bool,
3551) -> Result<()> {
3552 let fetch_cfg = if operation == "fetch" {
3555 match prepare(&opts.repo, opts) {
3556 Ok(cfg) => Some(cfg),
3557 Err(error) => {
3558 tracing::warn!("could not reload periodic fetch configuration: {error:#}");
3559 return Ok(());
3560 }
3561 }
3562 } else {
3563 None
3564 };
3565 let cfg = fetch_cfg.as_ref().or(cfg);
3566 let ticks = home.join("service-ticks");
3567 std::fs::create_dir_all(&ticks)?;
3568 let receipt = ticks.join(format!("daemon-{operation}"));
3569 let last = std::fs::read(&receipt)
3570 .ok()
3571 .and_then(|body| serde_json::from_slice::<Timestamp>(&body).ok());
3572 let interval = match operation {
3573 "disk-pressure" => 30,
3574 "cache" => CACHE_CHECK_INTERVAL_SECS,
3575 "fetch" => cfg.map_or(60, |cfg| cfg.repos.fetch_interval.max(1)),
3576 _ => opts.poll.as_secs().max(1),
3577 };
3578 let now = Timestamp::now();
3579 if !cache_check_due(last, now, interval) {
3580 return Ok(());
3581 }
3582 let key = format!("service:daemon-{operation}");
3583 if crate::worker::active(home)?
3584 .iter()
3585 .any(|job| job.key == key)
3586 {
3587 return Ok(());
3588 }
3589 let eligible = match operation {
3590 "conductor" => {
3591 let (queued, stalled, finished) = conductor_work(queue, home);
3592 !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3593 && load_conductor(home)?.worth_a_look(queue, &stalled, &finished)
3594 }
3595 "maintenance" => Questions::at(home.join("questions"))
3596 .list()
3597 .iter()
3598 .any(|q| {
3599 q.chosen_action().is_some_and(|action| action.maintenance())
3600 && !home
3601 .join("choice-actions")
3602 .join(format!("{}.json", q.id))
3603 .exists()
3604 }),
3605 "housekeeping" => !busy && runnable(queue).is_empty(),
3606 "cache" => {
3607 !busy
3608 && !runnable(queue).is_empty()
3609 && cfg.is_some_and(|cfg| cfg.disk.cache_limit_bytes > 0)
3610 }
3611 "disk-pressure" => busy,
3612 "fetch" => {
3613 cfg.is_some_and(|cfg| cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty())
3614 }
3615 _ => false,
3616 };
3617 if !eligible {
3618 return Ok(());
3619 }
3620 let request = ServiceWorker {
3621 opts: opts.absolute()?,
3622 queue: std::path::absolute(queue.root())?,
3623 worktrees_root: std::path::absolute(worktrees_root)?,
3624 operation: operation.to_owned(),
3625 };
3626 if let Err(error) =
3627 crate::worker::spawn(home, "daemon-service", &key, serde_json::to_value(request)?).await
3628 {
3629 tracing::warn!(operation, "could not launch daemon service: {error:#}");
3630 }
3631 std::fs::write(receipt, serde_json::to_vec(&now)?)?;
3632 Ok(())
3633}
3634
3635pub async fn run_service_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
3638 let request: ServiceWorker = serde_json::from_value(payload)?;
3639 let opts = &request.opts;
3640 let queue = Queue::at(request.queue);
3641 let questions = Questions::at(home.join("questions"));
3642 match request.operation.as_str() {
3643 "maintenance" => apply_maintenance_choices(&queue, &questions, home).await,
3644 "conductor" => {
3645 let path = home.join("conductor-state.json");
3646 let mut conductor = load_conductor(home)?;
3647 let (queued, stalled, finished) = conductor_work(&queue, home);
3648 if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
3649 && conductor.worth_a_look(&queue, &stalled, &finished)
3650 {
3651 let cfg = prepare(&opts.repo, opts)?;
3652 conductor
3653 .maybe_run(
3654 &cfg,
3655 &opts.repo,
3656 &queue,
3657 &questions,
3658 home,
3659 &queued,
3660 &stalled,
3661 &finished,
3662 opts.max_attempts,
3663 )
3664 .await;
3665 let temporary = path.with_extension("json.tmp");
3666 std::fs::write(&temporary, serde_json::to_vec(&conductor)?)?;
3667 std::fs::rename(temporary, path)?;
3668 }
3669 }
3670 "fetch" => {
3671 let cfg = prepare(&opts.repo, opts)?;
3672 if cfg.repos.fetch_interval > 0 && !cfg.repos.roots.is_empty() {
3673 let outcome = clean::fetch_origins(&cfg.repos.roots, FETCH_TIMEOUT, || false).await;
3674 tracing::debug!("fetch origins: {outcome:?}");
3675 }
3676 }
3677 "cache" => {
3678 if task_jobs(home)?.is_empty() {
3679 let mut checked = None;
3680 maybe_prune_cache_between_runs(
3681 &opts.repo,
3682 opts,
3683 home,
3684 &Stop::new(),
3685 &mut checked,
3686 Timestamp::now(),
3687 )
3688 .await;
3689 }
3690 }
3691 "disk-pressure" => {
3692 if let Ok(cfg) = prepare(&opts.repo, opts) {
3693 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3694 if let Err(e) = clean::prune_cache_if_over_limit(&cfg, home) {
3695 tracing::warn!("disk pressure cache cleanup failed: {e:#}");
3696 }
3697 for current in current_work(home, Timestamp::now()) {
3698 if let Ok(state) = RunState::load_under(¤t.run, home)
3699 && let Some(reason) = disk_gate(&state.repo, &state.config)
3700 {
3701 for job in owned_task_jobs(home)?.into_iter().filter(|job| {
3702 job.key.strip_prefix("task:") == Some(current.task.as_str())
3703 }) {
3704 request_task_park(
3705 home,
3706 &job,
3707 crate::run::ParkCause::Interrupt {
3708 reason: reason.clone(),
3709 },
3710 )?;
3711 }
3712 notices::raise_in(
3713 home,
3714 Notice::warn(
3715 "disk:pressure",
3716 "Disk pressure persists after cleanup; active workers were asked to checkpoint and park. New starts and resumes wait for free space.",
3717 ),
3718 );
3719 }
3720 }
3721 }
3722 }
3723 "housekeeping" => {
3724 if task_jobs(home)?.is_empty() {
3725 if let Ok(cfg) = prepare(&opts.repo, opts) {
3726 crate::retention::sweep(home, &cfg.disk, Timestamp::now()).await;
3727 }
3728 janitor(&opts.repo, opts, home, &request.worktrees_root).await;
3729 resweep_superseded_attempts(&queue, home);
3730 triage_held(&queue, home, opts).await;
3731 }
3732 }
3733 operation => bail!("unknown daemon service {operation}"),
3734 }
3735 Ok(())
3736}
3737
3738fn task_worker_spawn_error(home: &Path, request: &TaskWorker, error: anyhow::Error) -> Result<()> {
3742 let key = format!("task:{}", request.task);
3743 let payload = serde_json::to_value(request)?;
3744 if crate::worker::active(home)?
3745 .iter()
3746 .any(|job| job.key == key && job.kind == "task" && job.payload != payload)
3747 {
3748 tracing::warn!(
3749 "task {} acquired a competing worker reservation: {error:#}",
3750 crate::run::short_of(&request.task)
3751 );
3752 return Ok(());
3753 }
3754 tracing::error!("task worker startup stopped dispatch: {error:#}");
3755 notices::raise_in(
3756 home,
3757 Notice::error(
3758 "loop:worker-startup",
3759 format!("Task {} worker could not be started or acknowledged. Scheduler dispatch stopped; already reserved workers may still complete. Inspect the local execution log and reservation worker.log for startup details.", request.task),
3762 ).link(Link::Task { id: request.task.clone() }),
3763 );
3764 Err(error).with_context(|| {
3765 format!(
3766 "cannot start task {} worker; dispatcher stopped; any unacknowledged reservation remains occupied",
3767 crate::run::short_of(&request.task)
3768 )
3769 })
3770}
3771
3772async fn poll(
3773 opts: &Opts,
3774 queue: &Queue,
3775 status: &Arc<Mutex<Status>>,
3776 home: &Path,
3777 worktrees_root: &Path,
3778 stop: &Stop,
3779 limits: DispatchLimits,
3780) -> Result<()> {
3781 let mut attempted = Vec::new();
3782 let mut settling_once_services = false;
3783 let interrupt_path = home.join("scheduler-interrupt.json");
3784 let mut interrupt = match std::fs::read(&interrupt_path) {
3785 Ok(body) => serde_json::from_slice(&body).context("read scheduler interrupt state")?,
3786 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Interrupt::Idle,
3787 Err(error) => return Err(error.into()),
3788 };
3789 let mut previous = std::collections::BTreeSet::<String>::new();
3790 while !stop.stopped() {
3791 let jobs: Vec<_> = crate::worker::active(home)?
3792 .into_iter()
3793 .filter(|job| job.kind == "task")
3794 .collect();
3795 let requests: Vec<_> = jobs
3796 .iter()
3797 .map(|job| {
3798 serde_json::from_value::<TaskWorker>(job.payload.clone())
3799 .with_context(|| format!("read live task worker {}", job.id))
3800 })
3801 .collect::<Result<_>>()?;
3802 let live: std::collections::BTreeSet<_> = requests.iter().map(|r| r.task.clone()).collect();
3803 let owned: std::collections::BTreeSet<_> = requests
3804 .iter()
3805 .filter(|r| !r.standalone)
3806 .map(|r| r.task.clone())
3807 .collect();
3808 let current: Vec<_> = requests
3809 .iter()
3810 .zip(&jobs)
3811 .filter(|(request, _)| !request.standalone)
3812 .map(|(request, job)| Current {
3813 worker: Some(job.id.clone()),
3814 task: request.task.clone(),
3815 run: queue
3816 .get(&request.task)
3817 .ok()
3818 .and_then(|task| task.execution_runs().last().cloned())
3819 .unwrap_or_default(),
3820 })
3821 .collect();
3822 {
3823 let mut snapshot = lock(status);
3824 snapshot.polls += 1;
3825 snapshot.completed += previous.difference(&owned).count();
3826 snapshot.current = current;
3827 snapshot.idle = owned.is_empty();
3828 }
3829 previous = owned.clone();
3830 stop.busy.store(owned.len(), Ordering::SeqCst);
3831
3832 let (ordinary, urgent) = available_slots(&requests, limits.max_concurrent);
3835 let sem = Arc::new(tokio::sync::Semaphore::new(ordinary));
3836 let urgent_sem = Arc::new(tokio::sync::Semaphore::new(urgent));
3837 let mut permits = Vec::new();
3838 sweep_stale_claims(queue, STALE_CLAIM);
3839 reclaim_orphaned_running(queue, opts.max_attempts, &live);
3840 reclaim_abandoned_runs(home, Timestamp::now());
3841 let questions = Questions::at(home.join("questions"));
3842
3843 crate::task_chat::sweep(
3846 queue,
3847 &crate::talk::Talks::at(home.join("talks")),
3848 &questions,
3849 home,
3850 );
3851 repair_choice_offers(&questions);
3852 resolve_blockers(queue, &questions);
3853 apply_task_choice_actions(queue, &questions, home);
3854 triage::apply_answers(queue, &questions);
3855 reconcile_task_questions(queue, &questions);
3856 let talks = crate::talk::Talks::at(home.join("talks"));
3857 crate::chat_notify::sweep(queue, &talks, home);
3858 crate::notice_delivery::sweep(queue, &talks, home, &opts.repo);
3859 crate::chat_watch::sweep(home, &talks);
3860 let mut turns = tokio::task::JoinSet::new();
3861 crate::chat_notify::start_turns(&talks, stop.parking(), &mut turns);
3862 while turns.join_next().await.is_some() {}
3864
3865 crate::consult::start_services(
3866 home,
3867 &opts.repo,
3868 limits.recovery_cfg.as_ref(),
3869 opts.config.as_deref(),
3870 )
3871 .await?;
3872 for operation in [
3873 "conductor",
3874 "maintenance",
3875 "housekeeping",
3876 "disk-pressure",
3877 "cache",
3878 "fetch",
3879 ] {
3880 if settling_once_services {
3884 break;
3885 }
3886 start_service(
3887 home,
3888 opts,
3889 queue,
3890 worktrees_root,
3891 operation,
3892 limits.recovery_cfg.as_ref(),
3893 !live.is_empty(),
3894 )
3895 .await?;
3896 }
3897 let latest = crate::worker::active(home)?;
3898 if latest.iter().any(|job| job.key == "service:daemon-cache")
3899 || latest
3900 .iter()
3901 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3902 {
3903 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
3904 continue;
3905 }
3906 let candidates: Vec<Task> = runnable(queue)
3907 .into_iter()
3908 .filter(|task| !live.contains(&task.id))
3909 .filter(|task| !opts.once || !attempted.contains(&task.id))
3910 .collect();
3911 let in_flight: Vec<_> = owned.iter().cloned().collect();
3912 let prior_interrupt = std::mem::discriminant(&interrupt);
3913 interrupt = advance_interrupt_tick(
3914 limits.pause_for_interrupts,
3915 interrupt,
3916 &in_flight,
3917 &candidates,
3918 );
3919 if prior_interrupt != std::mem::discriminant(&interrupt) {
3922 let temporary = interrupt_path.with_extension("json.tmp");
3923 std::fs::write(&temporary, serde_json::to_vec(&interrupt)?)?;
3924 std::fs::rename(temporary, &interrupt_path)?;
3925 }
3926 if let Interrupt::Parking {
3927 parked,
3928 interrupt_task,
3929 } = &interrupt
3930 {
3931 for job in jobs.iter().filter(|job| {
3932 parked
3933 .iter()
3934 .any(|id| job.key.strip_prefix("task:") == Some(id.as_str()))
3935 }) {
3936 if crate::worker::control(home, &job.key)?.is_none() {
3937 request_task_park(
3938 home,
3939 job,
3940 crate::run::ParkCause::Interrupt {
3941 reason: format!(
3942 "task {} asked to run first",
3943 crate::run::short_of(interrupt_task)
3944 ),
3945 },
3946 )?;
3947 }
3948 }
3949 }
3950 let candidates = interrupt_gate(&interrupt, &in_flight, candidates);
3951 let cooling = read_cooldown(home)?.is_some_and(|until| Timestamp::now() < until);
3952 let mut started = false;
3953 let mut resync = false;
3954 for candidate in candidates {
3955 if stop.stopped() {
3956 break;
3957 }
3958 let Some(DispatchClaim {
3959 task,
3960 claim,
3961 permit,
3962 }) = claim_dispatch(queue, &candidate, cooling, &sem, &urgent_sem)
3963 else {
3964 continue;
3965 };
3966 if crate::worker::active(home)?
3967 .iter()
3968 .any(|job| job.kind == "task" && !jobs.iter().any(|known| known.id == job.id))
3969 {
3970 drop(claim);
3971 resync = true;
3972 break;
3973 }
3974 let lane = permit_kind(land_resume_state(&task) == LandResume::Ready, task.urgent);
3975 let request = TaskWorker {
3976 opts: opts.absolute()?,
3977 queue: std::path::absolute(queue.root())?,
3978 task: task.id.clone(),
3979 lane,
3980 standalone: false,
3981 };
3982 if let Err(error) = crate::worker::spawn(
3985 home,
3986 "task",
3987 &format!("task:{}", task.id),
3988 serde_json::to_value(&request)?,
3989 )
3990 .await
3991 {
3992 task_worker_spawn_error(home, &request, error)?;
3993 drop(claim);
3994 resync = true;
3995 break;
3996 }
3997 drop(claim);
3998 permits.extend(permit);
3999 attempted.push(task.id);
4000 started = true;
4001 }
4002 if started || resync {
4003 continue;
4004 }
4005 if owned.is_empty() && opts.once {
4006 settling_once_services = true;
4007 if crate::worker::active(home)?.iter().any(|job| {
4011 job.kind == "daemon-service"
4012 && matches!(
4013 job.key.as_str(),
4014 "service:daemon-housekeeping"
4015 | "service:daemon-conductor"
4016 | "service:daemon-maintenance"
4017 )
4018 }) {
4019 stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
4020 continue;
4021 }
4022 break;
4023 }
4024 stop.idle(if live.is_empty() {
4025 opts.poll
4026 } else {
4027 RECHECK_WHILE_BUSY.min(opts.poll)
4028 })
4029 .await;
4030 }
4031 stop.busy.store(0, Ordering::SeqCst);
4034 Ok(())
4035}
4036
4037async fn resume_owner_lock(home: &Path, id: &str) -> Result<crate::pr_head::OwnerLock> {
4041 let busy = |error: &anyhow::Error| {
4042 error.downcast_ref::<std::io::Error>().is_some_and(|error| {
4043 error.kind() == std::io::ErrorKind::WouldBlock
4044 || cfg!(windows) && error.raw_os_error() == Some(33)
4045 })
4046 };
4047 match crate::pr_head::owner_lock(home, id) {
4048 Ok(lock) => return Ok(lock),
4049 Err(error) if busy(&error) => {}
4050 Err(error) => return Err(error),
4051 }
4052 let wait = RunState::load_under(id, home)?
4053 .config
4054 .graph
4055 .timeout_resource_wait;
4056 crate::pr_head::owner_lock_wait(home, id, Duration::from_secs(wait)).await
4057}
4058
4059async fn public_resume_owner(
4063 home: &Path,
4064 runs: &[String],
4065 repo: &Path,
4066) -> Result<Option<RunState>> {
4067 let mut owner = None;
4068 for id in runs.iter().collect::<std::collections::BTreeSet<_>>() {
4069 let _lock = resume_owner_lock(home, id).await?;
4070 let state = RunState::load_under(id, home)?;
4071 if state.released()
4072 || state.retired_to.is_some()
4073 || state.pr_taken_over_by.is_some()
4074 || !crate::handover::landing_checkpoint(&state)
4075 {
4076 continue;
4077 }
4078 anyhow::ensure!(
4079 crate::git::toplevel(&state.repo).await?.canonicalize()?
4080 == crate::git::toplevel(repo).await?.canonicalize()?,
4081 "public owner repository differs; inspect run lineage"
4082 );
4083 anyhow::ensure!(
4084 owner.is_none(),
4085 "multiple retained public owners; inspect run lineage"
4086 );
4087 if let Some(reason) = crate::handover::resume_refusal_under(&state, Some(home)) {
4088 anyhow::bail!(reason.diagnostic(&state));
4089 }
4090 let claimed = is_working_on_other(home, id, Timestamp::now())?;
4091 anyhow::ensure!(
4092 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
4093 "public owner's driver is live or unknown; wait before retrying"
4094 );
4095 owner = Some(state);
4096 }
4097 Ok(owner)
4098}
4099
4100async fn attempt(
4106 opts: &Opts,
4107 queue: &Queue,
4108 status: &Arc<Mutex<Status>>,
4109 stop: &Stop,
4110 interrupt_pause: crate::graph::Pause,
4111 task: &mut Task,
4112) -> Vec<QuotaLoss> {
4113 let repo = repo_for(task, &opts.repo);
4114 tracing::info!(
4115 "task {} — {} (repo {})",
4116 task.short(),
4117 task.title,
4118 repo.display()
4119 );
4120
4121 let public_owner = match public_resume_owner(&crate::run::home(), &task.runs, &repo).await {
4122 Ok(owner) => owner,
4123 Err(e) => {
4124 task.hold_machine(Some(format!(
4125 "Cannot verify existing public owner: {e:#}; no attempt started."
4126 )));
4127 record(queue, task);
4128 return Vec::new();
4129 }
4130 };
4131 let resume_runs = public_owner
4132 .as_ref()
4133 .map(|s| vec![s.id.clone()])
4134 .unwrap_or_else(|| task.execution_runs().to_vec());
4135
4136 if let Some(previous) = resume_runs.last().and_then(|id| RunState::load(id).ok())
4140 && previous
4141 .merge_decision
4142 .as_ref()
4143 .is_some_and(|decision| !decision.closed)
4144 && previous.status != RunStatus::Landing
4145 && !crate::merge_decision::resumable_hold(&previous)
4146 {
4147 task.hold_unrequested_retry(Some(format!("The existing owner stop decision remains held. Inspect run {} and its PR; request an explicit fresh review/decision before further work. No replacement or merge was started.", previous.id)));
4148 record(queue, task);
4149 return Vec::new();
4150 }
4151 let stop_owner = resume_runs
4152 .last()
4153 .and_then(|id| RunState::load(id).ok())
4154 .filter(|state| {
4155 crate::merge_decision::resumable_hold(state)
4156 || state.status == RunStatus::Landing
4157 && state.land_approval.as_ref().is_some_and(|binding| {
4158 Questions::open().get(&binding.question).is_ok_and(|q| {
4159 q.run == state.id
4160 && q.resolution()
4161 .as_deref()
4162 .is_some_and(crate::merge_decision::stops)
4163 })
4164 })
4165 });
4166 if let Some(owner) = stop_owner {
4167 let mut runner = match Runner::resume(&owner.id) {
4168 Ok(runner) => runner,
4169 Err(e) => {
4170 task.hold_manual(Some(format!("Cannot stop the existing landing owner: {e:#}. No replacement or merge was started.")));
4171 record(queue, task);
4172 return Vec::new();
4173 }
4174 };
4175 runner.on_pause(stop.pause());
4176 runner.watch_interrupt(interrupt_pause);
4177 let quota_before = runner.state.quota.clone();
4178 task.start(owner.id.clone());
4179 record(queue, task);
4180 lock(status).current.push(Current {
4181 worker: None,
4182 task: task.id.clone(),
4183 run: owner.id,
4184 });
4185 runner.state.next_pass_task_entry = true;
4186 let result = runner.execute().await;
4187 return finish_attempt(
4188 opts.max_attempts,
4189 queue,
4190 task,
4191 &runner.state,
4192 "a_before,
4193 result,
4194 );
4195 }
4196
4197 if let Some(owner) = &public_owner
4198 && (task.fresh_start || !may_resume_run(owner, task))
4199 {
4200 task.hold_machine(Some(format!("Existing PR belongs to run {}; resume its authorized instructions or explicitly resolve its owner decision before starting a successor. No attempt started.", owner.id)));
4201 record(queue, task);
4202 return Vec::new();
4203 }
4204
4205 if let Some(state) = task
4206 .execution_runs()
4207 .last()
4208 .and_then(|id| RunState::load(id).ok())
4209 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
4210 {
4211 task.hold_machine(Some(reason));
4212 record(queue, task);
4213 return Vec::new();
4214 }
4215
4216 if let Some(held) = crate::base_sync_recovery::held_run(queue, task)
4217 && !task.fresh_start
4218 && task.pr_recovery.is_none()
4219 {
4220 let authorized = if crate::repair_budget::conflict_pending(task, &held) {
4223 true
4224 } else if task.resumption.is_some() && compatible_run_instruction(&held, task) {
4225 let home = queue.root().parent().expect("queue has a home");
4226 match Runner::authorize_base_recovery_from_resumption(home, &held.id, &task.id).await {
4227 Ok(()) => true,
4228 Err(error) => {
4229 tracing::debug!(run = %held.id, %error, "base recovery resumption refused");
4230 false
4231 }
4232 }
4233 } else {
4234 false
4235 };
4236 if !authorized {
4237 task.hold_machine(Some(format!("Base synchronization remains stopped. {} A release without a new owner approval grants no additional synchronization budget and starts no replacement competition.", held.base_sync_next_action())));
4238 task.base_sync_notice(&held);
4240 task.resume_override = None;
4241 record(queue, task);
4242 return Vec::new();
4243 }
4244 }
4245 let unfinished = (!task.fresh_start)
4246 .then(|| unfinished_run(&resume_runs, task.short()))
4247 .flatten()
4248 .filter(|id| RunState::load(id).is_ok_and(|run| may_resume_run(&run, task)));
4249 let previous = resume_runs.last().and_then(|id| RunState::load(id).ok());
4250 if !task.fresh_start
4251 && task.pr_recovery.is_none()
4252 && task.review_branch.is_none()
4253 && task.resume_override.is_none()
4254 && previous
4255 .as_ref()
4256 .is_some_and(|run| run.retired_to.is_some())
4257 && unfinished.is_none()
4258 {
4259 task.hold_unrequested_retry(Some("Recorded recovery authority is unavailable, no longer resumable or no longer authorized for these instructions. Inspect the retained lineage and select a recovery source explicitly; no replacement competition was started.".into()));
4260 record(queue, task);
4261 return Vec::new();
4262 }
4263
4264 let saved_recovery = previous.as_ref().filter(|run| {
4265 task.replacement.is_none()
4266 && task.pr_recovery.as_ref().is_some_and(|selected| {
4267 selected
4268 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4269 .is_ok()
4270 && selected_starter(selected, Some(run), unfinished.as_deref())
4271 == Starter::Resume(run.id.clone())
4272 })
4273 });
4274
4275 let explicit = match &task.overrides {
4279 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
4280 Some(_) => opts.config.as_deref(),
4281 None => opts.config.as_deref(),
4282 };
4283 let (mut config, layers) = if let Some(run) = saved_recovery {
4284 let mut saved = run.clone();
4287 if let Err(e) = saved.configure_recovery_resume() {
4288 task.hold_manual(Some(format!("[recovery] saved run {} cannot resume: {e:#}. Inspect its saved configuration and evidence before requesting a new selection.", run.id)));
4289 record(queue, task);
4290 return Vec::new();
4291 }
4292 (saved.config, Vec::new())
4293 } else {
4294 let (mut config, layers) = match Config::discover_fetched(&repo, explicit).await {
4295 Ok(loaded) => loaded,
4296 Err(e) => {
4297 let reason = format!("[config] {e:#}");
4298 record_config_failure(task, &e);
4299 task.last_error = Some(reason.clone());
4300 task.hold_machine(Some(reason));
4301 record(queue, task);
4302 return Vec::new();
4303 }
4304 };
4305 if let Err(e) = apply_options(&mut config, opts, task) {
4306 task.hold_manual(Some(format!(
4307 "[config] {e:#}; correct the merge override before releasing the task"
4308 )));
4309 record(queue, task);
4310 return Vec::new();
4311 }
4312 if let Some(selected) = &task.pr_recovery
4313 && let Err(e) = selected.configure(&mut config)
4314 {
4315 task.hold_manual(Some(format!("[recovery] {e:#}. No run started; select the PR again with an explicit disposition and corrected configuration.")));
4316 record(queue, task);
4317 return Vec::new();
4318 }
4319 (config, layers)
4320 };
4321 let config_evidence = crate::pr_recovery::ConfigEvidence {
4322 layers,
4323 overrides: task.overrides.clone(),
4324 daemon_merge: task
4325 .overrides
4326 .as_ref()
4327 .is_none_or(|o| o.inherit_daemon_options)
4328 .then(|| opts.merge.clone())
4329 .flatten(),
4330 };
4331 apply_task_implementers(&mut config, task);
4332 if let Some(replacement) = &task.replacement {
4333 let safe = RunState::load(&replacement.run).is_ok_and(|old| {
4334 old.liveness(false) == Liveness::Dead
4335 && old.merge_decision.as_ref().is_some_and(|d| {
4336 d.closed
4337 && d.question == replacement.question
4338 && d.replacement_run.as_ref().is_none_or(|run| {
4339 task.execution_runs().last() == Some(run)
4340 && RunState::load(run).is_ok_and(|current| {
4341 current.replaces.as_ref().is_some_and(|pin| {
4342 pin.question == replacement.question
4343 && pin.agent == replacement.agent
4344 })
4345 })
4346 })
4347 })
4348 });
4349 if !safe {
4350 task.hold_manual(Some("Replacement ownership cannot be confirmed, or a replacement run was already minted before a crash. Inspect the task and its linked runs; no second replacement was started.".into()));
4351 record(queue, task);
4352 return Vec::new();
4353 }
4354 match config.agent(&replacement.agent) {
4355 Ok(agent) if crate::agent::installed(agent) => {}
4356 _ => {
4357 task.hold_manual(Some(format!("Selected implementer `{}` is unavailable or no longer configured. No substitute was dispatched. Restore it and explicitly release the task.", replacement.agent)));
4358 record(queue, task);
4359 return Vec::new();
4360 }
4361 }
4362 config.graph.implementers = 1;
4363 config.graph.land_approval = true;
4364 config.merge.mode = MergeMode::Pr;
4365 config.graph.selected_implementer = Some(replacement.agent.clone());
4366 }
4367 let p = phrases(&config.graph.language);
4368 let start_failed = p.could_not_start;
4369 if let Ok(base) =
4374 crate::git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
4375 .await
4376 && let Ok(head) =
4377 crate::git::rev_parse(&repo, &format!("{}/{}", config.merge.remote, base)).await
4378 {
4379 let mut inputs = crate::queue::recovery_inputs(&task.sources);
4380 if let Some(branch) = task.review_of.as_ref().or(task.review_branch.as_ref()) {
4381 let source = crate::git::rev_parse(&repo, branch)
4382 .await
4383 .unwrap_or_default();
4384 inputs.push_str(&format!(";review:{branch}:{source}"));
4385 }
4386 if let Some(reason) = task
4387 .recovery_exhausted(
4388 &head,
4389 &inputs,
4390 task.attempt_limit
4391 .map_or(opts.max_attempts, std::num::NonZeroUsize::get),
4392 )
4393 .filter(|_| {
4394 !task.repair_grants.last().is_some_and(|receipt| {
4395 RunState::load(&receipt.request.run)
4396 .is_ok_and(|run| crate::repair_budget::pending(task, &run))
4397 })
4398 })
4399 {
4400 task.hold_unrequested_retry(Some(reason));
4401 record(queue, task);
4402 return Vec::new();
4403 }
4404 task.recovery_admission = Some(crate::queue::RecoveryAdmission { base: head, inputs });
4405 }
4406
4407 if let Some(reason) = disk_gate(&repo, &config) {
4415 task.notice_cause("disk-space", None, None);
4416 task.last_error = Some(reason.clone());
4417 task.hold_machine(Some(reason.clone()));
4418 record(queue, task);
4419 tracing::warn!("holding {} for want of disk space: {reason}", task.short());
4420 notices::raise(
4423 Notice::warn(
4424 &format!("disk:{}", repo.display()),
4425 format!("Task {} held for want of free disk space: {reason}. Free some, then release it from the queue.", task.id),
4426 )
4427 .link(Link::Task {
4428 id: task.id.clone(),
4429 }),
4430 );
4431 return Vec::new();
4432 }
4433
4434 let review_branch = if previous
4459 .as_ref()
4460 .is_some_and(|state| crate::repair_budget::pending(task, state))
4461 {
4462 None
4463 } else {
4464 take_review_branch(task, previous.as_ref())
4465 };
4466 let branch_exists = match &review_branch {
4467 Some(branch) => crate::git::branch_exists(&repo, branch)
4468 .await
4469 .unwrap_or(false),
4470 None => false,
4471 };
4472 let proposed = review_retry_starter(
4476 task,
4477 previous.as_ref(),
4478 review_branch.as_deref(),
4479 branch_exists,
4480 unfinished.as_deref(),
4481 );
4482 let retry_selected = task.replacement.is_some()
4483 && task.resumption.is_some()
4484 && !task.fresh_start
4485 && previous.as_ref().is_some_and(|run| {
4486 run.replaces.is_some()
4487 && ((run.status == RunStatus::Failed && run.viable().is_empty())
4488 || (run.status == RunStatus::Blocked && run.tally.is_none()))
4489 });
4490 let stop_pin = task
4491 .resume_override
4492 .as_ref()
4493 .filter(|pin| {
4494 Questions::open().get(&pin.question_id).is_ok_and(|q| {
4495 q.node == land::APPROVAL_NODE
4496 && q.resolution()
4497 .as_deref()
4498 .is_some_and(crate::merge_decision::stops)
4499 })
4500 })
4501 .and_then(|pin| pin.pinned_run.clone());
4502 let granted_run = task.repair_grants.last().and_then(|receipt| {
4503 let state = RunState::load(&receipt.request.run).ok()?;
4504 crate::repair_budget::pending(task, &state).then(|| receipt.request.run.clone())
4505 });
4506 let starter = if let Some(run) = granted_run {
4507 Starter::Resume(run)
4508 } else if let Some(owner) = &public_owner {
4509 Starter::Resume(owner.id.clone())
4510 } else if let Some(run) = stop_pin {
4511 Starter::Resume(run)
4514 } else if retry_selected {
4515 Starter::Resume(previous.as_ref().unwrap().id.clone())
4516 } else if let Some(selected) = &task.pr_recovery {
4517 if selected
4518 .check_task_assignment(&task.id, &task.instruction, task.reassignments.len())
4519 .is_ok()
4520 {
4521 selected_starter(selected, previous.as_ref(), unfinished.as_deref())
4522 } else {
4523 Starter::Review(selected.branch.clone())
4524 }
4525 } else {
4526 starter_with_implementers(task, previous.as_ref(), proposed)
4527 };
4528 if let Some(pin) = &task.resume_override
4531 && let Some(run) = &pin.pinned_run
4532 && starter != Starter::Resume(run.clone())
4533 {
4534 let run = run.clone();
4535 task.notice_cause("resume-owner", Some(&run), None);
4536 task.hold_machine(Some(format!(
4537 "pinned run cannot resume; inspect `magi show {run}` and restore the existing run before `magi task release {}`; no replacement task was started",
4538 task.id
4539 )));
4540 record(queue, task);
4541 return Vec::new();
4542 }
4543 if task.replacement.is_some()
4544 && previous.as_ref().is_some_and(|run| run.replaces.is_some())
4545 && starter != Starter::Resume(previous.as_ref().unwrap().id.clone())
4546 {
4547 task.hold_manual(Some("The selected replacement already has an attempt. It cannot be resumed safely; inspect its saved pin and restore the existing run. No additional replacement was started.".into()));
4548 record(queue, task);
4549 return Vec::new();
4550 }
4551 if let Some(reason) = retained_attempt_hold(
4552 task,
4553 &starter,
4554 config.disk.retained_attempt_limit,
4555 RunState::load,
4556 ) {
4557 if let Starter::Review(branch) = &starter {
4560 task.review_branch = Some(branch.clone());
4561 }
4562 task.hold_unrequested_retry(Some(reason));
4563 record(queue, task);
4564 return Vec::new();
4565 }
4566 let attachments = match task_attachments(queue, task) {
4571 Ok(a) => a,
4572 Err(e) => {
4573 task.attempts += 1;
4574 task.fail_startup_target(
4575 "startup-attachments",
4576 e.name.clone(),
4577 format!("{start_failed}{e:#}"),
4578 opts.max_attempts,
4579 );
4580 record(queue, task);
4581 return Vec::new();
4582 }
4583 };
4584 let startup_pause = stop.pause();
4585 let startup_interrupt = interrupt_pause.clone();
4586 let wait = crate::repo_lock::WaitPolicy::new(
4587 Duration::from_secs(config.graph.timeout_resource_wait),
4588 move || startup_pause.parked() || startup_interrupt.parked(),
4589 );
4590 let started = wait.scope(Box::pin(async { match &starter {
4593 Starter::Review(_) if task.pr_recovery.is_some() => {
4594 let selected = task.pr_recovery.clone().expect("selected recovery");
4595 match selected.check_task_assignment(&task.id, &task.instruction, task.reassignments.len()) {
4596 Ok(()) => Runner::recover_pr(&repo, &task.id, selected, config)
4597 .await
4598 .map(|mut r| {
4599 r.state.attachments = attachments.clone();
4600 r
4601 }),
4602 Err(e) => Err(e),
4603 }
4604 }
4605 Starter::Review(branch) => {
4606 tracing::info!(
4607 "task {} reopens `{branch}` as a review-only pass",
4608 task.short()
4609 );
4610 let takeover = crate::handover::Takeover {
4613 earlier: task.earlier_attempts().to_vec(),
4614 home: crate::run::home(),
4615 choice: take_divergence_answer(branch, &config.merge.remote, task),
4616 task_instruction: Some(instruction_for(task)),
4617 };
4618 Runner::review_taking_over(
4619 &repo,
4620 branch,
4621 config,
4622 Some(takeover),
4623 crate::run::Origin::queue(&task.id),
4624 )
4625 .await
4626 }
4627 Starter::Resume(id) => {
4628 tracing::info!("resuming run {id} rather than competing again");
4629 Runner::resume(id).and_then(|mut r| {
4630 if retry_selected {
4631 for candidate in &mut r.state.candidates {
4634 candidate.failed = None;
4635 candidate.empty = false;
4636 candidate.commits = 0;
4637 }
4638 r.state.status = RunStatus::Implementing;
4639 r.state.event("implement", "owner released the held selected implementer; retrying the same attempt without fallback");
4640 }
4641 if r.state.prep_complete == Some(false) && r.state.candidates.is_empty() {
4642 r.state.sources = task.sources.clone();
4643 }
4644 if r.state.prep_failure.is_some() {
4645 let approval = task
4646 .resumption
4647 .as_ref()
4648 .context("preparation retry requires an explicit task release")?;
4649 r.retry_preparation(approval.at)?;
4650 }
4651 if let Some(instruction) =
4652 prepare_instruction(&starter, Some(&r.state.instruction), task)
4653 {
4654 r.state.instruction = instruction;
4655 }
4656 r.state.attachments = attachments.clone();
4657 if task.replacement.is_some() {
4658 r.state.config.merge.mode = MergeMode::Pr;
4659 r.state.config.graph.land_approval = true;
4660 }
4661 if task.replacement.is_none() && task.pr_recovery.is_none() && let Some(mode) = task
4665 .overrides
4666 .as_ref()
4667 .and_then(|o| o.merge.as_deref())
4668 .and_then(|m| merge_mode(m).ok())
4669 {
4670 r.state.config.merge.mode = mode;
4671 }
4672 Ok(r)
4673 })
4674 }
4675 Starter::Start => {
4676 if let Some(branch) = &review_branch {
4677 tracing::warn!(
4678 "conductor chose review for task {} but branch `{branch}` no longer \
4679 exists; requeuing as a fresh competition instead",
4680 task.short()
4681 );
4682 }
4683 let instruction = prepare_instruction(&starter, None, task)
4684 .unwrap_or_else(|| task.instruction.clone());
4685 Runner::start_naming(
4686 &repo,
4687 instruction,
4688 &task.title,
4689 config,
4690 crate::run::Origin::queue(&task.id),
4691 )
4692 .await
4693 .map(|mut r| {
4694 r.state.attachments = attachments.clone();
4695 r.state.sources = task.sources.clone();
4696 r
4697 })
4698 }
4699 }})).await;
4700 if let Err(error) = &started
4701 && let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>()
4702 {
4703 task.record_startup_failure(failure.clone());
4704 }
4705 let mut runner = match started {
4706 Ok(r) => r,
4707 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4708 task.last_error = Some(e.to_string());
4709 if stop.parking() || interrupt_pause.parked() {
4710 let attempts = task.attempts;
4711 task.park(e.to_string());
4712 task.attempts = attempts;
4714 } else {
4715 task.hold_machine(Some(format!("{}{e}", crate::cache::RESOURCE_HOLD)));
4716 }
4717 record(queue, task);
4718 return Vec::new();
4719 }
4720 Err(e) if task.pr_recovery.is_some() => {
4724 task.startup_notice_cause("startup-public-head");
4725 task.attempts += 1;
4726 task.last_error = Some(format!("{start_failed}{e:#}"));
4727 task.hold_unrequested_retry(Some(format!(
4728 "Selected public-head recovery could not start: {e:#}. Correct the cause and use `magi task recover-pr` with the approved public head; automatic recovery is stopped."
4729 )));
4730 record(queue, task);
4731 return Vec::new();
4732 }
4733 Err(e) if e.downcast_ref::<crate::handover::Refused>().is_some() => {
4734 let detail = match e.downcast_ref::<crate::handover::Refused>() {
4735 Some(r) => (p.handover_refused)(r),
4736 None => format!("{e:#}"),
4737 };
4738 task.startup_notice_cause("startup-handover");
4739 let reason = format!("{start_failed}{detail}");
4740 task.last_error = Some(reason.clone());
4741 let branch = match &starter {
4744 Starter::Review(branch) => Some(branch.clone()),
4745 _ => None,
4746 };
4747 task.hold_for_handover(branch, format!("{reason}{}", p.handover_hint));
4751 record(queue, task);
4752 tracing::warn!(
4753 "holding {} for a branch it cannot take over: {e:#}",
4754 task.short()
4755 );
4756 return Vec::new();
4757 }
4758 Err(e) if e.downcast_ref::<crate::reconcile::Diverged>().is_some() => {
4763 let d = e
4764 .downcast_ref::<crate::reconcile::Diverged>()
4765 .expect("checked by the guard");
4766 let mut q = ask::Question::new(
4767 task.id.clone(),
4768 crate::reconcile::NODE.to_owned(),
4769 crate::reconcile::SEAT.to_owned(),
4770 d.summary(),
4771 d.detail(),
4772 d.choices(),
4773 );
4774 match Questions::open().put(&mut q) {
4775 Ok(()) => {
4776 task.last_error = Some(format!("{e:#}"));
4777 task.review_branch = Some(d.branch.clone());
4780 task.review_instruction = Some(task.instruction.clone());
4781 task.block(vec![q.id.clone()], Some(d.summary()));
4782 }
4783 Err(put) => {
4784 tracing::warn!("could not file the divergence question: {put:#}");
4785 task.attempts += 1;
4786 task.fail_startup(
4787 "startup-question-store",
4788 format!("{start_failed}{e:#}"),
4789 opts.max_attempts,
4790 );
4791 }
4792 }
4793 record(queue, task);
4794 return Vec::new();
4795 }
4796 Err(e) => {
4797 if e.downcast_ref::<crate::reconcile::Stale>().is_some()
4803 && let Starter::Review(branch) = &starter
4804 {
4805 task.review_branch = Some(branch.clone());
4806 task.review_instruction = Some(task.instruction.clone());
4807 task.last_error = Some(format!("{start_failed}{e:#}"));
4808 task.status = crate::queue::TaskStatus::Failed;
4809 record(queue, task);
4810 return Vec::new();
4811 }
4812 task.attempts += 1;
4813 task.last_error = Some(format!("{start_failed}{e:#}"));
4814 task.startup_notice_cause("startup-source");
4815 task.hold_unrequested_retry(Some(format!(
4816 "Startup failed: {e:#}. Correct the startup source or configuration, then explicitly release the task; automatic recovery is stopped."
4817 )));
4818 record(queue, task);
4819 return Vec::new();
4820 }
4821 };
4822 if task.pr_recovery.is_some() && !matches!(starter, Starter::Resume(_)) {
4823 runner.state.recovery_config = Some(config_evidence);
4824 }
4825 if let Some(replacement) = &task.replacement {
4826 if matches!(starter, Starter::Start) {
4827 runner.state.instruction.push_str(&format!(
4828 "\n\n# Replacement review context\n{}",
4829 replacement.context
4830 ));
4831 }
4832 runner.state.replaces = Some(replacement.clone());
4833 runner.state.config.graph.selected_implementer = Some(replacement.agent.clone());
4834 if let Ok(mut old) = RunState::load(&replacement.run)
4835 && let Some(d) = &mut old.merge_decision
4836 {
4837 d.replacement_run = Some(runner.state.id.clone());
4838 old.event(
4839 "replacement",
4840 format!(
4841 "replacement run {} owns the next attempt; old PR remains closed",
4842 runner.state.id
4843 ),
4844 );
4845 if let Err(e) = old.save() {
4846 let error = runner.fail_startup("replacement-history", e);
4847 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4848 task.record_startup_failure(failure.clone());
4849 }
4850 task.hold_manual(Some(format!(
4851 "Could not link replacement history: {error:#}; no agents dispatched"
4852 )));
4853 record(queue, task);
4854 return Vec::new();
4855 }
4856 }
4857 }
4858 runner.state.task_instruction = Some(task.instruction.clone());
4859 runner.state.followup_generation = Some(task.followup.as_ref().map_or(0, |f| f.generation));
4862 if runner.state.origin_chat.is_none() {
4863 runner.state.origin_chat = task.chat_talk().map(str::to_owned);
4864 }
4865 runner.on_pause(stop.pause());
4867 runner.watch_interrupt(interrupt_pause);
4871
4872 let run = runner.state.id.clone();
4875 let before_start = task.clone();
4876 task.start(run.clone());
4877 if let Err(error) = queue.put(task) {
4878 *task = before_start;
4879 let error = runner.fail_startup("task-start", error);
4880 if let Some(failure) = error.downcast_ref::<crate::run::StartupFailure>() {
4881 task.record_startup_failure(failure.clone());
4882 }
4883 task.hold_manual(Some(format!(
4884 "Could not persist task start; no agents dispatched: {error:#}"
4885 )));
4886 record(queue, task);
4887 return Vec::new();
4888 }
4889 lock(status).current.push(Current {
4890 worker: None,
4891 task: task.id.clone(),
4892 run,
4893 });
4894
4895 let quota_before = runner.state.quota.clone();
4898 runner.state.next_pass_task_entry = true;
4899 let result = runner.execute().await;
4900 finish_attempt(
4901 opts.max_attempts,
4902 queue,
4903 task,
4904 &runner.state,
4905 "a_before,
4906 result,
4907 )
4908}
4909
4910pub fn finish_attempt(
4915 max_attempts: usize,
4916 queue: &Queue,
4917 task: &mut Task,
4918 state: &RunState,
4919 quota_before: &[QuotaLoss],
4920 result: Result<()>,
4921) -> Vec<QuotaLoss> {
4922 let detail = match result {
4923 Ok(()) => describe(state),
4924 Err(e) => format!("{e:#}"),
4925 };
4926 let fresh = losses_this_attempt(quota_before, &state.quota);
4927 let verdict = Verdict {
4928 status: state.status,
4929 left_pr: state.pr.is_some(),
4932 quota_hit: !fresh.is_empty(),
4938 parked: state.parked,
4942 no_viable_candidates: state.viable().is_empty(),
4945 };
4946 if let Some(replacement) = &state.replaces
4947 && let Some(pr) = &state.pr
4948 && let Ok(mut old) = RunState::load(&replacement.run)
4949 && let Some(d) = &mut old.merge_decision
4950 {
4951 d.replacement_pr = Some(pr.url.clone());
4952 let _ = old.save();
4953 }
4954 settle_and_diagnose(task, verdict, &detail, max_attempts, state);
4955 if state.status == RunStatus::Blocked && state.prep_failure.is_some() {
4956 task.hold_unrequested_retry(Some(format!(
4957 "Candidate setup failed: {detail}. Correct the explicit sources with `magi task sources {}` (README, \"Use\": implementation imports), or select a complete implementation with `magi task recover-pr`, then explicitly release the task.",
4958 task.short()
4959 )));
4960 }
4961 if task.status == TaskStatus::Done {
4962 supersede_prior_runs(task, &crate::run::home());
4963 }
4964 record(queue, task);
4965 tracing::info!(
4966 "task {} is {} after run {} ({})",
4967 task.short(),
4968 task.status.as_str(),
4969 state.short(),
4970 label(state.status)
4971 );
4972 fresh
4973}
4974
4975pub fn hold_if_runnable(queue: &Queue, task: &mut Task) {
4979 if matches!(task.status, TaskStatus::Queued | TaskStatus::Failed)
4982 && !awaiting_resume_with(task, RunState::load)
4983 && withheld_text_wait(task).is_none()
4987 && !(task.fresh_start && task.replacement.is_some())
4988 {
4989 let why = task.last_error.clone().map_or_else(
4990 || "the run did not finish".to_owned(),
4991 |e| format!("the run did not finish: {e}"),
4992 );
4993 task.hold_unrequested_retry(Some(format!(
4994 "{why}. It was started by hand, so it is not retried \
4995 automatically; `magi task release` retries it."
4996 )));
4997 record(queue, task);
4998 }
4999}
5000
5001#[must_use]
5005pub fn loop_would_resume(run: &str) -> bool {
5006 unfinished_run(&[run.to_owned()], crate::run::short_of(run)).is_some()
5007}
5008
5009#[must_use]
5019pub fn foreign_loop(
5020 reading: Option<&Reading>,
5021 now: Timestamp,
5022 own_pid: u32,
5023) -> Option<Option<u32>> {
5024 let reading = reading.filter(|r| r.running(now))?;
5025 match reading.pid {
5026 Some(pid) if pid == own_pid => None,
5027 pid => Some(pid),
5028 }
5029}
5030
5031fn legacy_scheduler_running(reading: &Reading, now: Timestamp, own_pid: u32) -> bool {
5032 reading.worker_protocol != Some(1)
5033 && (foreign_loop(Some(reading), now, own_pid).is_some()
5034 || reading
5035 .pid
5036 .is_some_and(|pid| pid != own_pid && crate::proc::pid_alive(pid)))
5037}
5038
5039pub async fn run_claimed(opts: &Opts, queue: &Queue, task: &mut Task) {
5050 run_claimed_until(opts, queue, task, &Stop::new()).await;
5051}
5052
5053async fn run_claimed_until(
5054 opts: &Opts,
5055 queue: &Queue,
5056 task: &mut Task,
5057 stop: &Stop,
5058) -> Vec<QuotaLoss> {
5059 let status = Arc::new(Mutex::new(Status::new()));
5060 let mut quota = Box::pin(attempt(
5063 opts,
5064 queue,
5065 &status,
5066 stop,
5067 crate::graph::Pause::new(),
5068 task,
5069 ))
5070 .await;
5071 let mut resumes = 0;
5080 while resumes < MAX_WITHHELD_RESUMES && !stop.parking() {
5081 let Some(timeout) = withheld_text_wait(task) else {
5082 break;
5083 };
5084 eprintln!(
5085 "waiting for the owner's answer on a withheld pull request text \
5086 (up to {timeout}s; Ctrl-C leaves the run parked for a later `magi serve`)"
5087 );
5088 let limit = std::time::Instant::now() + Duration::from_secs(timeout + 60);
5091 let ready = loop {
5092 match land_resume_state(task) {
5093 LandResume::StillWaiting
5094 if !stop.parking() && std::time::Instant::now() < limit =>
5095 {
5096 tokio::time::sleep(WITHHELD_POLL).await;
5097 }
5098 LandResume::Ready => break true,
5099 _ => break false,
5100 }
5101 };
5102 if !ready || stop.parking() {
5103 break;
5104 }
5105 resumes += 1;
5106 quota.extend(
5107 Box::pin(attempt(
5108 opts,
5109 queue,
5110 &status,
5111 stop,
5112 crate::graph::Pause::new(),
5113 task,
5114 ))
5115 .await,
5116 );
5117 }
5118 hold_if_runnable(queue, task);
5119 quota
5120}
5121
5122const MAX_WITHHELD_RESUMES: u32 = 3;
5125
5126const WITHHELD_POLL: Duration = Duration::from_secs(15);
5128
5129fn withheld_text_wait(task: &Task) -> Option<u64> {
5132 let id = task.execution_runs().last()?;
5133 let s = RunState::load(id).ok()?;
5134 (s.parked && s.github_text.as_ref().is_some_and(|g| !g.resolved))
5135 .then_some(s.config.graph.answer_timeout)
5136}
5137
5138fn losses_this_attempt(before: &[QuotaLoss], after: &[QuotaLoss]) -> Vec<QuotaLoss> {
5148 after
5149 .iter()
5150 .filter(|q| !before.contains(q))
5151 .cloned()
5152 .collect()
5153}
5154
5155fn cooldown_until(quota: &[QuotaLoss], now: Timestamp) -> Option<Timestamp> {
5158 let quota: Vec<_> = quota
5159 .iter()
5160 .filter(|q| q.cause == crate::agent::QuotaCause::UsageLimit)
5161 .collect();
5162 if quota.is_empty() {
5163 return None;
5164 }
5165 let with_hint = quota.iter().find(|q| q.reset.is_some());
5166 let reset_at = with_hint.and_then(|q| parse_reset_hint(q.reset.as_deref()?, now, q.at));
5167 let wait = quota_wait(reset_at, now, QUOTA_WAIT_FALLBACK, QUOTA_WAIT_CAP);
5168 let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
5169 Some(
5170 now.checked_add(jiff::SignedDuration::from_secs(secs))
5171 .unwrap_or(Timestamp::MAX),
5172 )
5173}
5174
5175fn implementers_match(task: &Task, state: &RunState) -> bool {
5177 task.replacement.as_ref().map_or_else(
5178 || {
5179 task.implementers()
5180 .is_none_or(|n| n == state.config.graph.implementers)
5181 },
5182 |replacement| {
5183 state.config.graph.selected_implementer.as_deref() == Some(&replacement.agent)
5184 && state.config.graph.implementers == 1
5185 },
5186 )
5187}
5188
5189fn starter_with_implementers(
5191 task: &Task,
5192 previous: Option<&RunState>,
5193 proposed: Starter,
5194) -> Starter {
5195 if previous.is_some_and(|s| !implementers_match(task, s)) {
5196 task.review_of
5197 .as_ref()
5198 .map_or(Starter::Start, |branch| Starter::Review(branch.clone()))
5199 } else {
5200 proposed
5201 }
5202}
5203
5204fn apply_task_implementers(config: &mut Config, task: &Task) {
5206 if let Some(n) = task.implementers() {
5207 config.graph.implementers = n;
5208 }
5209}
5210
5211#[cfg(test)]
5216fn prepare_for(repo: &Path, opts: &Opts, task: &Task) -> Result<Config> {
5217 let explicit = match &task.overrides {
5218 Some(o) if !o.inherit_daemon_options => o.config.as_deref(),
5219 _ => opts.config.as_deref(),
5220 };
5221 let (mut config, _) = Config::discover(repo, explicit)?;
5222 apply_options(&mut config, opts, task)?;
5223 Ok(config)
5224}
5225
5226fn apply_options(config: &mut Config, opts: &Opts, task: &Task) -> Result<()> {
5227 if task
5228 .overrides
5229 .as_ref()
5230 .is_none_or(|o| o.inherit_daemon_options)
5231 && let Some(mode) = opts.merge.as_deref()
5232 {
5233 config.merge.mode = merge_mode(mode)?;
5234 }
5235 if let Some(o) = &task.overrides {
5236 o.apply(config);
5237 }
5238 Ok(())
5239}
5240
5241fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
5243 let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
5244 if let Some(mode) = &opts.merge {
5245 config.merge.mode = merge_mode(mode)?;
5246 }
5247 Ok(config)
5248}
5249
5250async fn maybe_prune_cache_between_runs(
5255 repo: &Path,
5256 opts: &Opts,
5257 home: &Path,
5258 stop: &Stop,
5259 last_checked: &mut Option<Timestamp>,
5260 now: Timestamp,
5261) {
5262 if stop.stopped() || !cache_check_due(*last_checked, now, CACHE_CHECK_INTERVAL_SECS) {
5263 return;
5264 }
5265 *last_checked = Some(now);
5266 let cfg = match prepare(repo, opts) {
5267 Ok(cfg) => cfg,
5268 Err(e) => {
5269 tracing::warn!("cache check: no config: {e:#}");
5270 return;
5271 }
5272 };
5273 match clean::prune_cache_if_over_limit(&cfg, home) {
5274 Ok(Some(pruned)) if pruned.files > 0 => tracing::info!(
5275 "housekeep: pruned {} file(s) ({} bytes) from the shared cache between runs",
5276 pruned.files,
5277 pruned.freed
5278 ),
5279 Ok(_) => {}
5280 Err(e) => {
5281 tracing::warn!("housekeep: prune cache: {e:#}");
5282 notices::raise_in(
5283 home,
5284 Notice::warn(
5285 "housekeep:cache",
5286 format!(
5287 "Pruning the shared build cache failed: {}. Disk usage may keep growing.",
5288 e.to_string()
5289 .lines()
5290 .next()
5291 .unwrap_or("unknown cleanup error")
5292 ),
5293 ),
5294 );
5295 }
5296 }
5297}
5298
5299fn cache_check_due(last_checked: Option<Timestamp>, now: Timestamp, interval_secs: u64) -> bool {
5303 last_checked.is_none_or(|last| clean::due(now, last, interval_secs))
5304}
5305
5306async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
5325 let cfg = match prepare(repo, opts) {
5326 Ok(cfg) => cfg,
5327 Err(e) => {
5328 tracing::warn!("housekeep: no config: {e:#}");
5329 return;
5330 }
5331 };
5332 let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
5341 let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
5342 if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
5347 let mut extra = Vec::new();
5348 if out.unreadable > 0 {
5349 extra.push(format!("{} unreadable", out.unreadable));
5350 }
5351 if out.orphaned_worktrees > 0 {
5352 extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
5353 }
5354 let detail = if extra.is_empty() {
5355 String::new()
5356 } else {
5357 format!(" ({})", extra.join(", "))
5358 };
5359 tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
5360 }
5361 if out.external_merges_recorded > 0 {
5362 tracing::info!(
5363 "housekeep: recorded {} run(s) as merged externally",
5364 out.external_merges_recorded
5365 );
5366 }
5367 if out.stale_pr_states_repaired > 0 {
5368 tracing::info!(
5369 "housekeep: rewrote {} run record(s) whose pull request had already settled",
5370 out.stale_pr_states_repaired
5371 );
5372 }
5373 if out.cache_files > 0 {
5374 tracing::info!(
5375 "housekeep: pruned {} file(s) ({} bytes) from the shared cache",
5376 out.cache_files,
5377 out.cache_freed
5378 );
5379 }
5380 if out.questions_abandoned > 0 {
5381 tracing::info!(
5382 "housekeep: abandoned {} question(s) left open by a finished run",
5383 out.questions_abandoned
5384 );
5385 }
5386}
5387
5388async fn triage_held(queue: &Queue, home: &Path, opts: &Opts) {
5397 let questions = Questions::at(home.join("questions"));
5398 let report = triage::run_once(queue, &questions, opts.config.as_deref(), Timestamp::now());
5399 if report.is_empty() {
5400 return;
5401 }
5402 if !report.quarantined.is_empty() {
5403 tracing::info!(
5404 "triage: held {} blocked task(s) whose blocked-on task or \
5405 question no longer exists: {}",
5406 report.quarantined.len(),
5407 report.quarantined.join(", ")
5408 );
5409 }
5410 if !report.resumed.is_empty() {
5411 tracing::info!(
5412 "triage: resumed {} held task(s) whose machine hold had resolved: {}",
5413 report.resumed.len(),
5414 report.resumed.join(", ")
5415 );
5416 }
5417 if !report.asked.is_empty() {
5418 tracing::info!(
5419 "triage: asked about {} held task(s): {}",
5420 report.asked.len(),
5421 report.asked.join(", ")
5422 );
5423 }
5424 if !report.answered.is_empty() {
5425 tracing::info!(
5426 "triage: applied {} operator answer(s): {}",
5427 report.answered.len(),
5428 report.answered.join(", ")
5429 );
5430 }
5431}
5432
5433pub(crate) fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
5440 disk_gate_with(repo, config, crate::disk::free_bytes)
5441}
5442
5443fn disk_gate_with<F: Fn(&Path) -> Result<u64>>(
5447 repo: &Path,
5448 config: &Config,
5449 free_bytes: F,
5450) -> Option<String> {
5451 let min = config.disk.min_free_bytes;
5452 if min == 0 {
5453 return None;
5454 }
5455 let mut paths = vec![repo.to_owned()];
5456 if let Some(cache) = config.cache_dir() {
5457 paths.push(cache);
5458 }
5459 if let Some(root) = &config.graph.worktree_root {
5460 paths.push(root.clone());
5461 }
5462 for path in paths {
5463 let probe = if path == repo {
5464 path.as_path()
5465 } else {
5466 path.ancestors().find(|p| p.exists()).unwrap_or(&path)
5467 };
5468 let reason = match free_bytes(probe) {
5469 Ok(free) => crate::disk::gate_in(free, min, &config.graph.language),
5470 Err(e) => Some(crate::disk::unmeasured_in(
5471 probe,
5472 &e,
5473 &config.graph.language,
5474 )),
5475 };
5476 if reason.is_some() {
5477 return reason;
5478 }
5479 }
5480 None
5481}
5482
5483const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
5490
5491const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
5495
5496fn quota_wait(
5505 reset_at: Option<Timestamp>,
5506 now: Timestamp,
5507 fallback: Duration,
5508 cap: Duration,
5509) -> Duration {
5510 match reset_at {
5511 Some(at) if at > now => {
5512 let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
5513 Duration::from_secs(secs).min(cap)
5514 }
5515 _ => fallback,
5516 }
5517}
5518
5519fn parse_reset_hint(text: &str, now: Timestamp, recorded: Timestamp) -> Option<Timestamp> {
5533 parse_reset_hint_zoned(text, now)
5534 .or_else(|| parse_reset_hint_dated(text))
5535 .or_else(|| parse_reset_hint_relative(text, recorded))
5536}
5537
5538fn parse_reset_hint_relative(text: &str, recorded: Timestamp) -> Option<Timestamp> {
5542 let rest = text.trim().trim_end_matches('.').strip_prefix("in ")?;
5543 let mut rest = rest.trim();
5544 if rest.is_empty() {
5545 return None;
5546 }
5547 let mut total: i64 = 0;
5548 let mut matched = false;
5549 for (unit, secs) in [('h', 3600), ('m', 60), ('s', 1)] {
5550 if let Some((digits, tail)) = rest.split_once(unit)
5551 && !digits.is_empty()
5552 && digits.bytes().all(|b| b.is_ascii_digit())
5553 {
5554 total += digits.parse::<i64>().ok()?.checked_mul(secs)?;
5555 rest = tail;
5556 matched = true;
5557 }
5558 }
5559 if !rest.is_empty() || !matched {
5560 return None;
5561 }
5562 recorded
5563 .checked_add(jiff::SignedDuration::from_secs(total))
5564 .ok()
5565}
5566
5567fn parse_12h_clock(clock: &str) -> Option<(i8, i8)> {
5571 let clock = clock.trim().to_lowercase();
5572 let (digits, pm) = clock
5573 .strip_suffix("am")
5574 .map(|d| (d, false))
5575 .or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
5576 let (h, m) = digits.trim().split_once(':')?;
5577 let mut hour: i8 = h.trim().parse().ok()?;
5578 let minute: i8 = m.trim().parse().ok()?;
5579 if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
5580 return None;
5581 }
5582 if pm && hour != 12 {
5583 hour += 12;
5584 } else if !pm && hour == 12 {
5585 hour = 0;
5586 }
5587 Some((hour, minute))
5588}
5589
5590fn parse_reset_hint_zoned(text: &str, now: Timestamp) -> Option<Timestamp> {
5595 let open = text.find('(')?;
5596 let close = text.rfind(')')?;
5597 if close <= open {
5598 return None;
5599 }
5600 let zone = text[open + 1..close].trim();
5601 let (hour, minute) = parse_12h_clock(&text[..open])?;
5602 let tz = jiff::tz::TimeZone::get(zone).ok()?;
5603 let candidate = now
5604 .to_zoned(tz)
5605 .with()
5606 .hour(hour)
5607 .minute(minute)
5608 .second(0)
5609 .millisecond(0)
5610 .microsecond(0)
5611 .nanosecond(0)
5612 .build()
5613 .ok()?;
5614 let mut at = candidate.timestamp();
5615 if at <= now {
5616 at += jiff::SignedDuration::from_hours(24);
5617 }
5618 Some(at)
5619}
5620
5621fn parse_reset_hint_dated(text: &str) -> Option<Timestamp> {
5630 let words: Vec<&str> = text.split_whitespace().collect();
5631 if words.len() < 5 {
5632 return None;
5633 }
5634 (0..=words.len() - 5)
5635 .find_map(|start| parse_dated_window(&words[start..start + 5], words.get(start + 5)))
5636}
5637
5638fn parse_dated_window(window: &[&str], trailing: Option<&&str>) -> Option<Timestamp> {
5644 if trailing.is_some_and(|next| next.starts_with('(')) {
5645 return None;
5646 }
5647 let month = month_number(window[0])?;
5648 let day_token = window[1].strip_suffix(',')?.to_lowercase();
5649 let day_digits = ["st", "nd", "rd", "th"]
5650 .iter()
5651 .find_map(|suffix| day_token.strip_suffix(*suffix))?;
5652 let day: i8 = day_digits.parse().ok()?;
5653 let year_token = window[2];
5654 if year_token.len() != 4 || !year_token.bytes().all(|b| b.is_ascii_digit()) {
5655 return None;
5656 }
5657 let year: i16 = year_token.parse().ok()?;
5658 let ampm = window[4].trim_matches(|c: char| !c.is_ascii_alphabetic());
5662 let (hour, minute) = parse_12h_clock(&format!("{}{}", window[3], ampm))?;
5663 let date = jiff::civil::Date::new(year, month, day).ok()?;
5664 let candidate = date
5665 .at(hour, minute, 0, 0)
5666 .to_zoned(jiff::tz::TimeZone::UTC)
5667 .ok()?;
5668 Some(candidate.timestamp())
5669}
5670
5671fn month_number(name: &str) -> Option<i8> {
5674 const NAMES: [&str; 12] = [
5675 "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
5676 ];
5677 let lower = name.to_lowercase();
5678 NAMES
5679 .iter()
5680 .position(|n| *n == lower.as_str())
5681 .map(|i| i as i8 + 1)
5682}
5683
5684#[cfg(test)]
5686fn exhausted_review_budget(state: &RunState) -> bool {
5687 !crate::handover::can_resume(state)
5690 && state.status == RunStatus::Blocked
5691 && !state.released()
5692 && state.retired_to.is_none()
5693}
5694
5695fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
5732 unfinished_run_with(runs, short, RunState::load)
5733}
5734
5735pub(crate) fn unfinished_run_with<F>(runs: &[String], short: &str, mut load: F) -> Option<String>
5738where
5739 F: FnMut(&str) -> Result<RunState>,
5740{
5741 let mut id = runs.last()?.clone();
5742 let mut visited = std::collections::BTreeSet::new();
5743 let mut repo = None;
5744 loop {
5745 if !runs.contains(&id) || !visited.insert(id.clone()) {
5746 tracing::warn!("invalid or cyclic recovery authority for task {short}");
5747 return None;
5748 }
5749 let state = match load(&id) {
5750 Ok(state) if state.id == id => state,
5751 Ok(_) => return None,
5752 Err(e) => {
5753 tracing::warn!("could not read run {id} for task {short}: {e:#}");
5754 return None;
5755 }
5756 };
5757 if repo.as_ref().is_some_and(|repo| *repo != state.repo) {
5758 return None;
5759 }
5760 repo = Some(state.repo.clone());
5761 if let Some(authority) = state.retired_to {
5762 id = authority;
5763 continue;
5764 }
5765 return (crate::handover::can_resume(&state)
5766 && state.liveness(false) != crate::run::Liveness::Live)
5767 .then_some(id);
5768 }
5769}
5770
5771fn awaiting_resume_with<F>(task: &Task, load: F) -> bool
5779where
5780 F: FnOnce(&str) -> Result<RunState>,
5781{
5782 if !matches!(task.status, TaskStatus::Parked | TaskStatus::Failed)
5784 || task.fresh_start
5785 || task.review_branch.is_some()
5786 {
5787 return false;
5788 }
5789 let Some(id) = task.execution_runs().last() else {
5790 return false;
5791 };
5792 load(id).is_ok_and(|s| s.parked && crate::handover::can_resume(&s))
5793}
5794
5795#[derive(Debug, Clone, PartialEq, Eq)]
5798enum Starter {
5799 Review(String),
5802 Resume(String),
5804 Start,
5806}
5807
5808fn retained_attempt_hold(
5813 task: &Task,
5814 starter: &Starter,
5815 limit: usize,
5816 mut load: impl FnMut(&str) -> Result<RunState>,
5817) -> Option<String> {
5818 if limit == 0 || matches!(starter, Starter::Resume(_)) {
5819 return None;
5820 }
5821 let mut retained = 0;
5822 for id in task
5823 .runs
5824 .iter()
5825 .chain(task.startup_failures.iter().map(|failure| &failure.run))
5826 .collect::<std::collections::BTreeSet<_>>()
5827 {
5828 let state = match load(id) {
5829 Ok(state) if state.id == *id => state,
5830 _ => {
5831 return Some(format!(
5832 "[recovery-backlog] Cannot verify historical run {id}. Restore its record and inspect the recovery lineage before starting another attempt. Existing history was retained."
5833 ));
5834 }
5835 };
5836 if state.retired_to.is_some() || state.released() {
5837 continue;
5838 }
5839 retained += 1;
5842 }
5843 (retained >= limit).then(|| format!(
5844 "[recovery-backlog] {retained} unretired attempts reach the limit of {limit}. Resume an existing recovery owner or inspect the lineage and retire proved-obsolete attempts before creating another run. Releases, refunds and changed upstream inputs do not reset this limit. No source or historical report was removed."
5845 ))
5846}
5847
5848fn selected_starter(
5850 selected: &crate::pr_recovery::Selection,
5851 previous: Option<&RunState>,
5852 unfinished: Option<&str>,
5853) -> Starter {
5854 if let Some(id) = unfinished
5855 && previous.is_some_and(|r| {
5856 r.id == id && r.pr_recovery.as_ref() == Some(selected) && r.reviewed_commits.is_some()
5857 })
5858 {
5859 Starter::Resume(id.to_owned())
5860 } else {
5861 Starter::Review(selected.branch.clone())
5862 }
5863}
5864
5865fn take_divergence_answer(
5869 branch: &str,
5870 remote: &str,
5871 task: &mut Task,
5872) -> Option<crate::reconcile::Choice> {
5873 let summary = crate::reconcile::summary_for(branch, remote);
5874 let (idx, choice) = task.answers.iter().enumerate().rev().find_map(|(i, a)| {
5875 (a.question == summary)
5876 .then(|| crate::reconcile::Choice::from_answer(&a.answer))
5877 .flatten()
5878 .map(|c| (i, c))
5879 })?;
5880 task.answers.remove(idx);
5881 Some(choice)
5882}
5883
5884fn choose_starter(
5896 review_branch: Option<&str>,
5897 branch_exists: bool,
5898 unfinished: Option<&str>,
5899) -> Starter {
5900 match review_branch {
5901 Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
5902 Some(_) => Starter::Start,
5903 None => match unfinished {
5904 Some(id) => Starter::Resume(id.to_owned()),
5905 None => Starter::Start,
5906 },
5907 }
5908}
5909
5910fn review_retry_starter(
5914 task: &Task,
5915 previous: Option<&RunState>,
5916 branch: Option<&str>,
5917 exists: bool,
5918 unfinished: Option<&str>,
5919) -> Starter {
5920 if let Some(run) = previous
5921 && unfinished == Some(run.id.as_str())
5922 && run.reviewed_commits.is_some()
5923 && branch.is_some_and(|branch| {
5924 exists && run.candidates.len() == 1 && run.candidates[0].branch == branch
5925 })
5926 && may_resume_run(run, task)
5927 {
5928 return Starter::Resume(run.id.clone());
5929 }
5930 match (branch, unfinished, &task.review_of) {
5931 (None, None, Some(branch)) => Starter::Review(branch.clone()),
5932 _ => choose_starter(branch, exists, unfinished),
5933 }
5934}
5935
5936fn take_review_branch(task: &mut Task, previous: Option<&RunState>) -> Option<String> {
5938 let branch = task.review_branch.take();
5939 let instruction = task.review_instruction.take();
5940 branch.filter(|branch| match &instruction {
5941 Some(authored) => *authored == task.instruction,
5942 None => previous.map_or_else(
5945 || task.review_of.as_ref() == Some(branch),
5946 |run| compatible_run_instruction(run, task),
5947 ),
5948 })
5949}
5950
5951fn may_resume_run(run: &RunState, task: &Task) -> bool {
5954 if !task.assignment_accepts(run) {
5955 return false;
5956 }
5957 task.resume_override
5958 .as_ref()
5959 .is_some_and(|r| r.pinned_run.as_ref() == Some(&run.id))
5960 || compatible_run_instruction(run, task)
5961}
5962
5963pub(crate) fn compatible_run_instruction(run: &RunState, task: &Task) -> bool {
5965 if let Some(authored) = &run.task_instruction {
5966 return compatible_instruction(authored, task);
5967 }
5968 if compatible_instruction(&run.instruction, task) {
5969 return true;
5970 }
5971 if run.reviewed_commits.is_some() {
5974 if let Some((authored, _)) = run.instruction.split_once("\n\n# Work already on branch `") {
5975 return strip_answers_block(authored, task).trim() == task.instruction.trim();
5976 }
5977 if let Some(branch) = &task.review_of {
5978 let canonical = format!(
5979 "Review the branch `{branch}` (review-only: no implementation, no judging)."
5980 );
5981 let generated = format!(
5982 "Review the work already on branch `{branch}`. There is no task statement: what the change claims to do is whatever its commits say.\n\n"
5983 );
5984 return task.instruction == canonical && run.instruction.starts_with(&generated);
5985 }
5986 }
5987 false
5988}
5989
5990pub(crate) fn compatible_instruction(old: &str, task: &Task) -> bool {
5992 strip_answers_block(old, task) == task.instruction
5993}
5994
5995fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
5998 if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
5999 return fallback.to_path_buf();
6000 }
6001 task.repo.clone()
6002}
6003
6004const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
6008
6009fn answers_block(task: &Task, count: usize) -> String {
6011 let mut s = ANSWERS_HEADER.to_owned();
6012 for a in &task.answers[..count] {
6013 s.push_str(&format!("- {}: {}\n", a.question, a.answer));
6014 }
6015 s
6016}
6017
6018fn resumption_block(r: &crate::queue::Resumption) -> String {
6021 format!(
6022 "\n\n# Explicit operator resumption\n\nGeneration {} at {} approved this instruction:\n{}\n\
6023 Earlier answers (first {}) are audit history; their waiting premises are superseded by this approval.\n",
6024 r.generation, r.at, r.instruction, r.answers_through,
6025 )
6026}
6027
6028fn append_answers(base: &str, task: &Task) -> String {
6030 let mut s = base.to_owned();
6031 if !task.answers.is_empty() {
6032 s.push_str(&answers_block(task, task.answers.len()));
6033 }
6034 if let Some(r) = &task.resumption {
6035 s.push_str(&resumption_block(r));
6036 }
6037 s
6038}
6039
6040fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
6044 let mut instruction = instruction;
6045 for approval in task
6046 .resumption
6047 .iter()
6048 .chain(task.resumption_history.iter().rev())
6049 {
6050 if let Some(base) = instruction.strip_suffix(&resumption_block(approval)) {
6051 instruction = base;
6052 break;
6053 }
6054 }
6055 for count in (1..=task.answers.len()).rev() {
6056 let block = answers_block(task, count);
6057 if let Some(base) = instruction.strip_suffix(&block) {
6058 return base;
6059 }
6060 }
6061 instruction
6062}
6063
6064fn instruction_for(task: &Task) -> String {
6072 append_answers(&task.instruction, task)
6073}
6074
6075fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
6087 append_answers(strip_answers_block(old_instruction, task), task)
6088}
6089
6090fn record_config_failure(task: &mut Task, error: &anyhow::Error) {
6092 let failure = error.downcast_ref::<crate::config::ConfigFailure>();
6093 task.notice_cause(failure.map_or("config-load", |f| f.stage), None, None);
6094 if let Some(cause) = task.hold_notice_cause.as_mut() {
6095 cause.target = failure.map(|f| f.target.clone());
6096 }
6097}
6098
6099#[derive(Debug, thiserror::Error)]
6100#[error("attachment `{name}` is recorded on the task but {path} is missing")]
6101struct MissingAttachment {
6102 name: String,
6103 path: PathBuf,
6104}
6105
6106fn task_attachments(
6108 queue: &Queue,
6109 task: &Task,
6110) -> std::result::Result<Vec<PathBuf>, MissingAttachment> {
6111 let paths = queue.attachment_paths(task);
6112 for (name, path) in task.attachments.iter().zip(&paths) {
6113 if !path.is_file() {
6114 return Err(MissingAttachment {
6115 name: name.clone(),
6116 path: path.clone(),
6117 });
6118 }
6119 }
6120 Ok(paths)
6121}
6122
6123fn prepare_instruction(
6134 starter: &Starter,
6135 old_instruction: Option<&str>,
6136 task: &Task,
6137) -> Option<String> {
6138 match starter {
6139 Starter::Start => Some(instruction_for(task)),
6140 Starter::Resume(_) => Some(resumed_instruction(
6141 old_instruction.expect("a resumed run always has a prior instruction"),
6142 task,
6143 )),
6144 Starter::Review(_) => None,
6145 }
6146}
6147
6148fn record(queue: &Queue, task: &mut Task) {
6152 if let Err(e) = queue.put(task) {
6153 tracing::error!("could not record task {}: {e:#}", task.short());
6154 let notice = Notice::error(
6155 "loop:record",
6156 format!(
6157 "Task {} state could not be saved: {}. Check the disk.",
6158 task.id,
6159 e.to_string().lines().next().unwrap_or("unknown save error")
6160 ),
6161 )
6162 .link(Link::Task {
6163 id: task.id.clone(),
6164 });
6165 match queue.root().parent().filter(|p| !p.as_os_str().is_empty()) {
6167 Some(home) => notices::raise_in(home, notice),
6168 None => notices::raise(notice),
6169 }
6170 }
6171}
6172
6173fn runnable(queue: &Queue) -> Vec<Task> {
6179 let mut tasks: Vec<Task> = queue
6180 .list()
6181 .into_iter()
6182 .filter(|t| t.status.runnable())
6183 .collect();
6184 tasks.sort_unstable_by(crate::queue::dispatch_order);
6185 tasks
6186}
6187
6188fn describe(state: &RunState) -> String {
6202 let p = phrases(&state.config.graph.language);
6203 let mut detail = if state.status == RunStatus::Stalled {
6204 let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
6205 seats.sort_unstable();
6206 seats.dedup();
6207 if seats.is_empty() {
6208 p.quorum_lost.to_owned()
6209 } else {
6210 format!("{}{}{}", p.quorum_lost, p.quota_took_out, seats.join(", "))
6211 }
6212 } else {
6213 format!("{}{}", p.run_ended, state.status.display_label())
6214 };
6215 if let Some(cause) = state.verification_blocking_cause() {
6216 detail.push_str(&format!(" ({cause})"));
6217 } else if let Some(last) = state.events.last() {
6218 let unanswered = state
6222 .reviews
6223 .last()
6224 .filter(|r| {
6225 state.status == RunStatus::Blocked
6226 && last.node == "review"
6227 && r.incomplete()
6228 && r.blocking == 0
6229 && r.round == state.config.graph.review_rounds
6230 && r.e2e.iter().all(crate::run::CommandOutcome::ok)
6231 })
6232 .map(|r| (r.expected - r.answered, r.round));
6233 match unanswered {
6234 Some((missing, rounds)) if crate::lang::is_japanese(&state.config.graph.language) => {
6235 detail.push_str(&format!(
6236 " ({}: {})",
6237 last.node,
6238 (p.reviewers_never_answered)(missing, rounds)
6239 ));
6240 }
6241 _ => detail.push_str(&format!(" ({}: {})", last.node, last.message)),
6242 }
6243 }
6244 detail.push_str(&format!(" [run {}]", state.id));
6245 detail
6246}
6247
6248const DIAGNOSTIC_MAX: usize = 4_000;
6254
6255const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
6260
6261fn diagnostic(state: &RunState) -> Option<String> {
6275 let mut parts: Vec<String> = Vec::new();
6276
6277 for o in state.gate.iter().filter(|o| !o.ok()) {
6279 parts.push(format!(
6280 "gate `{}` failed ({}):\n{}",
6281 o.command,
6282 notices::exit_status(o.code),
6283 crate::run::summarize_verify(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
6284 ));
6285 }
6286
6287 if let Some(last) = state
6290 .events
6291 .iter()
6292 .rev()
6293 .find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
6294 {
6295 parts.push(last.message.clone());
6296 }
6297
6298 if state.viable().is_empty() {
6305 for c in &state.candidates {
6306 if let Some(evidence) = &c.verified_noop {
6307 parts.push(format!(
6308 "candidate {} (agent-verified no-op, unconfirmed by magi): {evidence}",
6309 c.label
6310 ));
6311 } else if !c.summary.trim().is_empty() {
6312 parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
6313 } else if let Some(why) = &c.failed {
6314 parts.push(format!("candidate {}: {why}", c.label));
6315 }
6316 }
6317 }
6318
6319 if parts.is_empty() {
6320 return None;
6321 }
6322 Some(crate::run::tail(
6327 &parts.join("\n\n"),
6328 DIAGNOSTIC_MAX.saturating_sub(100),
6329 ))
6330}
6331
6332fn label(status: RunStatus) -> &'static str {
6340 status.as_str()
6341}
6342
6343pub(crate) fn merge_mode(mode: &str) -> Result<MergeMode> {
6345 match mode {
6346 "none" => Ok(MergeMode::None),
6347 "local" => Ok(MergeMode::Local),
6348 "pr" => Ok(MergeMode::Pr),
6349 other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
6350 }
6351}
6352
6353fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
6358 mutex
6359 .lock()
6360 .unwrap_or_else(std::sync::PoisonError::into_inner)
6361}
6362
6363#[cfg(test)]
6364mod tests {
6365 use super::*;
6366 use crate::queue::{Source, TaskStatus};
6367 use crate::run::{Candidate, CommandOutcome};
6368 use pretty_assertions::assert_eq;
6369
6370 fn task() -> Task {
6371 Task::new(
6372 "add retries".to_owned(),
6373 "add retries".to_owned(),
6374 PathBuf::from("/repo"),
6375 Source::Human,
6376 )
6377 }
6378
6379 fn starting_task_reservation(home: &Path, request: &TaskWorker) -> crate::worker::Job {
6380 let job = crate::worker::Job {
6381 id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".into(),
6382 sequence: 1,
6383 kind: "task".into(),
6384 key: format!("task:{}", request.task),
6385 payload: serde_json::to_value(request).unwrap(),
6386 generation: "a".repeat(64),
6387 version: None,
6388 status: crate::worker::Status::Starting,
6389 result: None,
6390 pid: None,
6391 created_at: u64::try_from(Timestamp::now().as_second()).unwrap(),
6392 finished_at: None,
6393 };
6394 let dir = home.join("workers/jobs").join(&job.id);
6395 std::fs::create_dir_all(&dir).unwrap();
6396 std::fs::write(dir.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
6397 job
6398 }
6399
6400 #[test]
6401 fn an_unacknowledged_own_reservation_stops_dispatch_without_cancelling_the_child() {
6402 let home = tempfile::tempdir().unwrap();
6403 let request = TaskWorker {
6404 opts: Opts::default(),
6405 queue: home.path().join("queue"),
6406 task: "startup".into(),
6407 lane: PermitKind::Ordinary,
6408 standalone: false,
6409 };
6410 let job = starting_task_reservation(home.path(), &request);
6411 let error = task_worker_spawn_error(
6412 home.path(),
6413 &request,
6414 anyhow::anyhow!("worker startup is unacknowledged: private-error-detail"),
6415 )
6416 .expect_err("our own reservation must not turn a failed spawn into a resync");
6417 let active = crate::worker::active(home.path()).unwrap();
6418 assert_eq!(active.len(), 1);
6419 assert_eq!(active[0].id, job.id);
6420 assert_eq!(active[0].status, crate::worker::Status::Starting);
6421 assert!(
6422 crate::worker::control(home.path(), &job.key)
6423 .unwrap()
6424 .is_none()
6425 );
6426 let notices = notices::Notices::at(home.path().join("notifications")).list();
6427 assert_eq!(notices.len(), 1);
6428 assert_eq!(notices[0].key, "loop:worker-startup");
6429 assert!(!notices[0].message.contains("private-error-detail"));
6430 assert!(notices[0].message.contains(&request.task));
6431 assert!(
6432 notices[0]
6433 .message
6434 .contains("could not be started or acknowledged")
6435 );
6436 assert!(notices[0].message.contains("worker.log"));
6437 assert_eq!(
6438 notices[0].link,
6439 Some(Link::Task {
6440 id: request.task.clone()
6441 })
6442 );
6443 assert!(format!("{error:#}").contains("private-error-detail"));
6444 }
6445
6446 #[test]
6447 fn a_standalone_handoff_reservation_resyncs_dispatch_without_replacing_it() {
6448 let home = tempfile::tempdir().unwrap();
6449 let mut request = TaskWorker {
6450 opts: Opts::default(),
6451 queue: home.path().join("queue"),
6452 task: "handoff".into(),
6453 lane: PermitKind::Ordinary,
6454 standalone: true,
6455 };
6456 let job = starting_task_reservation(home.path(), &request);
6457 request.standalone = false;
6458 task_worker_spawn_error(
6459 home.path(),
6460 &request,
6461 anyhow::anyhow!("operation key is already reserved for a different request"),
6462 )
6463 .expect("a different standalone request is the handoff race, not our startup failure");
6464 let active = crate::worker::active(home.path()).unwrap();
6465 assert_eq!(active.len(), 1);
6466 assert_eq!(active[0].id, job.id);
6467 assert_eq!(active[0].payload, job.payload);
6468 assert!(
6469 notices::Notices::at(home.path().join("notifications"))
6470 .list()
6471 .is_empty()
6472 );
6473 }
6474
6475 #[test]
6476 fn starting_worker_projects_the_previous_run_without_owning_its_execution() {
6477 let home = tempfile::tempdir().unwrap();
6478 let queue = Queue::at(home.path().join("queue"));
6479 let mut task = task();
6480 task.runs = vec!["previous".into()];
6481 queue.put(&mut task).unwrap();
6482 let request = TaskWorker {
6483 opts: Opts::default(),
6484 queue: home.path().join("queue"),
6485 task: task.id.clone(),
6486 lane: PermitKind::Ordinary,
6487 standalone: false,
6488 };
6489 let job = starting_task_reservation(home.path(), &request);
6490 let claims = current_work(home.path(), Timestamp::now());
6491 assert_eq!(claims.len(), 1);
6492 assert_eq!(claims[0].worker.as_deref(), Some(job.id.as_str()));
6493 assert_eq!(claims[0].run, "previous");
6494 assert!(other_claims_run(&claims, "previous", None));
6495 assert!(!other_claims_run(&claims, "previous", Some(&job.id)));
6498 assert!(is_working_on(home.path(), "previous", Timestamp::now()));
6499 }
6500
6501 #[test]
6502 fn admission_excludes_only_the_callers_exact_reservation() {
6503 let own = Current {
6504 worker: Some("own".into()),
6505 task: "same-task".into(),
6506 run: "previous".into(),
6507 };
6508 assert!(!other_claims_run(
6509 std::slice::from_ref(&own),
6510 "previous",
6511 Some("own")
6512 ));
6513 assert!(other_claims_run(
6514 std::slice::from_ref(&own),
6515 "previous",
6516 None
6517 ));
6518 for worker in [None, Some("other".into())] {
6519 let other = Current {
6520 worker,
6521 ..own.clone()
6522 };
6523 assert!(other_claims_run(
6524 &[own.clone(), other],
6525 "previous",
6526 Some("own")
6527 ));
6528 }
6529 let legacy: Current =
6530 serde_json::from_str(r#"{"task":"same-task","run":"previous"}"#).unwrap();
6531 assert!(other_claims_run(&[legacy], "previous", Some("own")));
6532 }
6533
6534 #[test]
6535 fn review_retry_resumes_only_the_matching_authorized_latest_owner() {
6536 let mut task = task();
6537 let mut run = run_state(RunStatus::Reviewing);
6538 run.task_instruction = Some(task.instruction.clone());
6539 run.reviewed_commits = Some(vec!["committed-fix".into()]);
6540 let branch = "magi/test/B";
6541 run.candidates.push(crate::run::Candidate {
6542 branch: branch.into(),
6543 ..candidate('B', "preserved work", false, None)
6544 });
6545 task.runs = vec![run.id.clone()];
6546 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(run.clone()));
6547 assert_eq!(
6548 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6549 Starter::Resume(run.id.clone())
6550 );
6551 assert!(matches!(
6552 review_retry_starter(
6553 &task,
6554 Some(&run),
6555 Some("different"),
6556 true,
6557 unfinished.as_deref()
6558 ),
6559 Starter::Review(_)
6560 ));
6561 assert_eq!(
6562 review_retry_starter(
6563 &task,
6564 Some(&run),
6565 Some(branch),
6566 false,
6567 unfinished.as_deref()
6568 ),
6569 Starter::Start
6570 );
6571 task.instruction.push_str(" changed criteria");
6572 assert!(matches!(
6573 review_retry_starter(&task, Some(&run), Some(branch), true, unfinished.as_deref()),
6574 Starter::Review(_)
6575 ));
6576 assert!(matches!(
6577 review_retry_starter(&task, Some(&run), Some(branch), true, None),
6578 Starter::Review(_)
6579 ));
6580 }
6581
6582 #[test]
6583 fn a_live_worker_reservation_prevents_reclaim_before_its_task_claim() {
6584 let dir = tempfile::tempdir().unwrap();
6585 let queue = Queue::at(dir.path().join("queue"));
6586 let mut task = task();
6587 task.status = TaskStatus::Running;
6588 task.attempts = 1;
6589 queue.put(&mut task).unwrap();
6590 let live = std::collections::BTreeSet::from([task.id.clone()]);
6591 assert!(reclaim_orphaned_running(&queue, 2, &live).is_empty());
6592 drop(queue.claim(&task.id).unwrap());
6593 let saved = queue.get(&task.id).unwrap();
6594 assert_eq!(saved.status, TaskStatus::Running);
6595 assert_eq!(saved.attempts, 1);
6596 assert!(saved.last_error.is_none());
6597 }
6598
6599 #[test]
6600 fn standalone_workers_do_not_consume_scheduler_lanes() {
6601 let request = |task: &str, lane, standalone| TaskWorker {
6602 opts: Opts::default(),
6603 queue: PathBuf::from("/queue"),
6604 task: task.to_owned(),
6605 lane,
6606 standalone,
6607 };
6608 let mut jobs = vec![
6609 request("ordinary", PermitKind::Ordinary, false),
6610 request("standalone-urgent", PermitKind::Urgent, true),
6611 request("standalone-ordinary", PermitKind::Ordinary, true),
6612 request("decision", PermitKind::None, false),
6613 ];
6614 assert_eq!(available_slots(&jobs, 2), (1, 1));
6615 jobs.push(request("urgent", PermitKind::Urgent, false));
6616 assert_eq!(available_slots(&jobs, 2), (1, 0));
6617 assert_eq!(available_slots(&jobs, 0), (0, 0));
6618 }
6619
6620 #[test]
6621 fn a_live_legacy_scheduler_blocks_migration_even_with_a_stale_heartbeat() {
6622 let now = Timestamp::now();
6623 let mut reading = Reading {
6624 pid: Some(std::process::id()),
6625 updated_at: Some(now - jiff::SignedDuration::from_secs(STALE_SECS + 1)),
6626 ..Reading::default()
6627 };
6628 assert!(!reading.running(now));
6629 assert!(legacy_scheduler_running(&reading, now, 0));
6630 reading.worker_protocol = Some(1);
6631 assert!(!legacy_scheduler_running(&reading, now, 0));
6632 }
6633
6634 #[test]
6635 fn selected_public_head_overrides_historical_resume_and_pin() {
6636 let mut t = task();
6637 let mut old = run_state(RunStatus::Blocked);
6638 let selection = crate::pr_recovery::Selection {
6639 url: "url".into(),
6640 branch: format!("magi/task-{}", t.id),
6641 head: "a".repeat(40),
6642 source_head: None,
6643 source_run: None,
6644 source_proof: None,
6645 disposition: Some(crate::pr_recovery::Disposition::Review),
6646 instruction: t.instruction.clone(),
6647 approved_at: Timestamp::now(),
6648 };
6649 t.resume_override = Some(crate::queue::OperatorResume {
6650 pinned_run: Some(old.id.clone()),
6651 question_id: "choice".into(),
6652 at: Timestamp::now(),
6653 conductor_rehold: None,
6654 forced: false,
6655 });
6656 assert_eq!(
6657 selected_starter(&selection, Some(&old), Some(&old.id)),
6658 Starter::Review(selection.branch.clone())
6659 );
6660 old.pr_recovery = Some(selection.clone());
6661 assert_eq!(
6662 selected_starter(&selection, Some(&old), Some(&old.id)),
6663 Starter::Review(selection.branch.clone())
6664 );
6665 old.reviewed_commits = Some(vec!["implementation".into()]);
6666 assert_eq!(
6667 selected_starter(&selection, Some(&old), Some(&old.id)),
6668 Starter::Resume(old.id.clone())
6669 );
6670 let mut changed = selection.clone();
6671 changed.head = "b".repeat(40);
6672 assert_eq!(
6673 selected_starter(&changed, Some(&old), Some(&old.id)),
6674 Starter::Review(changed.branch)
6675 );
6676 }
6677
6678 #[test]
6679 fn candidate_transplant_failure_is_one_actionable_hold() {
6680 let dir = tempfile::tempdir().unwrap();
6681 let q = Queue::at(dir.path().join("queue"));
6682 let mut t = task();
6683 let mut state = run_state(RunStatus::Blocked);
6684 state.event("prep", "cannot apply referenced commit: conflict");
6685 state.prep_failure = Some("conflict".into());
6686 t.start(state.id.clone());
6687 finish_attempt(3, &q, &mut t, &state, &[], Err(anyhow::anyhow!("conflict")));
6688 assert_eq!(t.attempts, 1);
6689 assert_eq!(t.status, TaskStatus::Held);
6690 assert!(t.operator_held());
6691 assert!(!t.status.runnable());
6692 assert!(t.hold_reason.as_deref().unwrap().contains("recover-pr"));
6693 let first = t.chat_notice_key().unwrap();
6694 t.chat_notices.push(first.clone());
6695 t.release_operator("test", None);
6696 state.id = "another-unprepared-run".into();
6697 t.start(state.id.clone());
6698 finish_attempt(
6699 3,
6700 &q,
6701 &mut t,
6702 &state,
6703 &[],
6704 Err(anyhow::anyhow!("still conflicts")),
6705 );
6706 assert_eq!(
6707 t.chat_notice_key().as_ref(),
6708 Some(&first),
6709 "no work happened"
6710 );
6711 t.release_operator("test", None);
6712 state.prep_failure = None;
6713 state.status = RunStatus::Failed;
6714 state.begin_pass();
6715 t.start(state.id.clone());
6716 finish_attempt(
6717 1,
6718 &q,
6719 &mut t,
6720 &state,
6721 &[],
6722 Err(anyhow::anyhow!("new gate failure")),
6723 );
6724 let failure = t.chat_notice_key().unwrap();
6725 assert_ne!(failure, first, "approved recovery reached a real failure");
6726 t.release_operator("test", None);
6727 state.begin_pass();
6728 t.start(state.id.clone());
6729 finish_attempt(
6730 1,
6731 &q,
6732 &mut t,
6733 &state,
6734 &[],
6735 Err(anyhow::anyhow!("new failure on resume")),
6736 );
6737 assert_ne!(
6738 t.chat_notice_key().as_ref(),
6739 Some(&failure),
6740 "a new settled pass"
6741 );
6742 }
6743
6744 #[test]
6745 fn revised_resumption_invalidates_old_pins_and_preserves_unchanged_pins() {
6746 for source in ["cli", "web"] {
6747 let mut t = task();
6748 let mut run = run_state(RunStatus::Blocked);
6749 run.instruction = "Run A original criteria".into();
6750 run.task_instruction = Some(run.instruction.clone());
6751 t.start(run.id.clone());
6752 t.release_operator("choice", Some("resume-question"));
6753 t.resume_override = Some(crate::queue::OperatorResume {
6754 question_id: "resume-question".into(),
6755 at: Timestamp::now(),
6756 conductor_rehold: None,
6757 forced: true,
6758 pinned_run: Some(run.id.clone()),
6759 });
6760 t.hold_machine(Some("not enough free space to start a run".into()));
6761 let mut unchanged = t.clone();
6762 unchanged
6763 .edit("title changed".into(), unchanged.instruction.clone())
6764 .unwrap();
6765 unchanged.release_operator(source, None);
6766 assert_eq!(
6767 unchanged
6768 .resume_override
6769 .as_ref()
6770 .unwrap()
6771 .pinned_run
6772 .as_deref(),
6773 Some(run.id.as_str())
6774 );
6775 assert!(!compatible_run_instruction(&run, &unchanged));
6776 assert!(may_resume_run(&run, &unchanged));
6777
6778 t.edit(
6779 "approved recovery".into(),
6780 "Reuse measured PR head B; fix Clippy; review".into(),
6781 )
6782 .unwrap();
6783 assert!(t.resume_override.is_none());
6784 t.release_operator(source, None);
6785 let reloaded: Task = serde_json::from_str(&serde_json::to_string(&t).unwrap()).unwrap();
6786 assert!(reloaded.resume_override.is_none());
6787 assert!(!may_resume_run(&run, &reloaded));
6788 let unfinished =
6789 unfinished_run_with(&reloaded.runs, reloaded.short(), |_| Ok(run.clone()))
6790 .filter(|_| may_resume_run(&run, &reloaded));
6791 assert_eq!(
6792 choose_starter(None, false, unfinished.as_deref()),
6793 Starter::Start
6794 );
6795 assert!(instruction_for(&reloaded).starts_with("Reuse measured PR head B"));
6796 assert_eq!(
6797 reloaded.resumption_history[0].question_id.as_deref(),
6798 Some("resume-question")
6799 );
6800
6801 unchanged.instruction = "Reuse measured PR head B".into();
6804 unchanged.hold_machine(Some("disk hold".into()));
6805 unchanged.release_operator(source, None);
6806 assert!(unchanged.resume_override.is_none());
6807 assert!(!may_resume_run(&run, &unchanged));
6808 }
6809 }
6810
6811 #[test]
6812 fn resumption_preserves_generated_review_runs_and_rejects_revised_targets() {
6813 let mut t = task();
6814 let mut run = run_state(RunStatus::Blocked);
6815 t.review_of = Some("approved-head".into());
6816 t.instruction =
6817 "Review the branch `approved-head` (review-only: no implementation, no judging)."
6818 .into();
6819 run.instruction = "Review the work already on branch `approved-head`. There is no task statement: what the change claims to do is whatever its commits say.\n\nCommit log".into();
6820 run.reviewed_commits = Some(vec!["Commit log".into()]);
6821 t.start(run.id.clone());
6822 t.hold_machine(Some("interrupted review".into()));
6823 t.release_operator("cli", None);
6824 assert!(
6825 compatible_run_instruction(&run, &t),
6826 "legacy explicit review remains resumable"
6827 );
6828 let unfinished = unfinished_run_with(&t.runs, t.short(), |_| Ok(run.clone()))
6829 .filter(|_| compatible_run_instruction(&run, &t));
6830 assert_eq!(
6831 choose_starter(None, false, unfinished.as_deref()),
6832 Starter::Resume(run.id.clone())
6833 );
6834 run.task_instruction = Some(t.instruction.clone());
6835 assert!(compatible_run_instruction(&run, &t));
6836 let reloaded: RunState =
6837 serde_json::from_str(&serde_json::to_string(&run).unwrap()).unwrap();
6838 assert!(compatible_run_instruction(&reloaded, &t));
6839 t.edit(
6840 "new approved head".into(),
6841 "Reuse measured PR head B".into(),
6842 )
6843 .unwrap();
6844 assert!(!compatible_run_instruction(&run, &t));
6845
6846 t.review_of = None;
6848 t.instruction = "Fix Clippy on approved head".into();
6849 run.task_instruction = None;
6850 run.instruction = format!(
6851 "{}\n\n# Work already on branch `approved-head` (commit subjects)\n\nCommit log",
6852 t.instruction
6853 );
6854 assert!(compatible_run_instruction(&run, &t));
6855 t.instruction.push('\n');
6856 assert!(
6857 compatible_run_instruction(&run, &t),
6858 "legacy generated review trimmed the original criteria"
6859 );
6860 t.hold_for_handover(Some("approved-head".into()), "refused".into());
6861 let mut unchanged = t.clone();
6862 assert_eq!(
6863 take_review_branch(&mut unchanged, Some(&run)).as_deref(),
6864 Some("approved-head")
6865 );
6866 t.edit("new plan".into(), "Reuse measured PR head B".into())
6867 .unwrap();
6868 assert!(take_review_branch(&mut t, Some(&run)).is_none());
6869 assert!(!compatible_run_instruction(&run, &t));
6870 t.review_branch = Some("approved-head".into());
6871 t.review_instruction = None;
6872 assert!(
6873 take_review_branch(&mut t, Some(&run)).is_none(),
6874 "legacy retained candidate cannot replace a revised approved head"
6875 );
6876 }
6877
6878 #[test]
6879 fn resumption_actions_survive_automatic_holds_of_hand_started_runs() {
6880 let home = crate::run::test_home();
6881 crate::run::pin_test_home();
6882 let dir = tempfile::tempdir().unwrap();
6883 let queue = Queue::at(dir.path().join("queue"));
6884 for resume in [true, false] {
6885 let state = run_state(RunStatus::Blocked);
6886 let mut t = task();
6887 t.start(state.id.clone());
6888 let action = if resume {
6889 ask::ChoiceAction::Resume {
6890 run: state.id.clone(),
6891 }
6892 } else {
6893 ask::ChoiceAction::Requeue
6894 };
6895 let mut q = action_question(&state.id, action);
6896 q.status = ask::QuestionStatus::Open;
6897 q.answer = None;
6898 q.answered_at = None;
6899 t.fail("run stopped with a question", 3);
6900 hold_if_runnable(&queue, &mut t);
6901 assert!(t.operator_decided_at.is_none());
6902 assert!(t.operator_held());
6903 q.answer(ask::Answer::Choice(q.choices[0].clone())).unwrap();
6904 let expected = if resume {
6905 ActionDecision::Resume(state.id.clone())
6906 } else {
6907 ActionDecision::Requeue
6908 };
6909 assert_eq!(
6910 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
6911 expected
6912 );
6913 t.hold_manual(Some("later explicit hold".into()));
6914 assert_eq!(
6915 decide_action(&home, &t, &q, phrases("en"), |_| Ok(state.clone())),
6916 ActionDecision::Stale
6917 );
6918 }
6919 }
6920
6921 #[test]
6922 fn explicit_resumption_keeps_revised_instruction_and_audit_answers() {
6923 let mut t = task();
6924 t.answers.push(crate::queue::AnsweredQuestion {
6925 question: "Inspect the remote?".into(),
6926 answer: "Inspect then wait".into(),
6927 });
6928 let old = instruction_for(&t);
6929 assert!(compatible_instruction(&old, &t));
6930 t.instruction = "Reuse approved PR head and review".into();
6931 t.release_operator("cli", None);
6932 assert!(!compatible_instruction(&old, &t));
6933 let approved = instruction_for(&t);
6934 assert!(approved.contains("waiting premises are superseded"));
6935 assert!(compatible_instruction(&approved, &t));
6936 assert_eq!(resumed_instruction(&approved, &t), approved);
6937 t.release_operator("web", None);
6938 assert!(compatible_instruction(&approved, &t));
6939 assert_eq!(t.resumption_history.len(), 1);
6940 t.start("run".into());
6941 assert!(instruction_for(&t).contains("Explicit operator resumption"));
6942 }
6943
6944 #[test]
6948 fn unrelated_ci_refunds_once_per_pass_and_holds_for_the_owner() {
6949 let mut state = RunState::new(
6950 PathBuf::from("/repo"),
6951 "main".into(),
6952 "base".into(),
6953 "task".into(),
6954 Config::default(),
6955 );
6956 state.status = RunStatus::Landing;
6957 state.parked = true;
6958 state.pr_pushed_head = Some("head".into());
6959 state.land_unrelated = Some(
6960 serde_json::from_value(serde_json::json!({
6961 "head": "head", "question": "ci-question",
6962 "evidence": {"head": "head", "base": "base", "changed": [], "checks": []}
6963 }))
6964 .unwrap(),
6965 );
6966 let mut t = task();
6967 t.start(state.id.clone());
6968 let verdict = Verdict {
6969 status: RunStatus::Landing,
6970 left_pr: true,
6971 quota_hit: false,
6972 parked: true,
6973 no_viable_candidates: false,
6974 };
6975 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6976 assert_eq!(t.attempts, 0);
6977 assert_eq!(t.status, TaskStatus::Held);
6978 assert!(!t.operator_held());
6979 assert!(
6980 t.hold_reason
6981 .as_deref()
6982 .unwrap()
6983 .starts_with("[ci-unrelated]")
6984 );
6985 t.attempts = 1;
6987 settle_and_diagnose(&mut t, verdict, "CI hold", 2, &state);
6988 assert_eq!(t.attempts, 1);
6989 assert_eq!(t.passes.len(), 1);
6990 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
6991 state.pr_pushed_head = Some("new-head".into());
6992 state.parked = false;
6993 state.status = RunStatus::Blocked;
6994 let stale = Verdict {
6995 status: RunStatus::Blocked,
6996 parked: false,
6997 ..verdict
6998 };
6999 settle_and_diagnose(&mut t, stale, "stale CI hold", 2, &state);
7000 assert_eq!(t.attempts, 1);
7001 state.begin_pass();
7003 state.begin_pass();
7004 state.status = RunStatus::Landing;
7005 state.parked = true;
7006 state.pr_pushed_head = Some("head".into());
7007 settle_and_diagnose(&mut t, verdict, "CI hold on resume", 2, &state);
7008 assert_eq!(t.attempts, 0);
7009 assert_eq!(t.passes.len(), 2);
7010 }
7011
7012 #[test]
7013 fn orphan_recovery_does_not_refund_a_previous_pass_loss() {
7014 let mut state = run_state(RunStatus::Failed);
7015 state
7016 .quota
7017 .push(loss("advisor-1", "2026-10-22T14:38:00Z", None));
7018 state.begin_pass();
7019 let mut t = task();
7020 t.start(state.id.clone());
7021 reclaim(&mut t, Some(state), 2, "en");
7022 assert_eq!(t.attempts, 1);
7023 }
7024
7025 #[test]
7026 fn an_all_seats_unauthenticated_node_refunds_and_holds_without_retry() {
7027 let mut state = run_state(RunStatus::Blocked);
7028 let losses: Vec<_> = ["alpha", "beta"]
7029 .into_iter()
7030 .enumerate()
7031 .map(|(i, agent)| {
7032 QuotaLoss::from_report(
7033 format!("advisor-{i}"),
7034 "advise",
7035 Some(&crate::agent::Quota {
7036 cause: crate::agent::QuotaCause::Authentication,
7037 agent: agent.to_owned(),
7038 reset: None,
7039 }),
7040 )
7041 })
7042 .collect();
7043 crate::graph::block_machine_outage(&mut state, "advise", &losses);
7044 state.quota = losses;
7045 let mut t = task();
7046 t.start(state.id.clone());
7047 let verdict = Verdict {
7048 status: state.status,
7049 left_pr: false,
7050 quota_hit: true,
7051 parked: false,
7052 no_viable_candidates: true,
7053 };
7054 settle_and_diagnose(&mut t, verdict, "authentication unavailable", 2, &state);
7055 assert_eq!(t.attempts, 0);
7056 assert_eq!(t.status, TaskStatus::Held);
7057 assert!(!t.operator_held());
7058 let reason = t.hold_reason.as_deref().unwrap();
7059 assert!(reason.starts_with("[authentication]"));
7060 assert!(
7061 reason.contains("alpha")
7062 && reason.contains("beta")
7063 && reason.contains("re-authenticate")
7064 );
7065 assert!(cooldown_until(&state.quota, Timestamp::now()).is_none());
7066 let mut orphan = task();
7068 orphan.start(state.id.clone());
7069 reclaim(&mut orphan, Some(state), 2, "en");
7070 assert_eq!(orphan.attempts, 0);
7071 assert_eq!(orphan.status, TaskStatus::Held);
7072 }
7073
7074 #[test]
7075 fn a_resource_blocked_run_holds_its_task_and_spends_no_attempt() {
7076 let mut state = RunState::new(
7077 PathBuf::from("/repo"),
7078 "main".to_owned(),
7079 "abc1234".to_owned(),
7080 "task".to_owned(),
7081 Config::default(),
7082 );
7083 state.status = RunStatus::Blocked;
7084 state.resource_block = Some(crate::run::ResourceBlock {
7085 cause: crate::run::ResourceBlockCause::Other,
7086 node: "graft".to_owned(),
7087 kind: "shared build cache".to_owned(),
7088 detail: "timed out".to_owned(),
7089 at: jiff::Timestamp::now(),
7090 });
7091 let mut t = task();
7092 t.start("20260902-000000-aaaa".to_owned());
7093 assert_eq!(t.attempts, 1);
7094 let verdict = Verdict {
7095 status: RunStatus::Blocked,
7096 left_pr: false,
7097 quota_hit: false,
7098 parked: false,
7099 no_viable_candidates: false,
7100 };
7101 settle_and_diagnose(&mut t, verdict, "blocked", 2, &state);
7102 assert_eq!(t.status, TaskStatus::Held);
7103 assert_eq!(t.attempts, 0, "the attempt is refunded");
7104 assert!(t.hold_reason.as_deref().unwrap().starts_with("[resource] "));
7105 assert!(
7106 !t.operator_held(),
7107 "a machine hold, released without a human"
7108 );
7109
7110 let mut selected = task();
7111 selected.start("20260902-000000-cccc".into());
7112 selected.replacement = Some(crate::merge_decision::Replacement {
7113 question: "fixture".into(),
7114 run: "earlier".into(),
7115 pr: "https://example.invalid/pr/1".into(),
7116 agent: "selected".into(),
7117 context: "fixture".into(),
7118 });
7119 state.resource_block.as_mut().unwrap().kind = "repository coordination".into();
7120 settle_and_diagnose(&mut selected, verdict, "blocked", 2, &state);
7121 assert_eq!(selected.attempts, 0);
7122 assert!(
7123 !selected.operator_held(),
7124 "a selected agent also retains a machine resource hold"
7125 );
7126 assert!(
7127 selected
7128 .hold_reason
7129 .as_deref()
7130 .unwrap()
7131 .starts_with("[resource] ")
7132 );
7133
7134 let mut p = task();
7136 p.start("20260902-000000-bbbb".to_owned());
7137 settle_and_diagnose(
7138 &mut p,
7139 Verdict {
7140 parked: true,
7141 ..verdict
7142 },
7143 "parked",
7144 2,
7145 &state,
7146 );
7147 assert_eq!(p.status, TaskStatus::Parked);
7148 assert_eq!(
7149 p.attempts, 0,
7150 "parking refunds once rather than also settling the resource hold"
7151 );
7152 }
7153
7154 #[test]
7155 fn record_failure_notice_links_the_task() {
7156 let dir = tempfile::tempdir().unwrap();
7157 std::fs::write(dir.path().join("queue"), b"not a directory").unwrap();
7160 let queue = Queue::at(dir.path().join("queue"));
7161 let mut t = interrupt_task("rec-fail-1");
7162 record(&queue, &mut t);
7163
7164 let found = crate::notices::Notices::at(dir.path().join("notifications")).list();
7165 assert_eq!(found.len(), 1);
7166 assert_eq!(found[0].key, "loop:record");
7167 assert_eq!(
7168 found[0].link,
7169 Some(Link::Task {
7170 id: "rec-fail-1".to_owned()
7171 })
7172 );
7173 }
7174
7175 fn interrupt_task(id: &str) -> Task {
7178 let mut t = task();
7179 t.id = id.to_owned();
7180 t.interrupt = true;
7181 t
7182 }
7183
7184 fn task_with_id(id: &str) -> Task {
7186 let mut t = task();
7187 t.id = id.to_owned();
7188 t
7189 }
7190
7191 fn urgent_task(id: &str) -> Task {
7193 let mut t = task();
7194 t.id = id.to_owned();
7195 t.urgent = true;
7196 t
7197 }
7198
7199 #[test]
7200 fn upgrade_resume_dispatch_keeps_claim_and_lane_limits() {
7201 let dir = tempfile::tempdir().unwrap();
7202 let queue = Queue::at(dir.path().join("queue"));
7203 let mut parked = task_with_id("parked");
7204 parked.start("run".into());
7205 parked.park("upgrade");
7206 parked.record_pass_at(
7207 "run",
7208 0,
7209 Some(Timestamp::now()),
7210 1,
7211 Some(crate::run::ParkCause::Upgrade {
7212 from: "old".into(),
7213 to: None,
7214 }),
7215 );
7216 let mut fresh = task_with_id("fresh");
7217 fresh.priority = 100;
7218 let mut urgent_task = urgent_task("urgent");
7219 for t in [&mut parked, &mut fresh, &mut urgent_task] {
7220 queue.put(t).unwrap();
7221 }
7222 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7223 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7224 let candidates = runnable(&queue);
7225 assert_eq!(candidates[0].id, parked.id);
7226 let first = claim_dispatch(&queue, &candidates[0], false, &ordinary, &urgent).unwrap();
7227 assert!(claim_dispatch(&queue, &fresh, false, &ordinary, &urgent).is_none());
7228 assert!(claim_dispatch(&queue, &urgent_task, false, &ordinary, &urgent).is_some());
7229 drop(first);
7230 let claimed_elsewhere = queue.claim(&parked.id).unwrap();
7231 let next = runnable(&queue)
7232 .into_iter()
7233 .find_map(|t| claim_dispatch(&queue, &t, false, &ordinary, &urgent))
7234 .unwrap();
7235 assert_eq!(next.task.id, fresh.id);
7236 assert_eq!(queue.get(&parked.id).unwrap().attempts, 0);
7237 drop(claimed_elsewhere);
7238 drop(next);
7239 assert!(claim_dispatch(&queue, &parked, true, &ordinary, &urgent).is_none());
7240 }
7241
7242 #[test]
7243 fn urgent_dispatch_has_one_extra_slot_and_never_uses_a_stale_flag() {
7244 let dir = tempfile::tempdir().unwrap();
7245 let queue = Queue::at(dir.path().join("queue"));
7246 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7247 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7248 let occupied = Arc::clone(&ordinary).try_acquire_owned().unwrap();
7249 let mut first = task();
7250 queue.put(&mut first).unwrap();
7251 let stale_plain = queue.get(&first.id).unwrap();
7252 first.set_urgent(true).unwrap();
7253 queue.put(&mut first).unwrap();
7254 let dispatched = claim_dispatch(&queue, &stale_plain, false, &ordinary, &urgent).unwrap();
7255 let mut second = urgent_task("second");
7256 queue.put(&mut second).unwrap();
7257 assert!(claim_dispatch(&queue, &second, false, &ordinary, &urgent).is_none());
7258 drop(queue.claim(&second.id).unwrap());
7259 assert_eq!(queue.get(&second.id).unwrap().attempts, 0);
7260 drop(dispatched);
7261 let stale_urgent = queue.get(&first.id).unwrap();
7262 first.set_urgent(false).unwrap();
7263 queue.put(&mut first).unwrap();
7264 assert!(claim_dispatch(&queue, &stale_urgent, false, &ordinary, &urgent).is_none());
7265 assert_eq!(urgent.available_permits(), 1);
7266 drop(queue.claim(&first.id).unwrap());
7267 drop(occupied);
7268 }
7269
7270 #[test]
7271 fn urgent_dispatch_uses_saved_lane_and_releases_unsuccessful_claims() {
7272 let dir = tempfile::tempdir().unwrap();
7273 let queue = Queue::at(dir.path().join("queue"));
7274 let ordinary = Arc::new(tokio::sync::Semaphore::new(1));
7275 let urgent = Arc::new(tokio::sync::Semaphore::new(1));
7276 let mut t = task();
7277 queue.put(&mut t).unwrap();
7278 for enabled in [true, false] {
7279 let stale = queue.get(&t.id).unwrap();
7280 t.set_urgent(enabled).unwrap();
7281 queue.put(&mut t).unwrap();
7282 assert_ne!(stale.urgent, enabled);
7283 let DispatchClaim {
7284 task: fresh,
7285 claim,
7286 permit,
7287 } = claim_dispatch(&queue, &stale, false, &ordinary, &urgent).unwrap();
7288 assert_eq!(fresh.urgent, enabled);
7289 assert_eq!(ordinary.available_permits(), usize::from(enabled));
7290 assert_eq!(urgent.available_permits(), usize::from(!enabled));
7291 assert!(queue.claim(&t.id).is_err());
7292 assert_eq!(fresh.attempts, stale.attempts);
7293 drop((claim, permit));
7294 assert_eq!(ordinary.available_permits(), 1);
7295 assert_eq!(urgent.available_permits(), 1);
7296 }
7297 t.set_urgent(true).unwrap();
7298 queue.put(&mut t).unwrap();
7299 let occupied = Arc::clone(&urgent).try_acquire_owned().unwrap();
7300 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7301 assert_eq!(ordinary.available_permits(), 1, "no ordinary fallback");
7302 drop(queue.claim(&t.id).unwrap());
7303 assert_eq!(queue.get(&t.id).unwrap().attempts, t.attempts);
7304 drop(occupied);
7305 assert!(claim_dispatch(&queue, &t, true, &ordinary, &urgent).is_none());
7306 drop(queue.claim(&t.id).unwrap());
7307 let held = queue.claim(&t.id).unwrap();
7308 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7309 drop(held);
7310 t.status = TaskStatus::Held;
7311 queue.put(&mut t).unwrap();
7312 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7313 drop(queue.claim(&t.id).unwrap());
7314 std::fs::remove_file(queue.path_of(&t.id)).unwrap();
7315 assert!(claim_dispatch(&queue, &t, false, &ordinary, &urgent).is_none());
7316 drop(queue.claim(&t.id).unwrap());
7317 assert_eq!(urgent.available_permits(), 1);
7318 }
7319
7320 #[test]
7325 fn permit_kind_prefers_a_land_resume_over_the_urgent_slot() {
7326 assert_eq!(permit_kind(true, false), PermitKind::None);
7327 assert_eq!(permit_kind(true, true), PermitKind::None);
7328 }
7329
7330 #[test]
7335 fn permit_kind_separates_urgent_from_ordinary() {
7336 assert_eq!(permit_kind(false, true), PermitKind::Urgent);
7337 assert_eq!(permit_kind(false, false), PermitKind::Ordinary);
7338 }
7339
7340 #[test]
7347 fn disk_gate_with_holds_a_task_below_the_threshold_and_names_both_numbers() {
7348 let cfg = Config::default();
7349 let repo = Path::new("/any/repo/path");
7350
7351 let reason =
7352 disk_gate_with(repo, &cfg, |_| Ok(1024)).expect("must hold below the threshold");
7353 assert!(reason.contains("1024"), "{reason}");
7354 assert!(
7355 reason.contains(&cfg.disk.min_free_bytes.to_string()),
7356 "{reason}"
7357 );
7358
7359 assert_eq!(
7360 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes)),
7361 None,
7362 "exactly at the floor is open"
7363 );
7364 assert_eq!(
7365 disk_gate_with(repo, &cfg, |_| Ok(cfg.disk.min_free_bytes + 1)),
7366 None,
7367 "comfortably above the floor is open"
7368 );
7369 }
7370
7371 #[test]
7372 fn disk_gate_with_opens_unconditionally_when_the_operator_opted_out() {
7373 let mut cfg = Config::default();
7374 cfg.disk.min_free_bytes = 0;
7375 let repo = Path::new("/any/repo/path");
7376 assert_eq!(
7377 disk_gate_with(repo, &cfg, |_| Ok(0)),
7378 None,
7379 "a zero floor never measures at all"
7380 );
7381 }
7382
7383 #[test]
7384 fn disk_gate_with_closes_rather_than_starts_blind_when_it_cannot_measure() {
7385 let cfg = Config::default();
7386 let repo = Path::new("/any/repo/path");
7387 let reason = disk_gate_with(repo, &cfg, |_| Err(anyhow::anyhow!("no df on this box")))
7388 .expect("a measurement failure must close the gate, not open it");
7389 assert!(reason.contains("could not measure"), "{reason}");
7390 }
7391
7392 #[test]
7393 fn no_interrupt_task_leaves_the_sequence_idle_even_with_something_in_flight() {
7394 let ordinary = task();
7395 let next = advance_interrupt(
7396 Interrupt::Idle,
7397 std::slice::from_ref(&ordinary.id),
7398 std::slice::from_ref(&ordinary),
7399 );
7400 assert_eq!(next, Interrupt::Idle);
7401 }
7402
7403 #[test]
7404 fn an_interrupt_task_with_nothing_in_flight_never_starts_a_sequence() {
7405 let marked = interrupt_task("marked");
7408 let next = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7409 assert_eq!(next, Interrupt::Idle);
7410 }
7411
7412 #[test]
7413 fn an_interrupt_task_with_something_in_flight_starts_parking_it() {
7414 let marked = interrupt_task("marked");
7415 let next = advance_interrupt(
7416 Interrupt::Idle,
7417 &["running".to_owned()],
7418 std::slice::from_ref(&marked),
7419 );
7420 assert_eq!(
7421 next,
7422 Interrupt::Parking {
7423 parked: vec!["running".to_owned()],
7424 interrupt_task: "marked".to_owned(),
7425 }
7426 );
7427 }
7428
7429 #[test]
7438 fn more_than_one_run_in_flight_never_starts_an_interrupt_sequence() {
7439 let marked = interrupt_task("marked");
7440
7441 let two = advance_interrupt(
7442 Interrupt::Idle,
7443 &["a".to_owned(), "b".to_owned()],
7444 std::slice::from_ref(&marked),
7445 );
7446 assert_eq!(two, Interrupt::Idle);
7447
7448 let none = advance_interrupt(Interrupt::Idle, &[], std::slice::from_ref(&marked));
7449 assert_eq!(none, Interrupt::Idle, "nothing to interrupt either");
7450 }
7451
7452 #[test]
7453 fn parking_holds_until_every_parked_id_has_actually_left_flight() {
7454 let state = Interrupt::Parking {
7455 parked: vec!["running".to_owned()],
7456 interrupt_task: "marked".to_owned(),
7457 };
7458 let still_going = advance_interrupt(state.clone(), &["running".to_owned()], &[]);
7460 assert_eq!(still_going, state);
7461
7462 let stopped_but_not_yet_dispatched =
7466 advance_interrupt(state.clone(), &[], &[interrupt_task("marked")]);
7467 assert_eq!(stopped_but_not_yet_dispatched, state);
7468
7469 let dispatched = advance_interrupt(state, &["marked".to_owned()], &[]);
7471 assert_eq!(
7472 dispatched,
7473 Interrupt::Running {
7474 parked: vec!["running".to_owned()],
7475 interrupt_task: "marked".to_owned(),
7476 }
7477 );
7478 }
7479
7480 #[test]
7481 fn the_sequence_moves_to_resuming_the_instant_the_interrupt_tasks_own_run_leaves_flight() {
7482 let state = Interrupt::Running {
7483 parked: vec!["running".to_owned()],
7484 interrupt_task: "marked".to_owned(),
7485 };
7486 let still_running = advance_interrupt(state.clone(), &["marked".to_owned()], &[]);
7487 assert_eq!(still_running, state);
7488
7489 let ended = advance_interrupt(state, &[], &[task_with_id("running")]);
7496 assert_eq!(
7497 ended,
7498 Interrupt::Resuming {
7499 parked: vec!["running".to_owned()]
7500 }
7501 );
7502 }
7503
7504 #[test]
7505 fn resuming_ends_the_instant_a_parked_task_is_seen_in_flight() {
7506 let state = Interrupt::Resuming {
7507 parked: vec!["running".to_owned()],
7508 };
7509 let still_waiting = advance_interrupt(state.clone(), &[], &[task_with_id("running")]);
7510 assert_eq!(still_waiting, state);
7511
7512 let dispatched = advance_interrupt(state, &["running".to_owned()], &[]);
7513 assert_eq!(dispatched, Interrupt::Idle);
7514 }
7515
7516 #[test]
7522 fn an_interrupt_task_that_stops_being_runnable_abandons_the_wait_without_losing_the_parked_run()
7523 {
7524 let state = Interrupt::Parking {
7525 parked: vec!["running".to_owned()],
7526 interrupt_task: "marked".to_owned(),
7527 };
7528 let next = advance_interrupt(state, &[], &[]);
7531 assert_eq!(
7532 next,
7533 Interrupt::Resuming {
7534 parked: vec!["running".to_owned()]
7535 },
7536 "abandoning the interrupt must not abandon the resume it owes"
7537 );
7538 }
7539
7540 #[test]
7543 fn resuming_abandons_a_parked_task_that_stops_being_runnable() {
7544 let state = Interrupt::Resuming {
7545 parked: vec!["running".to_owned()],
7546 };
7547 let next = advance_interrupt(state, &[], &[]);
7548 assert_eq!(
7549 next,
7550 Interrupt::Idle,
7551 "nothing is left to wait for; the loop must not stay wedged"
7552 );
7553 }
7554
7555 #[test]
7556 fn disabled_by_config_the_sequence_can_never_leave_idle() {
7557 let marked = interrupt_task("marked");
7558 let next = advance_interrupt_tick(
7559 false,
7560 Interrupt::Idle,
7561 &["running".to_owned()],
7562 std::slice::from_ref(&marked),
7563 );
7564 assert_eq!(
7565 next,
7566 Interrupt::Idle,
7567 "an unmarked, unconfigured daemon must behave exactly as before"
7568 );
7569 }
7570
7571 #[test]
7572 fn the_gate_blocks_everyone_while_something_parked_is_still_in_flight() {
7573 let state = Interrupt::Parking {
7574 parked: vec!["running".to_owned()],
7575 interrupt_task: "marked".to_owned(),
7576 };
7577 let candidates = vec![interrupt_task("marked"), task()];
7578 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7579 assert!(
7580 allowed.is_empty(),
7581 "nothing may dispatch - not even the interrupt task itself - \
7582 until the parked run has actually stopped"
7583 );
7584 }
7585
7586 #[test]
7600 fn urgent_gains_no_exemption_from_an_active_interrupt_sequence() {
7601 for state in [
7602 Interrupt::Parking {
7603 parked: vec!["running".to_owned()],
7604 interrupt_task: "marked".to_owned(),
7605 },
7606 Interrupt::Running {
7607 parked: vec!["running".to_owned()],
7608 interrupt_task: "marked".to_owned(),
7609 },
7610 Interrupt::Resuming {
7611 parked: vec!["running".to_owned()],
7612 },
7613 ] {
7614 let candidates = vec![
7615 interrupt_task("marked"),
7616 urgent_task("hot"),
7617 task_with_id("ordinary"),
7618 ];
7619 let allowed = interrupt_gate(&state, &["running".to_owned()], candidates);
7620 assert!(
7621 !allowed.iter().any(|t| t.id == "hot"),
7622 "an urgent candidate must wait out the same gate as anything \
7623 else while the run it would run alongside has not actually \
7624 left flight, for state {state:?}: {allowed:?}"
7625 );
7626 }
7627 }
7628
7629 #[test]
7635 fn a_task_marked_both_urgent_and_interrupt_is_admitted_once_the_gate_itself_says_so() {
7636 let state = Interrupt::Resuming {
7637 parked: vec!["hot".to_owned()],
7638 };
7639 let candidates = vec![urgent_task("hot"), task()];
7640 let allowed = interrupt_gate(&state, &[], candidates);
7641 assert_eq!(
7642 allowed.iter().filter(|t| t.id == "hot").count(),
7643 1,
7644 "the gate's own decision is unaffected by the urgent flag: {allowed:?}"
7645 );
7646 }
7647
7648 #[test]
7649 fn the_gate_lets_only_the_interrupt_task_through_once_parked_work_has_stopped() {
7650 let state = Interrupt::Parking {
7651 parked: vec!["running".to_owned()],
7652 interrupt_task: "marked".to_owned(),
7653 };
7654 let other = task();
7655 let candidates = vec![interrupt_task("marked"), other.clone()];
7656 let allowed = interrupt_gate(&state, &[], candidates);
7657 assert_eq!(allowed.len(), 1);
7658 assert_eq!(allowed[0].id, "marked");
7659 }
7660
7661 #[test]
7662 fn the_gate_blocks_everyone_while_the_interrupt_task_itself_is_in_flight() {
7663 let state = Interrupt::Running {
7664 parked: vec!["running".to_owned()],
7665 interrupt_task: "marked".to_owned(),
7666 };
7667 let candidates = vec![task(), task()];
7668 let allowed = interrupt_gate(&state, &["marked".to_owned()], candidates);
7669 assert!(allowed.is_empty());
7670 }
7671
7672 #[test]
7679 fn the_gate_offers_at_most_one_candidate_while_resuming_even_with_two_parked() {
7680 let state = Interrupt::Resuming {
7681 parked: vec!["a".to_owned(), "c".to_owned()],
7682 };
7683 let candidates = vec![task_with_id("a"), task_with_id("c"), task_with_id("other")];
7684 let allowed = interrupt_gate(&state, &[], candidates);
7685 assert_eq!(
7686 allowed.len(),
7687 1,
7688 "at most one candidate may be offered while resuming: {allowed:?}"
7689 );
7690 assert_eq!(allowed[0].id, "a");
7691 }
7692
7693 #[test]
7694 fn the_gate_offers_nothing_while_resuming_if_no_parked_task_is_runnable() {
7695 let state = Interrupt::Resuming {
7696 parked: vec!["a".to_owned()],
7697 };
7698 let allowed = interrupt_gate(&state, &[], vec![task_with_id("other")]);
7699 assert!(allowed.is_empty());
7700 }
7701
7702 #[test]
7708 fn a_full_sequence_never_gates_two_runs_through_at_once_and_resumes_exactly_one() {
7709 let running = task(); let marked = interrupt_task("marked");
7711
7712 let mut state = Interrupt::Idle;
7713 let in_flight = vec![running.id.clone()];
7715 state = advance_interrupt_tick(true, state, &in_flight, std::slice::from_ref(&marked));
7716 let gated = interrupt_gate(&state, &in_flight, vec![marked.clone(), running.clone()]);
7717 assert!(gated.is_empty(), "still waiting on `running` to park");
7718
7719 state = advance_interrupt_tick(true, state, &[], &[marked.clone(), running.clone()]);
7721 let gated = interrupt_gate(&state, &[], vec![marked.clone(), running.clone()]);
7722 assert_eq!(
7723 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7724 vec!["marked"],
7725 "only the interrupt task may be offered to the dispatcher now"
7726 );
7727
7728 state = advance_interrupt_tick(
7730 true,
7731 state,
7732 &["marked".to_owned()],
7733 std::slice::from_ref(&running),
7734 );
7735 let gated = interrupt_gate(
7736 &state,
7737 &["marked".to_owned()],
7738 vec![marked.clone(), running.clone()],
7739 );
7740 assert!(
7741 gated.is_empty(),
7742 "the parked run must not be offered back while the interrupt \
7743 task is still running"
7744 );
7745
7746 let other = task_with_id("other");
7750 state = advance_interrupt_tick(true, state, &[], &[running.clone(), other.clone()]);
7751 assert_eq!(
7752 state,
7753 Interrupt::Resuming {
7754 parked: vec![running.id.clone()]
7755 }
7756 );
7757 let gated = interrupt_gate(&state, &[], vec![other.clone(), running.clone()]);
7758 assert_eq!(
7759 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7760 vec![running.id.as_str()],
7761 "exactly the parked run resumes - not the unrelated task, even \
7762 though it was offered first"
7763 );
7764
7765 state = advance_interrupt_tick(
7769 true,
7770 state,
7771 std::slice::from_ref(&running.id),
7772 std::slice::from_ref(&other),
7773 );
7774 assert_eq!(state, Interrupt::Idle);
7775 let gated = interrupt_gate(
7776 &state,
7777 std::slice::from_ref(&running.id),
7778 vec![other.clone()],
7779 );
7780 assert_eq!(
7781 gated.iter().map(|t| t.id.as_str()).collect::<Vec<_>>(),
7782 vec![other.id.as_str()],
7783 "ordinary dispatch is unrestricted again"
7784 );
7785 }
7786
7787 #[test]
7788 fn every_run_status_settles_the_task_it_came_from() {
7789 let table = [
7791 (RunStatus::Merged, TaskStatus::Done, 1),
7792 (RunStatus::Ready, TaskStatus::Done, 1),
7793 (RunStatus::Stalled, TaskStatus::Failed, 0),
7794 (RunStatus::Blocked, TaskStatus::Failed, 1),
7795 (RunStatus::Failed, TaskStatus::Failed, 1),
7796 (RunStatus::VerifiedNoop, TaskStatus::Held, 1),
7797 (RunStatus::Prep, TaskStatus::Failed, 1),
7798 (RunStatus::Implementing, TaskStatus::Failed, 1),
7799 (RunStatus::Judging, TaskStatus::Failed, 1),
7800 (RunStatus::Deliberating, TaskStatus::Failed, 1),
7801 (RunStatus::Voting, TaskStatus::Failed, 1),
7802 (RunStatus::Reviewing, TaskStatus::Failed, 1),
7803 (RunStatus::Gating, TaskStatus::Failed, 1),
7804 ];
7805 for (run, want, attempts) in table {
7806 let mut t = task();
7807 t.start("20260902-000000-aaaa".to_owned());
7808 settle(
7809 &mut t,
7810 Verdict {
7811 status: run,
7812 left_pr: false,
7813 parked: false,
7814 quota_hit: matches!(run, RunStatus::Stalled),
7815 no_viable_candidates: false,
7816 },
7817 "why",
7818 2,
7819 );
7820 assert_eq!(t.status, want, "task status after {}", label(run));
7821 assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
7822 }
7823 }
7824
7825 #[test]
7826 fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
7827 let mut stalled = task();
7828 stalled.start("20260902-000000-aaaa".to_owned());
7829 settle(
7830 &mut stalled,
7831 Verdict {
7832 status: RunStatus::Stalled,
7833 left_pr: false,
7834 parked: false,
7835 quota_hit: true,
7836 no_viable_candidates: false,
7837 },
7838 "quota",
7839 1,
7840 );
7841 assert_eq!(stalled.attempts, 0);
7842 assert!(
7843 stalled.status.runnable(),
7844 "a machine problem must leave the task in line"
7845 );
7846
7847 let mut blocked = task();
7848 blocked.start("20260902-000000-aaaa".to_owned());
7849 settle(
7850 &mut blocked,
7851 Verdict {
7852 status: RunStatus::Blocked,
7853 left_pr: false,
7854 parked: false,
7855 quota_hit: false,
7856 no_viable_candidates: false,
7857 },
7858 "findings open",
7859 1,
7860 );
7861 assert_eq!(blocked.attempts, 1);
7862 assert_eq!(
7863 blocked.status,
7864 TaskStatus::Held,
7865 "the last attempt hands the task to a human"
7866 );
7867 }
7868
7869 #[test]
7870 fn a_run_that_opened_a_pull_request_is_never_re_competed() {
7871 let mut delivered = task();
7874 delivered.start("20260903-080619-01c2".to_owned());
7875 settle(
7876 &mut delivered,
7877 Verdict {
7878 status: RunStatus::Blocked,
7879 left_pr: true,
7880 parked: false,
7881 quota_hit: false,
7882 no_viable_candidates: false,
7883 },
7884 "no check status",
7885 4,
7886 );
7887 assert_eq!(
7888 delivered.status,
7889 TaskStatus::Held,
7890 "a pull request waiting on CI or a person is not a retryable failure"
7891 );
7892 assert!(
7893 !delivered.status.runnable(),
7894 "the loop must not pick this task up again"
7895 );
7896 assert_eq!(
7897 delivered.last_error.as_deref(),
7898 Some("no check status"),
7899 "the operator needs to be told what the gate was waiting for"
7900 );
7901
7902 let mut empty_handed = task();
7905 empty_handed.start("20260903-080619-01c2".to_owned());
7906 settle(
7907 &mut empty_handed,
7908 Verdict {
7909 status: RunStatus::Blocked,
7910 left_pr: false,
7911 parked: false,
7912 quota_hit: false,
7913 no_viable_candidates: false,
7914 },
7915 "findings open",
7916 4,
7917 );
7918 assert_eq!(empty_handed.status, TaskStatus::Failed);
7919 assert!(empty_handed.status.runnable());
7920 }
7921
7922 #[test]
7923 fn a_verified_noop_run_hands_off_rather_than_closing_or_auto_retrying() {
7924 let mut noop = task();
7930 noop.start("20260912-131304-391f".to_owned());
7931 settle(
7932 &mut noop,
7933 Verdict {
7934 status: RunStatus::VerifiedNoop,
7935 left_pr: false,
7936 parked: false,
7937 quota_hit: false,
7938 no_viable_candidates: true,
7939 },
7940 "candidate A: already fixed by b32cfc4, on main",
7941 4,
7942 );
7943 assert_eq!(
7944 noop.status,
7945 TaskStatus::Held,
7946 "an unverified claim is a request for a human, not a failure"
7947 );
7948 assert!(
7949 !noop.status.runnable(),
7950 "the loop must not requeue this on the same unverified claim"
7951 );
7952 assert_eq!(noop.attempts, 1);
7957 }
7958
7959 #[test]
7960 fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
7961 let mut parked = task();
7966 parked.start("20260903-183634-2d98".to_owned());
7967 settle(
7968 &mut parked,
7969 Verdict {
7970 status: RunStatus::Implementing,
7971 left_pr: false,
7972 quota_hit: false,
7973 parked: true,
7974 no_viable_candidates: false,
7975 },
7976 "parked after `implementing`",
7977 2,
7978 );
7979 assert_eq!(parked.attempts, 0, "a park is refunded");
7980 assert!(
7981 parked.status.runnable(),
7982 "and the task stays in line so the next loop resumes its run"
7983 );
7984 assert_eq!(parked.status, TaskStatus::Parked);
7985 assert_eq!(
7986 parked.park_reason.as_deref(),
7987 Some("parked after `implementing`"),
7988 "the card says where it stopped"
7989 );
7990 assert_eq!(parked.last_error, None, "nothing failed");
7991
7992 let mut broken = task();
7996 broken.start("20260903-183634-2d98".to_owned());
7997 settle(
7998 &mut broken,
7999 Verdict {
8000 status: RunStatus::Implementing,
8001 left_pr: false,
8002 quota_hit: false,
8003 parked: false,
8004 no_viable_candidates: false,
8005 },
8006 "returned mid-flight",
8007 2,
8008 );
8009 assert_eq!(broken.attempts, 1);
8010 }
8011
8012 #[test]
8013 fn only_a_rate_limit_buys_the_task_its_attempt_back() {
8014 let mut flaky = task();
8019 flaky.start("20260903-123023-e633".to_owned());
8020 settle(
8021 &mut flaky,
8022 Verdict {
8023 status: RunStatus::Stalled,
8024 left_pr: false,
8025 parked: false,
8026 quota_hit: false,
8027 no_viable_candidates: false,
8028 },
8029 "verdict rests on 1 of 3 judges",
8030 2,
8031 );
8032 assert_eq!(
8033 flaky.attempts, 1,
8034 "flakiness spends an attempt, so `max_attempts` still bounds it"
8035 );
8036 assert!(flaky.status.runnable(), "and it is still worth retrying");
8037
8038 let mut limited = task();
8040 limited.start("20260903-123023-e633".to_owned());
8041 settle(
8042 &mut limited,
8043 Verdict {
8044 status: RunStatus::Stalled,
8045 left_pr: false,
8046 parked: false,
8047 quota_hit: true,
8048 no_viable_candidates: false,
8049 },
8050 "judge-2, judge-3 out of quota",
8051 2,
8052 );
8053 assert_eq!(limited.attempts, 0, "a quota window is refunded");
8054 assert!(limited.status.runnable());
8055
8056 let mut worn = task();
8059 for _ in 0..2 {
8060 worn.release();
8061 }
8062 worn.start("20260903-123023-e633".to_owned());
8063 worn.attempts = 2;
8064 settle(
8065 &mut worn,
8066 Verdict {
8067 status: RunStatus::Stalled,
8068 left_pr: false,
8069 parked: false,
8070 quota_hit: false,
8071 no_viable_candidates: false,
8072 },
8073 "no quorum again",
8074 2,
8075 );
8076 assert_eq!(worn.status, TaskStatus::Held);
8077 assert!(!worn.status.runnable());
8078 }
8079
8080 #[test]
8081 fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
8082 let mut wiped_out = task();
8089 wiped_out.start("20260907-025000-a1b2".to_owned());
8090 settle(
8091 &mut wiped_out,
8092 Verdict {
8093 status: RunStatus::Failed,
8094 left_pr: false,
8095 parked: false,
8096 quota_hit: true,
8097 no_viable_candidates: true,
8098 },
8099 "no candidate produced a change; nothing to judge",
8100 2,
8101 );
8102 assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
8103 assert!(
8104 wiped_out.status.runnable(),
8105 "a machine problem must leave the task in line"
8106 );
8107
8108 let mut partial_progress = task();
8114 partial_progress.start("20260907-025500-c3d4".to_owned());
8115 settle(
8116 &mut partial_progress,
8117 Verdict {
8118 status: RunStatus::Failed,
8119 left_pr: false,
8120 parked: false,
8121 quota_hit: true,
8122 no_viable_candidates: false,
8123 },
8124 "gate failed on the winning candidate",
8125 2,
8126 );
8127 assert_eq!(
8128 partial_progress.attempts, 1,
8129 "a candidate that actually produced a change spends the attempt \
8130 even though some other seat hit its quota"
8131 );
8132 assert!(partial_progress.status.runnable());
8133 }
8134
8135 #[test]
8136 fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
8137 let mut t = task();
8143 t.start("20260907-025000-a1b2".to_owned());
8144 let mut state = run_state(RunStatus::Failed);
8145 state.quota.push(QuotaLoss {
8146 cause: crate::agent::QuotaCause::UsageLimit,
8147 agent: String::new(),
8148 seat: "cand-a".to_owned(),
8149 node: "implement".to_owned(),
8150 at: Timestamp::now(),
8151 reset: None,
8152 });
8153 assert!(
8154 state.viable().is_empty(),
8155 "no candidate was added, so nothing is viable"
8156 );
8157 reclaim(&mut t, Some(state), 2, "en");
8158 assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
8159 assert!(t.status.runnable());
8160 }
8161
8162 #[test]
8163 fn a_held_task_is_never_offered_to_the_loop() {
8164 let dir = tempfile::tempdir().unwrap();
8165 let queue = Queue::at(dir.path().to_path_buf());
8166 for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
8167 let mut t = task();
8168 t.id = format!("2026090{n}-000000-000{n}");
8169 t.priority = priority;
8170 queue.put(&mut t).unwrap();
8171 }
8172 let mut held = task();
8173 held.id = "20260909-000000-9999".to_owned();
8174 held.priority = 99;
8175 held.hold_machine(None);
8176 queue.put(&mut held).unwrap();
8177
8178 let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
8179 assert_eq!(order.len(), 3);
8180 assert!(!order.contains(&held.id));
8181 assert_eq!(
8182 order.first().cloned(),
8183 queue.next_runnable().map(|t| t.id),
8184 "the loop's first candidate is exactly what the queue offers"
8185 );
8186 assert_eq!(
8187 order,
8188 vec![
8189 "20260902-000000-0002".to_owned(),
8190 "20260903-000000-0003".to_owned(),
8191 "20260901-000000-0001".to_owned(),
8192 ],
8193 "priority first, then oldest, so nothing starves"
8194 );
8195 }
8196
8197 #[test]
8198 fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
8199 let dir = tempfile::tempdir().unwrap();
8200 let queue = Queue::at(dir.path().to_path_buf());
8201 let mut old = task();
8202 old.id = "20260101-000000-old0".to_owned();
8203 queue.put(&mut old).unwrap();
8204 let mut fresh = task();
8205 fresh.id = "20260101-000000-new0".to_owned();
8206 queue.put(&mut fresh).unwrap();
8207
8208 std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
8212 std::thread::sleep(Duration::from_millis(60));
8213 let live = queue.claim(&fresh.id).unwrap();
8214
8215 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8216 assert_eq!(swept, vec![old.id.clone()]);
8217 assert!(
8218 queue.claim(&old.id).is_ok(),
8219 "an unparseable lock older than the threshold is swept"
8220 );
8221 assert!(
8222 queue.claim(&fresh.id).is_err(),
8223 "a live pid protects its lock regardless of age"
8224 );
8225 drop(live);
8226 }
8227
8228 #[test]
8229 fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
8230 let dir = tempfile::tempdir().unwrap();
8233 let queue = Queue::at(dir.path().to_path_buf());
8234 let mut t = task();
8235 t.id = "20260101-000000-live".to_owned();
8236 queue.put(&mut t).unwrap();
8237
8238 let claim = queue.claim(&t.id).unwrap();
8239 std::thread::sleep(Duration::from_millis(60));
8240
8241 let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
8242 assert!(
8243 swept.is_empty(),
8244 "a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
8245 );
8246 assert!(
8247 queue.claim(&t.id).is_err(),
8248 "the lock still protects its task"
8249 );
8250 drop(claim);
8251 }
8252
8253 fn injected_dead_pid() -> u32 {
8256 std::process::id().checked_add(1).unwrap_or(1)
8257 }
8258
8259 #[test]
8260 fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
8261 let dir = tempfile::tempdir().unwrap();
8262 let queue = Queue::at(dir.path().to_path_buf());
8263 let mut t = task();
8264 t.id = "20260101-000000-dead".to_owned();
8265 queue.put(&mut t).unwrap();
8266 let dead_pid = injected_dead_pid();
8267
8268 std::fs::write(
8273 dir.path().join(format!("{}.lock", t.id)),
8274 dead_pid.to_string(),
8275 )
8276 .unwrap();
8277
8278 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8279 pid != dead_pid
8280 });
8281 assert_eq!(
8282 swept,
8283 vec![t.id.clone()],
8284 "a dead owner is reclaimed immediately, not after STALE_CLAIM"
8285 );
8286 assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
8287 }
8288
8289 #[test]
8290 fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
8291 let dir = tempfile::tempdir().unwrap();
8292 let queue = Queue::at(dir.path().to_path_buf());
8293 let mut t = task();
8294 t.id = "20260101-000000-late".to_owned();
8295 queue.put(&mut t).unwrap();
8296 let dead_pid = injected_dead_pid();
8297
8298 assert!(
8301 sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
8302 "nothing has claimed the task yet"
8303 );
8304
8305 std::fs::write(
8308 dir.path().join(format!("{}.lock", t.id)),
8309 dead_pid.to_string(),
8310 )
8311 .unwrap();
8312
8313 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8317 pid != dead_pid
8318 });
8319 assert_eq!(swept, vec![t.id.clone()]);
8320 }
8321
8322 #[test]
8323 fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
8324 crate::run::pin_test_home();
8329 let dir = tempfile::tempdir().unwrap();
8330 let queue = Queue::at(dir.path().to_path_buf());
8331 let mut t = task();
8332 t.id = "20260101-000000-crsh".to_owned();
8333 t.status = TaskStatus::Running;
8334 t.attempts = 1;
8335 t.runs.push("20260904-000000-4043".to_owned());
8339 queue.put(&mut t).unwrap();
8340 let dead_pid = injected_dead_pid();
8341
8342 std::fs::write(
8345 dir.path().join(format!("{}.lock", t.id)),
8346 dead_pid.to_string(),
8347 )
8348 .unwrap();
8349
8350 assert!(reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty());
8356 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
8357
8358 let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
8359 pid != dead_pid
8360 });
8361 assert_eq!(swept, vec![t.id.clone()]);
8362
8363 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
8364 assert_eq!(reclaimed, vec![t.id.clone()]);
8365 let after = queue.get(&t.id).unwrap();
8366 assert_eq!(
8367 after.status,
8368 TaskStatus::Held,
8369 "no run.json to recover from, so a human is asked"
8370 );
8371 assert_eq!(
8372 after.runs,
8373 vec!["20260904-000000-4043".to_owned()],
8374 "the crashed run's id is kept as evidence, not discarded"
8375 );
8376 }
8377
8378 #[test]
8379 fn a_lock_is_kept_when_the_process_query_is_unavailable() {
8380 let dir = tempfile::tempdir().unwrap();
8381 let queue = Queue::at(dir.path().to_path_buf());
8382 let mut t = task();
8383 t.id = "20260101-000000-unknown".to_owned();
8384 queue.put(&mut t).unwrap();
8385 let dead_pid = injected_dead_pid();
8386 std::fs::write(
8387 dir.path().join(format!("{}.lock", t.id)),
8388 dead_pid.to_string(),
8389 )
8390 .unwrap();
8391
8392 let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
8393 assert!(swept.is_empty(), "an unknown pid must keep its lock");
8394 assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
8395 }
8396
8397 fn run_state_in(status: RunStatus, language: &str) -> RunState {
8398 let mut s = run_state(status);
8399 s.config.graph.language = language.to_owned();
8400 s
8401 }
8402
8403 fn unstarted_verdict(status: RunStatus) -> Verdict {
8404 Verdict {
8405 status,
8406 left_pr: false,
8407 quota_hit: false,
8408 parked: false,
8409 no_viable_candidates: false,
8410 }
8411 }
8412
8413 #[test]
8414 fn an_already_in_base_run_finishes_the_task_without_spending_an_attempt() {
8415 let mut t = task();
8416 t.attempts = 1;
8417 settle_in(
8418 &mut t,
8419 unstarted_verdict(RunStatus::AlreadyInBase),
8420 "already in main",
8421 1,
8422 phrases("en"),
8423 );
8424 assert_eq!(t.status, TaskStatus::Done);
8425 assert_eq!(t.attempts, 0);
8426 }
8427
8428 #[test]
8429 fn settle_renders_the_non_terminal_reason_in_the_configured_language() {
8430 let reason = |language: &str| {
8431 let mut t = task();
8432 settle_in(
8433 &mut t,
8434 unstarted_verdict(RunStatus::Judging),
8435 "boom",
8436 1,
8437 phrases(language),
8438 );
8439 t.last_error.or(t.hold_reason).unwrap_or_default()
8440 };
8441 assert!(
8442 reason("en").starts_with("the graph stopped at `"),
8443 "{}",
8444 reason("en")
8445 );
8446 assert!(reason("ja").starts_with("グラフが終端状態に達しないまま"));
8447 assert!(reason("日本語").contains("boom"));
8448 assert_eq!(reason("fr"), reason("en"));
8449 }
8450
8451 #[test]
8452 fn describe_follows_the_run_language_and_keeps_the_run_id() {
8453 let en = describe(&run_state_in(RunStatus::Stalled, "en"));
8454 assert!(en.starts_with("the judging panel lost its quorum"), "{en}");
8455 let ja = describe(&run_state_in(RunStatus::Stalled, "ja"));
8456 assert!(ja.starts_with("審査パネルが定足数を失いました"), "{ja}");
8457 assert!(ja.contains("[run "), "{ja}");
8458 let ended = describe(&run_state_in(RunStatus::Failed, "jp"));
8459 assert!(ended.starts_with("run 終了: "), "{ended}");
8460 let mut de = run_state_in(RunStatus::Failed, "de");
8461 let mut en = run_state_in(RunStatus::Failed, "en");
8462 de.id = "same".to_owned();
8463 en.id = "same".to_owned();
8464 assert_eq!(describe(&de), describe(&en));
8465 }
8466
8467 #[test]
8468 fn handover_refusals_follow_the_language_and_keep_the_detail_apart() {
8469 use crate::handover::Refused;
8470 let cases = [
8471 Refused::Foreign {
8472 branch: "b".into(),
8473 path: "/w/x".into(),
8474 why: "made by hand".into(),
8475 },
8476 Refused::Unsafe {
8477 branch: "b".into(),
8478 path: "/w/x".into(),
8479 why: "its worktree has uncommitted changes (a.rs)".into(),
8480 },
8481 Refused::ReleaseFailed {
8482 branch: "b".into(),
8483 path: "/w/x".into(),
8484 run: "ab12".into(),
8485 },
8486 ];
8487 for r in &cases {
8488 let en = (phrases("en").handover_refused)(r);
8489 assert_eq!(en, r.to_string());
8490 let ja = (phrases("ja").handover_refused)(r);
8491 assert!(
8492 !ja.contains("is checked out") && !ja.contains("try again"),
8493 "{ja}"
8494 );
8495 assert!(ja.contains("`b`") && ja.contains("/w/x"), "{ja}");
8496 if let Refused::Foreign { why, .. } | Refused::Unsafe { why, .. } = r {
8497 assert!(ja.contains(&format!("(詳細: {why})")), "{ja}");
8498 }
8499 }
8500 assert!(phrases("en").handover_hint.contains("release the task"));
8501 assert!(phrases("ja").handover_hint.contains("解放"));
8502 }
8503
8504 #[test]
8505 fn describe_translates_the_unanswered_reviewer_stop_only_in_ja() {
8506 let build = |lang: &str| {
8507 let mut s = run_state_in(RunStatus::Blocked, lang);
8508 s.id = "same".to_owned();
8509 s.config.graph.review_rounds = 3;
8510 let mut r = review_round(3);
8511 r.expected = 3;
8512 r.answered = 1;
8513 s.reviews.push(r);
8514 s.event(
8515 "review",
8516 "2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean",
8517 );
8518 s
8519 };
8520 let en = describe(&build("en"));
8521 assert!(en.contains("(review: 2 reviewer seat(s) never answered after 3 rounds; refusing to call it clean)"), "{en}");
8522 let ja = describe(&build("ja"));
8523 assert!(ja.contains("2 席のレビュアーが 3 ラウンド"), "{ja}");
8524 assert!(!ja.contains("never answered"), "{ja}");
8525 assert!(ja.contains("[run "), "{ja}");
8526
8527 let mut failed = build("ja");
8530 failed.reviews[0].e2e.push(crate::run::CommandOutcome {
8531 command: "cargo test".to_owned(),
8532 code: Some(1),
8533 output_tail: String::new(),
8534 duration_ms: 0,
8535 resource_blocked: false,
8536 });
8537 failed.event("review", "stopped; e2e failed: cargo test");
8538 let ja = describe(&failed);
8539 assert!(ja.contains("stopped; e2e failed: cargo test"), "{ja}");
8540 assert!(!ja.contains("席のレビュアー"), "{ja}");
8541 }
8542
8543 #[test]
8544 fn refusals_and_recovery_prose_follow_the_language() {
8545 let home = crate::run::test_home();
8546 let t = held_task_with("r1");
8547 let q = action_question(
8548 "r1",
8549 ask::ChoiceAction::Resume {
8550 run: "r1".to_owned(),
8551 },
8552 );
8553 let refuse = |p: &Phrases| match decide_action(&home, &t, &q, p, |_: &str| bail!("gone")) {
8554 ActionDecision::Refuse(s) => s,
8555 other => panic!("{other:?}"),
8556 };
8557 assert!(refuse(phrases("en")).contains("could not be read: gone"));
8558 assert!(refuse(phrases("ja")).contains("読み込めませんでした: gone"));
8559 assert_eq!(refuse(phrases("fr")), refuse(phrases("en")));
8560
8561 let mut held = task();
8562 reclaim(&mut held, None, 2, "ja");
8563 assert!(held.hold_reason.unwrap().contains("保留にしました"));
8564 let mut held = task();
8565 reclaim(&mut held, None, 2, "xx");
8566 assert!(held.hold_reason.unwrap().contains("held for a human"));
8567 }
8568
8569 fn run_state(status: RunStatus) -> RunState {
8570 let mut state = RunState::new(
8571 PathBuf::from("/repo"),
8572 "main".to_owned(),
8573 "abc1234def".to_owned(),
8574 "add retries".to_owned(),
8575 Config::default(),
8576 );
8577 state.status = status;
8578 state
8579 }
8580
8581 fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
8582 Candidate {
8583 index: 0,
8584 label,
8585 agent: "claude".to_owned(),
8586 branch: format!("magi/x/{label}"),
8587 worktree: PathBuf::from("/repo"),
8588 summary: summary.to_owned(),
8589 stat: String::new(),
8590 files: 0,
8591 commits: usize::from(!empty),
8592 empty,
8593 failed: failed.map(str::to_owned),
8594 verified_noop: None,
8595 duration_ms: 0,
8596 folded: false,
8597 }
8598 }
8599
8600 #[test]
8601 fn diagnostic_names_the_failing_gate_checks_and_their_output() {
8602 let mut state = run_state(RunStatus::Blocked);
8603 state.gate = vec![
8604 CommandOutcome {
8605 command: "cargo make check".to_owned(),
8606 code: Some(0),
8607 output_tail: "ok".to_owned(),
8608 duration_ms: 0,
8609 resource_blocked: false,
8610 },
8611 CommandOutcome {
8612 command: "cargo test".to_owned(),
8613 code: Some(101),
8614 output_tail: "thread 'x' panicked: assertion failed".to_owned(),
8615 duration_ms: 0,
8616 resource_blocked: false,
8617 },
8618 ];
8619 let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
8620 assert!(d.contains("cargo test"), "{d}");
8621 assert!(
8622 !d.contains("cargo make check"),
8623 "a passing check is not a diagnostic: {d}"
8624 );
8625 assert!(d.contains("assertion failed"), "{d}");
8626 }
8627
8628 #[test]
8629 fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
8630 let mut state = run_state(RunStatus::Blocked);
8631 state.event(
8632 "land",
8633 "stopped: the fixer produced no commit while 2 check(s) were failing \
8634 (build, lint); stopping instead of looping on an unchanged tree",
8635 );
8636 let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
8637 assert!(d.contains("build"), "{d}");
8638 assert!(d.contains("lint"), "{d}");
8639 assert!(d.contains("fixer produced no commit"), "{d}");
8640 }
8641
8642 #[test]
8643 fn describe_never_leaves_a_verified_noop_reading_as_a_bare_status_code() {
8644 let state = run_state(RunStatus::VerifiedNoop);
8649 let d = describe(&state);
8650 assert!(
8651 d.contains("agent-verified no-op"),
8652 "expected the display label, not the wire spelling: {d}"
8653 );
8654 assert!(!d.contains("verified_noop"), "{d}");
8655 }
8656
8657 #[test]
8658 fn diagnostic_keeps_gate_panic_before_large_backtrace() {
8659 let mut state = run_state(RunStatus::Failed);
8660 state.gate.push(crate::run::CommandOutcome {
8661 command: "cargo test".into(),
8662 code: Some(101),
8663 output_tail: format!(
8664 "---- widget stdout ----\nwidget log\nthread 'widget' panicked at widget.rs:12:\nassertion failed: widget missing\n{}test result: FAILED\n",
8665 " 41: 0x1234 - backtrace frame\n".repeat(10000)
8666 ),
8667 duration_ms: 1,
8668 resource_blocked: false,
8669 });
8670 let d = diagnostic(&state).unwrap();
8671 assert!(d.contains("panicked at widget.rs:12"), "{d}");
8672 assert!(d.contains("assertion failed: widget missing"));
8673 assert!(d.contains("widget log"));
8674 }
8675
8676 #[test]
8677 fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
8678 let mut state = run_state(RunStatus::Failed);
8684 state.candidates = vec![candidate(
8685 'A',
8686 "opened pull request #42, merged it, tagged v1.2.3 and published the release",
8687 true,
8688 None,
8689 )];
8690 let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
8691 assert!(d.contains("candidate A"), "{d}");
8692 assert!(d.contains("tagged v1.2.3"), "{d}");
8693 }
8694
8695 #[test]
8696 fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
8697 let mut state = run_state(RunStatus::Failed);
8698 state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
8699 let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
8700 assert!(d.contains("candidate A"), "{d}");
8701 assert!(d.contains("agent timed out"), "{d}");
8702 }
8703
8704 #[test]
8705 fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
8706 let mut state = run_state(RunStatus::Failed);
8709 state.candidates = vec![candidate('A', "did the work", false, None)];
8710 assert!(diagnostic(&state).is_none());
8711 }
8712
8713 #[test]
8714 fn diagnostic_is_bounded_however_much_a_run_printed() {
8715 let mut state = run_state(RunStatus::Blocked);
8716 state.gate = vec![
8717 CommandOutcome {
8718 command: "cargo test".to_owned(),
8719 code: Some(101),
8720 output_tail: "x".repeat(50_000),
8721 duration_ms: 0,
8722 resource_blocked: false,
8723 },
8724 CommandOutcome {
8725 command: "cargo clippy".to_owned(),
8726 code: Some(1),
8727 output_tail: "y".repeat(50_000),
8728 duration_ms: 0,
8729 resource_blocked: false,
8730 },
8731 ];
8732 state.candidates = vec![
8733 candidate('A', &"z".repeat(50_000), true, None),
8734 candidate('B', &"w".repeat(50_000), true, None),
8735 ];
8736 let d = diagnostic(&state).expect("plenty here to diagnose");
8737 assert!(
8738 d.len() <= DIAGNOSTIC_MAX,
8739 "diagnostic grew to {} bytes, unbounded",
8740 d.len()
8741 );
8742 }
8743
8744 #[test]
8745 fn selected_delivery_guards_survive_recovery_failure_accounting() {
8746 for status in [RunStatus::Ready, RunStatus::Blocked, RunStatus::Failed] {
8747 let mut t = task();
8748 let mut state = run_state(status);
8749 t.pr_recovery = Some(crate::pr_recovery::Selection {
8750 url: "url".into(),
8751 branch: format!("magi/task-{}", t.id),
8752 head: "a".repeat(40),
8753 source_head: None,
8754 source_run: None,
8755 source_proof: None,
8756 disposition: Some(crate::pr_recovery::Disposition::Land),
8757 instruction: t.instruction.clone(),
8758 approved_at: Timestamp::now(),
8759 });
8760 state.pr_recovery = t.pr_recovery.clone();
8761 t.start(state.id.clone());
8762 settle_and_diagnose(
8763 &mut t,
8764 Verdict {
8765 status,
8766 left_pr: false,
8767 quota_hit: false,
8768 parked: false,
8769 no_viable_candidates: false,
8770 },
8771 "delivery stopped",
8772 5,
8773 &state,
8774 );
8775 assert_eq!(t.status, TaskStatus::Held);
8776 let reason = t.hold_reason.as_deref().unwrap();
8777 if status == RunStatus::Ready {
8778 assert!(reason.contains("Selected PR delivery is incomplete"));
8779 assert!(t.recovery_budget.is_none());
8780 } else {
8781 assert!(reason.contains("Selected recovery stopped"));
8782 assert_eq!(t.recovery_budget.as_ref().unwrap().failures, 1);
8783 }
8784 assert!(t.pr_recovery.is_some());
8785 }
8786 }
8787
8788 #[test]
8789 fn base_recovery_settlement_holds_without_replacement_or_duplicate_notice_cause() {
8790 let mut state = run_state(RunStatus::Voting);
8791 state.base_sync = Some(crate::run::BaseSync {
8792 tip: "base-tip".into(),
8793 behind: 3,
8794 attempts: 4,
8795 conflict: Some("base stopped".into()),
8796 already_in: None,
8797 });
8798 state.record_base_sync_stop("candidate-head");
8799 let mut t = task();
8800 t.start(state.id.clone());
8801 let verdict = Verdict {
8802 status: RunStatus::Blocked,
8803 left_pr: false,
8804 quota_hit: false,
8805 parked: false,
8806 no_viable_candidates: false,
8807 };
8808 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
8809 assert_eq!(
8810 t.status,
8811 TaskStatus::Held,
8812 "unspent task attempts cannot start another competition"
8813 );
8814 let at = t.held_at;
8815 let cause = t.hold_notice_cause.clone();
8816 state.begin_pass();
8817 t.release_operator("test-resume", None);
8818 t.start(state.id.clone());
8819 settle_and_diagnose(&mut t, verdict, "base stopped", 10, &state);
8820 assert_eq!(t.status, TaskStatus::Held);
8821 assert_eq!(t.held_at, at);
8822 assert_eq!(t.hold_notice_cause, cause);
8823 }
8824
8825 #[test]
8826 fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
8827 let mut state = run_state(RunStatus::Blocked);
8828 state.gate = vec![CommandOutcome {
8829 command: "cargo test".to_owned(),
8830 code: Some(101),
8831 output_tail: "assertion failed".to_owned(),
8832 duration_ms: 0,
8833 resource_blocked: false,
8834 }];
8835 let verdict = Verdict {
8836 status: RunStatus::Blocked,
8837 left_pr: false,
8838 quota_hit: false,
8839 parked: false,
8840 no_viable_candidates: false,
8841 };
8842
8843 let mut t = task();
8846 t.start("run-1".to_owned());
8847 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8848 assert_eq!(t.status, TaskStatus::Failed);
8849 assert!(t.diagnostic.is_none());
8850
8851 t.start("run-2".to_owned());
8854 settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
8855 assert_eq!(t.status, TaskStatus::Held);
8856 let d = t.diagnostic.expect("a held task must carry its diagnostic");
8857 assert!(d.contains("cargo test"), "{d}");
8858 }
8859
8860 #[test]
8861 fn a_held_task_names_the_open_question_it_is_waiting_on() {
8862 crate::run::pin_test_home();
8867 let home = crate::run::home();
8868 let state = run_state(RunStatus::VerifiedNoop);
8869 let mut q = ask::Question::new(
8870 state.id.clone(),
8871 "implement".to_owned(),
8872 "impl-A".to_owned(),
8873 "is this really a no-op?".to_owned(),
8874 String::new(),
8875 Vec::new(),
8876 );
8877 Questions::at(home.join("questions")).put(&mut q).unwrap();
8878
8879 let verdict = Verdict {
8880 status: RunStatus::VerifiedNoop,
8881 left_pr: false,
8882 quota_hit: false,
8883 parked: false,
8884 no_viable_candidates: false,
8885 };
8886 let mut t = task();
8887 t.start(state.id.clone());
8888 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8889
8890 assert_eq!(t.status, TaskStatus::Held);
8891 let reason = t.hold_reason.expect("a held task must record why");
8892 assert!(
8893 reason.starts_with("run ended agent-verified no-op"),
8894 "the original settle reason must survive unchanged: {reason}"
8895 );
8896 assert!(
8897 reason.contains(q.short()),
8898 "the open question's id must be named so the notice is actionable: {reason}"
8899 );
8900 }
8901
8902 #[test]
8903 fn a_held_task_with_no_open_question_keeps_its_plain_reason() {
8904 crate::run::pin_test_home();
8905 let state = run_state(RunStatus::VerifiedNoop);
8906
8907 let verdict = Verdict {
8908 status: RunStatus::VerifiedNoop,
8909 left_pr: false,
8910 quota_hit: false,
8911 parked: false,
8912 no_viable_candidates: false,
8913 };
8914 let mut t = task();
8915 t.start(state.id.clone());
8916 settle_and_diagnose(&mut t, verdict, "run ended agent-verified no-op", 5, &state);
8917
8918 assert_eq!(t.status, TaskStatus::Held);
8919 assert_eq!(
8920 t.hold_reason.as_deref(),
8921 Some("run ended agent-verified no-op"),
8922 "nothing to append when the question was already answered or never asked"
8923 );
8924 }
8925
8926 #[test]
8927 fn supersede_prior_runs_rewrites_an_earlier_blocked_attempt_once_a_later_one_lands() {
8928 crate::run::pin_test_home();
8929 let mut first = run_state(RunStatus::Blocked);
8930 first.id = "20260101-000000-sup1".to_owned();
8931 first.save().unwrap();
8932 let mut second = run_state(RunStatus::Merged);
8933 second.id = "20260101-000000-sup2".to_owned();
8934 second.save().unwrap();
8935
8936 let mut t = task();
8937 t.runs = vec![first.id.clone(), second.id.clone()];
8938 t.status = TaskStatus::Done;
8939
8940 supersede_prior_runs(&t, &crate::run::home());
8941
8942 assert_eq!(
8943 RunState::load(&first.id).unwrap().status,
8944 RunStatus::Superseded,
8945 "the first attempt's Blocked no longer needs anyone's attention"
8946 );
8947 assert_eq!(
8948 RunState::load(&second.id).unwrap().status,
8949 RunStatus::Merged,
8950 "the run that actually succeeded is left exactly as it was"
8951 );
8952 }
8953
8954 #[test]
8955 fn supersede_prior_runs_leaves_a_manually_resumed_attempt_alone() {
8956 crate::run::pin_test_home();
8962 let mut first = run_state(RunStatus::Blocked);
8963 first.id = "20260101-000000-sup9".to_owned();
8964 first.driver_pid = Some(std::process::id());
8967 first.driver_started_at = Some(
8968 crate::proc::process_started_at(std::process::id())
8969 .expect("this test process's own start time must be queryable"),
8970 );
8971 first.save().unwrap();
8972 let mut second = run_state(RunStatus::Merged);
8973 second.id = "20260101-000000-supa".to_owned();
8974 second.save().unwrap();
8975
8976 let mut t = task();
8977 t.runs = vec![first.id.clone(), second.id.clone()];
8978 t.status = TaskStatus::Done;
8979
8980 supersede_prior_runs(&t, &crate::run::home());
8981
8982 assert_eq!(
8983 RunState::load(&first.id).unwrap().status,
8984 RunStatus::Blocked,
8985 "a live driver_pid means something is still actually working this run, \
8986 even though no daemon claims it - rewriting under it would just be \
8987 undone the next time that process saves"
8988 );
8989 }
8990
8991 #[test]
8992 fn resweep_catches_up_a_run_left_live_once_its_manual_process_is_no_longer_driving_it() {
8993 let dir = tempfile::tempdir().unwrap();
8999 let home = dir.path().to_path_buf();
9000 let queue = Queue::at(dir.path().join("queue"));
9001
9002 let mut first = run_state(RunStatus::Blocked);
9003 first.id = "20260101-000000-supd".to_owned();
9004 first.driver_pid = Some(std::process::id());
9005 first.driver_started_at = Some(
9006 crate::proc::process_started_at(std::process::id())
9007 .expect("this test process's own start time must be queryable"),
9008 );
9009 first.save_under(&home).unwrap();
9010 let mut second = run_state(RunStatus::Merged);
9011 second.id = "20260101-000000-supe".to_owned();
9012 second.save_under(&home).unwrap();
9013
9014 let mut t = task();
9015 t.runs = vec![first.id.clone(), second.id.clone()];
9016 t.status = TaskStatus::Done;
9017 queue.put(&mut t).unwrap();
9018
9019 resweep_superseded_attempts(&queue, &home);
9020 assert_eq!(
9021 RunState::load_under(&first.id, &home).unwrap().status,
9022 RunStatus::Blocked,
9023 "still live on the first pass, so still untouched"
9024 );
9025
9026 let mut stale = RunState::load_under(&first.id, &home).unwrap();
9032 stale.driver_started_at = Some("1".to_owned());
9033 stale.save_under(&home).unwrap();
9034
9035 resweep_superseded_attempts(&queue, &home);
9036 assert_eq!(
9037 RunState::load_under(&first.id, &home).unwrap().status,
9038 RunStatus::Superseded,
9039 "the second pass catches up what the first one correctly skipped"
9040 );
9041 }
9042
9043 #[test]
9044 fn supersede_prior_runs_leaves_concurrent_blocked_attempts_alone_while_the_task_is_not_done() {
9045 crate::run::pin_test_home();
9046 let mut first = run_state(RunStatus::Blocked);
9047 first.id = "20260101-000000-sup3".to_owned();
9048 first.save().unwrap();
9049 let mut second = run_state(RunStatus::Blocked);
9050 second.id = "20260101-000000-sup4".to_owned();
9051 second.save().unwrap();
9052
9053 let mut t = task();
9054 t.runs = vec![first.id.clone(), second.id.clone()];
9055 t.status = TaskStatus::Failed;
9059
9060 supersede_prior_runs(&t, &crate::run::home());
9061
9062 assert_eq!(
9063 RunState::load(&first.id).unwrap().status,
9064 RunStatus::Blocked
9065 );
9066 assert_eq!(
9067 RunState::load(&second.id).unwrap().status,
9068 RunStatus::Blocked
9069 );
9070 }
9071
9072 #[test]
9073 fn supersede_prior_runs_does_nothing_when_the_task_was_closed_by_hand() {
9074 crate::run::pin_test_home();
9078 let mut first = run_state(RunStatus::Blocked);
9079 first.id = "20260101-000000-sup5".to_owned();
9080 first.save().unwrap();
9081
9082 let mut t = task();
9083 t.runs = vec![first.id.clone()];
9084 t.status = TaskStatus::Done;
9085
9086 supersede_prior_runs(&t, &crate::run::home());
9087
9088 assert_eq!(
9089 RunState::load(&first.id).unwrap().status,
9090 RunStatus::Blocked,
9091 "a single-attempt task has no earlier run to supersede"
9092 );
9093 }
9094
9095 #[test]
9096 fn supersede_prior_runs_does_nothing_when_the_last_recorded_attempt_never_landed() {
9097 crate::run::pin_test_home();
9104 let mut first = run_state(RunStatus::Blocked);
9105 first.id = "20260101-000000-supb".to_owned();
9106 first.save().unwrap();
9107 let mut second = run_state(RunStatus::Failed);
9108 second.id = "20260101-000000-supc".to_owned();
9109 second.save().unwrap();
9110
9111 let mut t = task();
9112 t.runs = vec![first.id.clone(), second.id.clone()];
9113 t.status = TaskStatus::Done;
9114
9115 supersede_prior_runs(&t, &crate::run::home());
9116
9117 assert_eq!(
9118 RunState::load(&first.id).unwrap().status,
9119 RunStatus::Blocked,
9120 "the task's last attempt never landed, so there is nothing here \
9121 actually superseding it"
9122 );
9123 }
9124
9125 #[test]
9126 fn supersede_prior_runs_leaves_a_failed_or_verified_noop_attempt_as_is() {
9127 crate::run::pin_test_home();
9131 let mut failed = run_state(RunStatus::Failed);
9132 failed.id = "20260101-000000-sup6".to_owned();
9133 failed.save().unwrap();
9134 let mut noop = run_state(RunStatus::VerifiedNoop);
9135 noop.id = "20260101-000000-sup7".to_owned();
9136 noop.save().unwrap();
9137 let mut winner = run_state(RunStatus::Ready);
9138 winner.id = "20260101-000000-sup8".to_owned();
9139 winner.save().unwrap();
9140
9141 let mut t = task();
9142 t.runs = vec![failed.id.clone(), noop.id.clone(), winner.id.clone()];
9143 t.status = TaskStatus::Done;
9144
9145 supersede_prior_runs(&t, &crate::run::home());
9146
9147 assert_eq!(
9148 RunState::load(&failed.id).unwrap().status,
9149 RunStatus::Failed
9150 );
9151 assert_eq!(
9152 RunState::load(&noop.id).unwrap().status,
9153 RunStatus::VerifiedNoop
9154 );
9155 }
9156
9157 fn approval_question(run: &str) -> ask::Question {
9158 ask::Question::new(
9159 run.to_owned(),
9160 land::APPROVAL_NODE.to_owned(),
9161 "land".to_owned(),
9162 "merge?".to_owned(),
9163 String::new(),
9164 vec!["merge".to_owned(), "hold".to_owned()],
9165 )
9166 }
9167
9168 #[test]
9169 fn land_resume_state_leaves_a_fresh_open_question_waiting() {
9170 crate::run::pin_test_home();
9171 let mut state = run_state(RunStatus::Landing);
9172 state.id = "20260101-000000-fre1".to_owned();
9173 state.parked = true;
9174 state.save().unwrap();
9175 ask::Questions::open()
9176 .put(&mut approval_question(&state.id))
9177 .unwrap();
9178
9179 let mut t = task();
9180 t.runs.push(state.id.clone());
9181 assert_eq!(
9182 land_resume_state(&t),
9183 LandResume::StillWaiting,
9184 "nobody has answered and the timeout has not passed"
9185 );
9186 }
9187
9188 #[test]
9189 fn land_resume_state_waits_on_a_withheld_text_question_then_resumes() {
9190 crate::run::pin_test_home();
9191 let mut state = run_state(RunStatus::Gating);
9192 state.id = "20260101-000000-gt01".to_owned();
9193 state.parked = true;
9194 state.config.graph.answer_timeout = 60;
9195 state.github_text = Some(crate::run::GithubTextAsk {
9196 rewrite: None,
9197 fingerprint: "f".into(),
9198 title: true,
9199 body: false,
9200 categories: vec!["title-language".into()],
9201 question: None,
9202 asks: 1,
9203 resolved: false,
9204 chosen_title: None,
9205 choice: None,
9206 });
9207 state.save().unwrap();
9208 let store = ask::Questions::open();
9209 let mut q = ask::Question::new(
9210 state.id.clone(),
9211 crate::github_text::ASK_NODE.to_owned(),
9212 crate::github_text::ASK_SEAT.to_owned(),
9213 "s".into(),
9214 String::new(),
9215 vec!["use fallback".into()],
9216 );
9217 store.put(&mut q).unwrap();
9218 let mut t = task();
9219 t.runs.push(state.id.clone());
9220 assert_eq!(land_resume_state(&t), LandResume::StillWaiting);
9221 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9222 store.put(&mut q).unwrap();
9223 assert_eq!(land_resume_state(&t), LandResume::Ready);
9224 assert!(!store.get(&q.id).unwrap().status.open());
9225 state.github_text.as_mut().unwrap().resolved = true;
9227 state.save().unwrap();
9228 assert_eq!(land_resume_state(&t), LandResume::NotLanding);
9229 }
9230
9231 #[test]
9232 fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
9233 crate::run::pin_test_home();
9238 let mut state = run_state(RunStatus::Landing);
9239 state.id = "20260101-000000-exp1".to_owned();
9240 state.parked = true;
9241 state.config.graph.answer_timeout = 60;
9242 state.save().unwrap();
9243
9244 let store = ask::Questions::open();
9245 let mut q = approval_question(&state.id);
9246 q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
9247 store.put(&mut q).unwrap();
9248
9249 let mut t = task();
9250 t.runs.push(state.id.clone());
9251 assert_eq!(
9252 land_resume_state(&t),
9253 LandResume::Ready,
9254 "an expired question must not be waited on forever"
9255 );
9256
9257 let after = store.get(&q.id).unwrap();
9258 assert!(
9259 !after.status.open(),
9260 "the question is abandoned, not silently ignored"
9261 );
9262 assert!(
9263 after.resolution().is_none(),
9264 "an abandoned question is not read as a decision"
9265 );
9266 }
9267
9268 #[test]
9269 fn reclaim_settles_a_running_task_against_its_last_run() {
9270 let mut t = task();
9271 t.start("20260904-000000-4043".to_owned());
9272 reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2, "en");
9273 assert_eq!(
9274 t.status,
9275 TaskStatus::Done,
9276 "a run that actually finished must not stay `running` forever"
9277 );
9278 }
9279
9280 #[test]
9281 fn reclaim_maps_a_parked_run_to_parked_without_touching_last_error() {
9282 let mut t = task();
9283 t.start("20260904-000000-4043".to_owned());
9284 t.last_error = Some("earlier trouble".to_owned());
9285 let mut state = run_state(RunStatus::Judging);
9286 state.parked = true;
9287 reclaim(&mut t, Some(state), 1, "en");
9288 assert_eq!(t.status, TaskStatus::Parked);
9289 assert_eq!(t.attempts, 0, "the park is refunded, even at max_attempts");
9290 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9291 assert!(t.park_reason.is_some());
9292 assert_eq!(t.runs, ["20260904-000000-4043"], "the same run is kept");
9293 assert!(!t.fresh_start);
9294 assert!(t.status.runnable());
9295 }
9296
9297 #[test]
9298 fn settle_parks_a_task_without_a_failure_and_a_new_start_clears_the_reason() {
9299 let mut t = task();
9300 t.start("20260904-000000-4043".to_owned());
9301 t.last_error = Some("earlier trouble".to_owned());
9302 settle(
9303 &mut t,
9304 Verdict {
9305 status: RunStatus::Judging,
9306 left_pr: false,
9307 quota_hit: false,
9308 parked: true,
9309 no_viable_candidates: false,
9310 },
9311 "parked after `judging`",
9312 1,
9313 );
9314 assert_eq!(t.status, TaskStatus::Parked);
9315 assert_eq!(t.attempts, 0);
9316 assert_eq!(t.last_error.as_deref(), Some("earlier trouble"));
9317 assert_eq!(t.park_reason.as_deref(), Some("parked after `judging`"));
9318 assert_eq!(t.schema, crate::queue::SCHEMA);
9319 assert!(
9320 crate::notices::task_held(&t).is_none(),
9321 "a park is not news"
9322 );
9323 t.start("20260904-000000-4043".to_owned());
9324 assert_eq!(t.park_reason, None);
9325 }
9326
9327 #[test]
9328 fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
9329 let mut t = task();
9333 t.start("20260904-000000-4043".to_owned());
9334 reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2, "en");
9335 assert_eq!(t.status, TaskStatus::Failed);
9336 assert!(t.status.runnable());
9337 }
9338
9339 #[test]
9340 fn a_settled_parked_pass_records_its_cause_and_the_refund() {
9341 let mut t = task();
9342 t.start("20260904-000000-4043".to_owned());
9343 assert_eq!(t.attempts, 1);
9344 let mut run = run_state(RunStatus::Implementing);
9345 run.id = "20260904-000000-4043".to_owned();
9346 run.begin_pass();
9347 let cause = crate::run::ParkCause::Upgrade {
9348 from: "0.121.0".to_owned(),
9349 to: Some("0.121.1".to_owned()),
9350 };
9351 run.end_pass_parked(cause.clone());
9352 run.parked = true;
9353 let verdict = Verdict {
9354 status: run.status,
9355 left_pr: false,
9356 quota_hit: false,
9357 parked: true,
9358 no_viable_candidates: false,
9359 };
9360 let mut orphan = t.clone();
9363 reclaim(&mut orphan, Some(run.clone()), 2, "en");
9364 assert_eq!(orphan.status, TaskStatus::Parked);
9365 assert_eq!(orphan.attempts, 0);
9366 assert_eq!(
9367 orphan.upgrade_resume_key(),
9368 Some((run.passes[0].started_at, orphan.id.as_str()))
9369 );
9370 settle_and_diagnose(&mut t, verdict, "parked", 2, &run);
9371 assert_eq!(t.attempts, 0);
9372 assert_eq!(t.passes.len(), 1);
9373 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9374 assert_eq!(t.passes[0].parked, Some(cause));
9375 assert_eq!(t.passes[0].pass_index, 0);
9376 assert_eq!(t.passes[0].started_at, Some(run.passes[0].started_at));
9377 assert!(t.upgrade_resume_key().is_some());
9378
9379 t.start("20260904-000000-4043".to_owned());
9382 run.begin_pass();
9383 run.status = RunStatus::Failed;
9384 let verdict = Verdict {
9385 status: RunStatus::Failed,
9386 left_pr: false,
9387 quota_hit: false,
9388 parked: false,
9389 no_viable_candidates: false,
9390 };
9391 settle_and_diagnose(&mut t, verdict, "failed", 2, &run);
9392 assert_eq!(t.passes.len(), 2);
9393 assert_eq!(t.passes[0].cost, crate::queue::PassCost::Refunded);
9394 assert_eq!(t.passes[1].cost, crate::queue::PassCost::Spent);
9395 assert_eq!(t.passes[1].pass_index, 1);
9396 assert_eq!(t.attempts, 1);
9397 }
9398
9399 #[test]
9400 fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
9401 let mut t = task();
9402 t.start("20260904-000000-4043".to_owned());
9403 reclaim(&mut t, None, 2, "en");
9404 assert_eq!(t.status, TaskStatus::Held);
9405 assert!(
9406 t.last_error
9407 .as_deref()
9408 .is_some_and(|e| e.contains("running")),
9409 "the operator needs to know why this task was held"
9410 );
9411 }
9412
9413 #[test]
9414 fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
9415 let dir = tempfile::tempdir().unwrap();
9416 let queue = Queue::at(dir.path().to_path_buf());
9417
9418 let mut orphaned = task();
9420 orphaned.id = "20260904-000000-orph".to_owned();
9421 orphaned.status = TaskStatus::Running;
9422 orphaned.attempts = 1;
9423 queue.put(&mut orphaned).unwrap();
9424
9425 let mut alive = task();
9426 alive.id = "20260904-000000-live".to_owned();
9427 alive.status = TaskStatus::Running;
9428 alive.attempts = 1;
9429 queue.put(&mut alive).unwrap();
9430 let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
9431
9432 let mut queued = task();
9433 queued.id = "20260904-000000-wait".to_owned();
9434 queue.put(&mut queued).unwrap();
9435
9436 let reclaimed = reclaim_orphaned_running(&queue, 2, &Default::default());
9437 assert_eq!(reclaimed, vec![orphaned.id.clone()]);
9438
9439 assert_eq!(
9440 queue.get(&orphaned.id).unwrap().status,
9441 TaskStatus::Held,
9442 "nothing was driving it and there was no run to recover"
9443 );
9444 assert_eq!(
9445 queue.get(&alive.id).unwrap().status,
9446 TaskStatus::Running,
9447 "a live claim must protect the task it belongs to"
9448 );
9449 assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
9450 }
9451
9452 fn read_run_under(home: &Path, id: &str) -> RunState {
9458 let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
9459 serde_json::from_str(&body).unwrap()
9460 }
9461
9462 #[test]
9463 fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
9464 let dir = tempfile::tempdir().unwrap();
9465 let home = dir.path().to_path_buf();
9466 let now = Timestamp::now();
9467 let overrun_seat = || crate::run::ActiveSeat {
9468 node: "implement".to_owned(),
9469 started_at: now - jiff::SignedDuration::new(21_000, 0),
9470 timeout_secs: 3_600,
9471 attempt: 0,
9472 task: None,
9473 command: None,
9474 index: None,
9475 total: None,
9476 };
9477
9478 let mut dead = run_state(RunStatus::Implementing);
9479 dead.id = "20260101-000000-dead".to_owned();
9480 dead.active.insert("impl-A".to_owned(), overrun_seat());
9481 dead.driver_pid = Some(4242);
9484 dead.save_under(&home).unwrap();
9485
9486 let mut alive = run_state(RunStatus::Implementing);
9489 alive.id = "20260101-000000-aliv".to_owned();
9490 alive.active.insert("impl-A".to_owned(), overrun_seat());
9491 alive.save_under(&home).unwrap();
9492 let mut status = Status::new();
9493 status.current = vec![Current {
9494 worker: None,
9495 task: "20260101-000000-task".to_owned(),
9496 run: alive.id.clone(),
9497 }];
9498 write_status_to(&home.join("daemon.json"), &status).unwrap();
9499
9500 let questions = Questions::at(home.join("questions"));
9504 let mut q = ask::Question::new(
9505 dead.id.clone(),
9506 "implement".to_owned(),
9507 "impl-A".to_owned(),
9508 "Which storage backend?".to_owned(),
9509 String::new(),
9510 vec!["SQLite".to_owned(), "Redis".to_owned()],
9511 );
9512 questions.put(&mut q).unwrap();
9513
9514 let abandoned = reclaim_abandoned_runs_with(
9515 &home,
9516 now,
9517 |pid| if pid == 4242 { Some(false) } else { None },
9518 |_| panic!("a query answering Dead outright needs no identity corroboration"),
9519 );
9520 assert_eq!(abandoned, vec![dead.id.clone()]);
9521
9522 let reloaded = read_run_under(&home, &dead.id);
9523 assert_eq!(reloaded.status, RunStatus::Failed);
9524 assert!(reloaded.active.is_empty());
9525 assert!(
9526 !questions.get(&q.id).unwrap().status.open(),
9527 "the failed run's own open question must be settled in the same pass"
9528 );
9529
9530 let still_alive = read_run_under(&home, &alive.id);
9531 assert_eq!(
9532 still_alive.status,
9533 RunStatus::Implementing,
9534 "a live daemon's claim protects it"
9535 );
9536 assert!(!still_alive.active.is_empty());
9537 }
9538
9539 #[test]
9549 fn reclaim_abandoned_runs_leaves_a_live_manual_run_alone_even_though_no_daemon_claims_it() {
9550 let dir = tempfile::tempdir().unwrap();
9551 let home = dir.path().to_path_buf();
9552 let now = Timestamp::now();
9553
9554 let mut manual = run_state(RunStatus::Reviewing);
9555 manual.id = "20260101-000000-manl".to_owned();
9556 manual.active.insert(
9557 "review-1".to_owned(),
9558 crate::run::ActiveSeat {
9559 node: "review".to_owned(),
9560 started_at: now - jiff::SignedDuration::new(21_000, 0),
9561 timeout_secs: 3_600,
9562 attempt: 0,
9563 task: None,
9564 command: None,
9565 index: None,
9566 total: None,
9567 },
9568 );
9569 manual.driver_pid = Some(4242);
9573 manual.driver_started_at = Some("1790000000".to_owned());
9574 manual.save_under(&home).unwrap();
9575
9576 let abandoned = reclaim_abandoned_runs_with(
9577 &home,
9578 now,
9579 |pid| if pid == 4242 { Some(true) } else { None },
9580 |pid| {
9581 if pid == 4242 {
9582 Some("1790000000".to_owned())
9583 } else {
9584 None
9585 }
9586 },
9587 );
9588 assert!(
9589 abandoned.is_empty(),
9590 "a manual run a real process is still driving must never be reclaimed: {abandoned:?}"
9591 );
9592
9593 let reloaded = read_run_under(&home, &manual.id);
9594 assert_eq!(reloaded.status, RunStatus::Reviewing);
9595 assert!(!reloaded.active.is_empty());
9596 }
9597
9598 #[test]
9599 fn an_already_claimed_task_is_skipped_rather_than_failed() {
9600 let dir = tempfile::tempdir().unwrap();
9601 let queue = Queue::at(dir.path().to_path_buf());
9602 let mut only = task();
9603 queue.put(&mut only).unwrap();
9604
9605 let _elsewhere = queue.claim(&only.id).unwrap();
9606 let candidates = runnable(&queue);
9607 assert_eq!(candidates.len(), 1, "the task is still runnable");
9608 assert!(
9609 queue.claim(&candidates[0].id).is_err(),
9610 "the loop cannot take a claim somebody else holds"
9611 );
9612
9613 let after = queue.get(&only.id).unwrap();
9614 assert_eq!(after.status, TaskStatus::Queued);
9615 assert_eq!(
9616 after.attempts, 0,
9617 "losing the race is not an attempt at the task"
9618 );
9619 assert_eq!(after.last_error, None);
9620 }
9621
9622 #[test]
9623 fn the_status_file_round_trips_and_its_heartbeat_advances() {
9624 let dir = tempfile::tempdir().unwrap();
9625 let path = dir.path().join("daemon.json");
9626
9627 let mut status = Status::new();
9628 status.idle = false;
9629 status.completed = 7;
9630 status.current = vec![Current {
9631 worker: None,
9632 task: "20260902-000000-t111".to_owned(),
9633 run: "20260902-000001-r111".to_owned(),
9634 }];
9635 write_status_to(&path, &status).unwrap();
9636 let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9637 assert_eq!(first.schema, SCHEMA);
9638 assert_eq!(first.pid, std::process::id());
9639 assert!(!first.idle);
9640 assert_eq!(first.completed, 7);
9641 assert_eq!(first.current, status.current);
9642 assert!(
9643 !path.with_extension("json.tmp").exists(),
9644 "the temp file is renamed, not left behind"
9645 );
9646
9647 std::thread::sleep(Duration::from_millis(5));
9648 status.updated_at = Timestamp::now();
9649 status.polls = 3;
9650 write_status_to(&path, &status).unwrap();
9651 let second: Status =
9652 serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
9653 assert!(
9654 second.updated_at > first.updated_at,
9655 "a reader can only detect staleness if the heartbeat moves"
9656 );
9657 assert_eq!(
9658 second.started_at, first.started_at,
9659 "the start time is not a heartbeat"
9660 );
9661 assert_eq!(second.polls, 3);
9662 }
9663
9664 #[test]
9665 fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
9666 let dir = tempfile::tempdir().unwrap();
9667
9668 assert!(read_status(dir.path()).is_none(), "no file, no daemon");
9669
9670 let mut status = Status::new();
9671 status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
9672 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9673 let stale = read_status(dir.path()).unwrap();
9674 assert!(
9675 !stale.running(Timestamp::now()),
9676 "a minute without a heartbeat is a dead daemon, not a busy one"
9677 );
9678 assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
9679
9680 status.updated_at = Timestamp::now();
9681 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9682 let fresh = read_status(dir.path()).unwrap();
9683 assert!(fresh.running(Timestamp::now()));
9684 }
9685
9686 #[test]
9687 fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
9688 let dir = tempfile::tempdir().unwrap();
9689 let now = Timestamp::now();
9690 let mine = "20260903-080619-01c2";
9691
9692 assert!(
9693 !is_working_on(dir.path(), mine, now),
9694 "no status file means nobody is working on anything"
9695 );
9696
9697 let mut status = Status::new();
9698 status.current = vec![Current {
9699 worker: None,
9700 task: "20260903-080340-0167".to_owned(),
9701 run: mine.to_owned(),
9702 }];
9703 status.updated_at = now;
9704 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9705 assert!(is_working_on(dir.path(), mine, now));
9706 assert!(
9707 !is_working_on(dir.path(), "20260903-105039-3cbf", now),
9708 "a daemon busy with one run is not working on another"
9709 );
9710
9711 status.updated_at = now - jiff::SignedDuration::from_secs(600);
9714 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9715 assert!(
9716 !is_working_on(dir.path(), mine, now),
9717 "a stale heartbeat is a dead daemon, so its run is a leftover"
9718 );
9719 }
9720
9721 #[test]
9722 fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
9723 let dir = tempfile::tempdir().unwrap();
9724 let now = Timestamp::now();
9725
9726 assert!(
9727 !is_working_on_short(dir.path(), "01c2", now),
9728 "no status file means nobody is working on anything"
9729 );
9730
9731 let mut status = Status::new();
9732 status.current = vec![Current {
9733 worker: None,
9734 task: "20260903-080340-0167".to_owned(),
9735 run: "20260903-080619-01c2".to_owned(),
9736 }];
9737 status.updated_at = now;
9738 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
9739 assert!(
9740 is_working_on_short(dir.path(), "01c2", now),
9741 "the run's short id is the last block of its full id"
9742 );
9743 assert!(
9744 !is_working_on_short(dir.path(), "3cbf", now),
9745 "a daemon busy with one worktree bay is not working on another"
9746 );
9747 }
9748
9749 #[test]
9750 fn a_newer_status_file_still_yields_a_reading() {
9751 let dir = tempfile::tempdir().unwrap();
9752 std::fs::write(
9755 dir.path().join("daemon.json"),
9756 serde_json::json!({
9757 "schema": 2,
9758 "updated_at": Timestamp::now().to_string(),
9759 "idle": true,
9760 "surprise": { "nested": [1, 2, 3] },
9761 })
9762 .to_string(),
9763 )
9764 .unwrap();
9765
9766 let reading = read_status(dir.path()).expect("a forward-compatible read");
9767 assert!(reading.running(Timestamp::now()));
9768 assert!(reading.idle);
9769 assert!(reading.current.is_empty());
9770 }
9771
9772 #[test]
9773 fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
9774 let dir = tempfile::tempdir().unwrap();
9780 std::fs::write(
9781 dir.path().join("daemon.json"),
9782 serde_json::json!({
9783 "schema": 1,
9784 "pid": 4242,
9785 "updated_at": Timestamp::now().to_string(),
9786 "idle": false,
9787 "current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
9788 "completed": 3,
9789 "polls": 9,
9790 })
9791 .to_string(),
9792 )
9793 .unwrap();
9794
9795 let reading = read_status(dir.path()).expect("an older shape must still parse");
9796 assert!(reading.running(Timestamp::now()));
9797 assert_eq!(
9798 reading.current,
9799 vec![Current {
9800 worker: None,
9801 task: "20260902-140501-aaaa".to_owned(),
9802 run: "20260902-140502-bbbb".to_owned(),
9803 }]
9804 );
9805 }
9806
9807 #[test]
9808 fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
9809 let dir = tempfile::tempdir().unwrap();
9810 std::fs::write(
9811 dir.path().join("daemon.json"),
9812 serde_json::json!({
9813 "schema": 1,
9814 "updated_at": Timestamp::now().to_string(),
9815 "idle": true,
9816 "current": null,
9817 })
9818 .to_string(),
9819 )
9820 .unwrap();
9821 let with_null = read_status(dir.path()).expect("null must still parse");
9822 assert!(with_null.current.is_empty());
9823
9824 std::fs::write(
9825 dir.path().join("daemon.json"),
9826 serde_json::json!({
9827 "schema": 1,
9828 "updated_at": Timestamp::now().to_string(),
9829 "idle": true,
9830 })
9831 .to_string(),
9832 )
9833 .unwrap();
9834 let absent = read_status(dir.path()).expect("a missing field must still parse");
9835 assert!(absent.current.is_empty());
9836 }
9837
9838 #[test]
9839 fn a_task_without_a_repository_runs_in_the_daemons_default() {
9840 let fallback = Path::new("/default");
9841 let mut blank = task();
9842 blank.repo = PathBuf::new();
9843 assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
9844 let mut dot = task();
9845 dot.repo = PathBuf::from(".");
9846 assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
9847 assert_eq!(
9848 repo_for(&task(), fallback),
9849 PathBuf::from("/repo"),
9850 "a task that names a repository keeps it"
9851 );
9852 }
9853
9854 #[test]
9855 fn task_implementers_edit_inherits_daemon_config_and_merge() {
9856 let dir = tempfile::tempdir().unwrap();
9857 let path = dir.path().join("explicit.toml");
9858 std::fs::write(&path, "[graph]\nimplementers = 3\njudges = 1\n").unwrap();
9859 let opts = Opts {
9860 config: Some(path),
9861 merge: Some("none".into()),
9862 ..Default::default()
9863 };
9864 let mut t = task();
9865 t.set_implementers(
9866 2,
9867 &toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9868 .unwrap(),
9869 )
9870 .unwrap();
9871 let cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9872 assert_eq!(cfg.graph.implementers, 2);
9873 assert_eq!(cfg.graph.judges, 1);
9874 assert_eq!(cfg.merge.mode, MergeMode::None);
9875 t.set_implementers(1, &cfg).unwrap();
9876 let mut cfg = prepare_for(dir.path(), &opts, &t).unwrap();
9877 apply_task_implementers(&mut cfg, &t);
9878 assert_eq!(cfg.graph.implementers, 1);
9879 assert_eq!(cfg.graph.judges, 1);
9880 assert_eq!(cfg.merge.mode, MergeMode::None);
9881 }
9882
9883 #[test]
9884 fn task_implementers_direct_overrides_stay_isolated_after_edits() {
9885 let dir = tempfile::tempdir().unwrap();
9886 let repo = dir.path().join("repo");
9887 std::fs::create_dir(&repo).unwrap();
9888 std::fs::write(
9889 repo.join("magi.toml"),
9890 "[graph]\njudges = 2\n[merge]\nmode = 'none'\n",
9891 )
9892 .unwrap();
9893 let daemon_config = dir.path().join("daemon.toml");
9894 std::fs::write(
9895 &daemon_config,
9896 "[graph]\njudges = 1\n[merge]\nmode = 'pr'\n",
9897 )
9898 .unwrap();
9899 let opts = Opts {
9900 config: Some(daemon_config),
9901 merge: Some("pr".into()),
9902 ..Default::default()
9903 };
9904 let q = Queue::at(dir.path().join("queue"));
9905 let validation: Config =
9906 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9907 .unwrap();
9908 for record in ["{}", r#"{"candidates":2}"#] {
9910 let overrides = serde_json::from_str(record).unwrap();
9911 let filing = crate::direct::Filing {
9912 instruction: "work".into(),
9913 title: "direct".into(),
9914 repo: repo.clone(),
9915 source: crate::queue::Source::Human,
9916 solo: false,
9917 overrides,
9918 review_of: None,
9919 sources: Default::default(),
9920 };
9921 let filed =
9922 crate::direct::file(&q, dir.path(), Timestamp::now(), 1, filing, true).unwrap();
9923 let crate::direct::Filed::Standalone {
9924 mut task,
9925 claim: _claim,
9926 } = filed
9927 else {
9928 panic!("local filing")
9929 };
9930 let before = prepare_for(&repo, &opts, &task).unwrap();
9931 assert_eq!(before.graph.judges, 2, "daemon --config must not leak");
9932 assert_eq!(
9933 before.merge.mode,
9934 MergeMode::None,
9935 "daemon --merge must not leak"
9936 );
9937 for n in [2, 1, 2] {
9938 task.set_implementers(n, &validation).unwrap();
9939 q.put(&mut task).unwrap();
9940 task = q.get(&task.id).unwrap();
9941 assert!(!task.overrides.as_ref().unwrap().inherit_daemon_options);
9942 let mut cfg = prepare_for(&repo, &opts, &task).unwrap();
9943 apply_task_implementers(&mut cfg, &task);
9944 assert_eq!(cfg.graph.implementers, usize::from(n));
9945 assert_eq!(cfg.graph.judges, 2);
9946 assert_eq!(cfg.merge.mode, MergeMode::None);
9947 }
9948 }
9949 }
9950
9951 #[test]
9952 fn task_implementers_mismatch_keeps_explicit_reviews_review_only() {
9953 let cfg: Config =
9954 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9955 .unwrap();
9956 let mut t = task();
9957 t.review_of = Some("review-target".into());
9958 t.hold_manual(Some("retry".into()));
9959 let mut previous = RunState::new(
9960 PathBuf::from("/repo"),
9961 "main".into(),
9962 "abc1234".into(),
9963 "review".into(),
9964 cfg.clone(),
9965 );
9966 previous.config.graph.implementers = 2;
9967 for count in [1, 3] {
9968 t.set_implementers(count, &cfg).unwrap();
9969 t.release();
9970 assert_eq!(t.review_of.as_deref(), Some("review-target"));
9971 for proposed in [
9972 Starter::Resume(previous.id.clone()),
9973 Starter::Review("recovery-target".into()),
9974 ] {
9975 assert_eq!(
9976 starter_with_implementers(&t, Some(&previous), proposed),
9977 Starter::Review("review-target".into())
9978 );
9979 }
9980 let mut configured = cfg.clone();
9981 apply_task_implementers(&mut configured, &t);
9982 assert_eq!(configured.graph.implementers, usize::from(count));
9983 assert_eq!(previous.config.graph.implementers, 2);
9984 }
9985 }
9986
9987 #[test]
9988 fn task_implementers_dispatch_and_resume_agree() {
9989 let mut t = task();
9990 let cfg: crate::config::Config =
9991 toml::from_str("[[agents]]\nid = 'mock'\nkind = 'command'\ncommand = ['true']\n")
9992 .unwrap();
9993 let mut run = RunState::new(
9994 PathBuf::from("/repo"),
9995 "main".into(),
9996 "abc1234".into(),
9997 "work".into(),
9998 cfg.clone(),
9999 );
10000 run.config.graph.implementers = 2;
10001 for n in [2, 1, 3, 2] {
10002 t.set_implementers(n, &cfg).unwrap();
10003 let mut dispatched = cfg.clone();
10004 apply_task_implementers(&mut dispatched, &t);
10005 assert_eq!(dispatched.graph.implementers, usize::from(n));
10006 assert_eq!(implementers_match(&t, &run), n == 2);
10007 for proposed in [
10008 Starter::Resume(run.id.clone()),
10009 Starter::Review("old-branch".into()),
10010 ] {
10011 let expected = if n == 2 {
10012 proposed.clone()
10013 } else {
10014 Starter::Start
10015 };
10016 assert_eq!(
10017 starter_with_implementers(&t, Some(&run), proposed),
10018 expected
10019 );
10020 }
10021 assert_eq!(
10022 run.config.graph.implementers, 2,
10023 "history must not be rewritten"
10024 );
10025 }
10026 t.solo = false;
10027 t.overrides = None;
10028 assert!(implementers_match(&t, &run));
10029 }
10030
10031 #[test]
10032 fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
10033 let mut solo_cfg = Config::default();
10039 solo_cfg.graph.implementers = 3;
10040 let mut solo_task = task();
10041 solo_task.solo = true;
10042 apply_task_implementers(&mut solo_cfg, &solo_task);
10043 assert_eq!(solo_cfg.graph.implementers, 1);
10044
10045 let mut plain_cfg = Config::default();
10046 plain_cfg.graph.implementers = 3;
10047 let plain_task = task();
10048 assert!(!plain_task.solo);
10049 apply_task_implementers(&mut plain_cfg, &plain_task);
10050 assert_eq!(
10051 plain_cfg.graph.implementers, 3,
10052 "a task that did not ask to run alone keeps the config's candidates"
10053 );
10054 }
10055
10056 fn loss(seat: &str, at: &str, reset: Option<&str>) -> QuotaLoss {
10057 QuotaLoss {
10058 cause: crate::agent::QuotaCause::UsageLimit,
10059 agent: String::new(),
10060 seat: seat.into(),
10061 node: "judge".into(),
10062 at: at.parse().unwrap(),
10063 reset: reset.map(str::to_string),
10064 }
10065 }
10066
10067 #[test]
10068 fn authentication_losses_never_arm_a_cooldown_even_with_reset_hints() {
10069 let now = Timestamp::now();
10070 for reset in [None, Some("in 1h")] {
10071 let auth = crate::agent::Quota {
10072 cause: crate::agent::QuotaCause::Authentication,
10073 agent: "alpha".into(),
10074 reset: reset.map(str::to_owned),
10075 };
10076 let loss = QuotaLoss::from_report("advisor-1".into(), "advise", Some(&auth));
10077 assert_eq!(cooldown_until(&[loss], now), None);
10078 }
10079 }
10080
10081 #[test]
10082 fn a_resumed_run_with_only_old_quota_losses_arms_no_cooldown() {
10083 let old: Vec<QuotaLoss> = (1..=4)
10084 .map(|i| {
10085 loss(
10086 &format!("judge-{i}"),
10087 "2026-09-23T05:23:00Z",
10088 Some("2:40pm (Asia/Tokyo)"),
10089 )
10090 })
10091 .collect();
10092 let fresh = losses_this_attempt(&old, &old);
10093 assert!(fresh.is_empty());
10094 assert_eq!(cooldown_until(&fresh, Timestamp::now()), None);
10095 }
10097
10098 #[test]
10099 fn a_new_quota_loss_during_the_attempt_still_arms_the_cooldown() {
10100 let old = vec![loss("judge-1", "2026-09-23T05:23:00Z", None)];
10101 let now = Timestamp::now();
10102 let mut after = old.clone();
10103 after.push(loss("judge-2", &now.to_string(), None));
10104 let fresh = losses_this_attempt(&old, &after);
10105 assert_eq!(fresh, vec![after[1].clone()]);
10106 let until = cooldown_until(&fresh, now).expect("a fresh loss arms the cooldown");
10107 assert_eq!(
10108 until,
10109 now + jiff::SignedDuration::from_secs(QUOTA_WAIT_FALLBACK.as_secs() as i64)
10110 );
10111 }
10112
10113 #[test]
10114 fn a_recovered_seat_dropping_out_of_the_history_does_not_hide_a_new_loss() {
10115 let before = vec![
10118 loss("judge-1", "2026-09-23T05:23:00Z", None),
10119 loss("judge-2", "2026-09-23T05:24:00Z", None),
10120 ];
10121 let after = vec![
10122 loss("judge-2", "2026-09-23T05:24:00Z", None),
10123 loss("judge-1", "2026-09-24T01:00:00Z", None),
10124 ];
10125 assert_eq!(losses_this_attempt(&before, &after), vec![after[1].clone()]);
10126 }
10127
10128 #[test]
10129 fn merge_overrides_are_parsed_or_refused() {
10130 assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
10131 assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
10132 assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
10133 assert!(merge_mode("squash").is_err());
10134 }
10135
10136 #[test]
10137 fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
10138 let now = Timestamp::now();
10139 let fallback = Duration::from_secs(300);
10140 let cap = Duration::from_secs(1800);
10141
10142 assert_eq!(quota_wait(None, now, fallback, cap), fallback);
10144
10145 let soon = now + jiff::SignedDuration::from_secs(600);
10147 assert_eq!(
10148 quota_wait(Some(soon), now, fallback, cap),
10149 Duration::from_secs(600)
10150 );
10151
10152 let past = now - jiff::SignedDuration::from_secs(60);
10155 assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
10156
10157 let far = now + jiff::SignedDuration::from_secs(3 * 3600);
10160 assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
10161 }
10162
10163 #[test]
10164 fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
10165 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10166
10167 let at = parse_reset_hint("4:50am (UTC)", now, now).expect("a recognised shape parses");
10168 assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
10169
10170 let already_past =
10174 parse_reset_hint("1:00am (UTC)", now, now).expect("a recognised shape parses");
10175 assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
10176
10177 assert!(
10178 parse_reset_hint("session limit reached", now, now).is_none(),
10179 "free text with no recognised shape is not guessed at"
10180 );
10181 assert!(
10182 parse_reset_hint("4:50am (Nowhere/Fake)", now, now).is_none(),
10183 "an unresolvable zone name is not guessed at either"
10184 );
10185 }
10186
10187 #[test]
10188 fn parse_reset_hint_reads_the_codex_cli_shape_with_no_year_rollover_needed() {
10189 let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
10190
10191 let at = parse_reset_hint(
10192 "You've hit your usage limit. Visit \
10193 https://chatgpt.com/codex/settings/usage to purchase more \
10194 credits or try again at Sep 19th, 2026 5:10 PM.",
10195 now,
10196 now,
10197 )
10198 .expect("the codex reset wording is a recognised shape");
10199 assert_eq!(at.to_string(), "2026-09-19T17:10:00Z");
10200
10201 let earlier = parse_reset_hint("try again at Jan 2nd, 2026 1:00 AM.", now, now)
10206 .expect("an explicit year needs no rollover");
10207 assert_eq!(earlier.to_string(), "2026-01-02T01:00:00Z");
10208
10209 assert!(
10210 parse_reset_hint("try again at Sep 19th, 26 5:10 PM.", now, now).is_none(),
10211 "a two-digit year is not the documented shape and is not guessed at"
10212 );
10213 assert!(
10214 parse_reset_hint("try again at Sept 19th, 2026 5:10 PM.", now, now).is_none(),
10215 "a four-letter month name is not the documented three-letter abbreviation"
10216 );
10217 assert!(
10218 parse_reset_hint("try again at Sep 19th, 2026 5:10 PM (UTC).", now, now).is_none(),
10219 "an explicit zone on the dated shape is a format nobody has \
10220 documented, and is refused rather than guessed at as UTC"
10221 );
10222 }
10223
10224 #[test]
10225 fn parse_reset_hint_reads_agys_relative_shape_from_when_the_loss_was_recorded() {
10226 let now = "2026-09-24T12:00:00Z".parse::<Timestamp>().unwrap();
10227 let recorded = "2026-09-24T08:00:00Z".parse::<Timestamp>().unwrap();
10228
10229 let at = parse_reset_hint("in 1h2m49s", now, recorded).expect("agy's shape parses");
10230 assert_eq!(at.as_second() - recorded.as_second(), 3769);
10231
10232 let partial = parse_reset_hint("in 45m", now, recorded).expect("units are optional");
10233 assert_eq!(partial.as_second() - recorded.as_second(), 45 * 60);
10234
10235 for bad in ["in ", "in 45", "in 3x", "in m", "in 1h junk", "1h2m"] {
10236 assert!(
10237 parse_reset_hint(bad, now, recorded).is_none(),
10238 "{bad:?} must not be guessed at"
10239 );
10240 }
10241 }
10242
10243 fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
10247 let config = dir.join("magi.toml");
10248 std::fs::write(
10249 &config,
10250 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
10251 )
10252 .unwrap();
10253 let opts = Opts {
10254 poll: Duration::from_secs(30),
10255 config: Some(config),
10256 repo: dir.join("repo"),
10260 ..Opts::default()
10261 };
10262 let home = dir.join("home");
10271 let stamps = home.join("service-ticks");
10274 std::fs::create_dir_all(&stamps).unwrap();
10275 for service in ["waiter", "deputies", "release-watch"] {
10276 std::fs::write(
10277 stamps.join(service),
10278 Timestamp::now().as_second().to_string(),
10279 )
10280 .unwrap();
10281 }
10282 for operation in [
10283 "conductor",
10284 "maintenance",
10285 "housekeeping",
10286 "disk-pressure",
10287 "cache",
10288 "fetch",
10289 ] {
10290 std::fs::write(
10291 stamps.join(format!("daemon-{operation}")),
10292 serde_json::to_vec(&Timestamp::now()).unwrap(),
10293 )
10294 .unwrap();
10295 }
10296 let worktrees = dir.join("wt");
10297 (
10298 opts,
10299 Queue::at(dir.join("queue")),
10300 home.join("daemon.json"),
10301 home,
10302 worktrees,
10303 )
10304 }
10305
10306 #[test]
10307 fn a_stop_is_idempotent_and_once_set_stays_set() {
10308 let stop = Stop::new();
10309 assert!(!stop.stopped());
10310
10311 stop.stop();
10312 assert!(stop.stopped());
10313 stop.stop();
10314 assert!(stop.stopped(), "a second stop is not a toggle");
10315
10316 let shared = stop.clone();
10317 assert!(
10318 shared.stopped(),
10319 "a clone is the same stop; that is how the loop and its caller share one"
10320 );
10321 }
10322
10323 #[tokio::test]
10324 async fn once_drain_rechecks_answers_after_queue_services_release_their_claims() {
10325 for operation in ["conductor", "maintenance", "housekeeping"] {
10326 let dir = tempfile::tempdir().unwrap();
10327 let (mut opts, queue, _, home, worktrees) = idle_loop(dir.path());
10328 opts.once = true;
10329 opts.poll = Duration::from_millis(1);
10330 let mut task = task();
10331 task.runs.push("fixture-run".into());
10332 queue.put(&mut task).unwrap();
10333 let claim = queue.claim(&task.id).unwrap();
10334
10335 let directory = home.join("workers/jobs/abcdef");
10338 std::fs::create_dir_all(&directory).unwrap();
10339 let lease = std::fs::File::create(directory.join("live.lock")).unwrap();
10340 crate::pr_head::try_lock_file(&lease).unwrap();
10341 let mut job = crate::worker::Job {
10342 id: "abcdef".into(),
10343 sequence: 1,
10344 kind: "daemon-service".into(),
10345 key: format!("service:daemon-{operation}"),
10346 payload: serde_json::Value::Null,
10347 generation: "fixture".into(),
10348 version: None,
10349 status: crate::worker::Status::Running,
10350 result: None,
10351 pid: Some(std::process::id()),
10352 created_at: Timestamp::now().as_second() as u64,
10353 finished_at: None,
10354 };
10355 let record = directory.join("job.json");
10356 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10357
10358 let status = Arc::new(Mutex::new(Status::new()));
10359 let stop = Stop::new();
10360 let mut drain = Box::pin(poll(
10361 &opts,
10362 &queue,
10363 &status,
10364 &home,
10365 &worktrees,
10366 &stop,
10367 DispatchLimits {
10368 max_concurrent: 1,
10369 pause_for_interrupts: false,
10370 recovery_cfg: None,
10371 },
10372 ));
10373 std::future::poll_fn(|cx| {
10374 use std::future::Future;
10375 assert!(
10376 drain.as_mut().poll(cx).is_pending(),
10377 "--once must wait for {operation} to release its queue claim"
10378 );
10379 std::task::Poll::Ready(())
10380 })
10381 .await;
10382 assert!(lock(&status).polls > 0, "the dispatch scan was exercised");
10383
10384 let questions = Questions::at(home.join("questions"));
10387 let mut q = action_question("fixture-run", ask::ChoiceAction::Done);
10388 questions.put(&mut q).unwrap();
10389 drop(claim);
10390 let expired =
10394 serde_json::to_vec(&"2000-01-01T00:00:00Z".parse::<Timestamp>().unwrap()).unwrap();
10395 for service in [operation, "housekeeping"] {
10396 std::fs::write(
10397 home.join("service-ticks").join(format!("daemon-{service}")),
10398 &expired,
10399 )
10400 .unwrap();
10401 }
10402 job.status = crate::worker::Status::Finished;
10403 std::fs::write(&record, serde_json::to_vec(&job).unwrap()).unwrap();
10404 drop(lease);
10405 tokio::time::timeout(Duration::from_secs(2), drain)
10406 .await
10407 .expect("the settled service must allow the drain to finish")
10408 .unwrap();
10409 let after = queue.get(&task.id).unwrap();
10410 assert_eq!(after.status, TaskStatus::Done, "answer after {operation}");
10411 assert!(after.action_applied(&q.id));
10412 for service in [operation, "housekeeping"] {
10413 assert_eq!(
10414 std::fs::read(home.join("service-ticks").join(format!("daemon-{service}")))
10415 .unwrap(),
10416 expired,
10417 "the final once rescan must not restart {service}"
10418 );
10419 }
10420 }
10421 }
10422
10423 #[test]
10424 fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
10425 let stop = Stop::new();
10426 stop.busy.store(1, Ordering::SeqCst);
10427 assert!(
10428 !stop.finishing(),
10429 "a busy loop nobody has asked to stop is just running"
10430 );
10431
10432 stop.stop();
10433 assert!(
10434 stop.finishing(),
10435 "a stop asked for mid-run has not landed until the run is settled"
10436 );
10437
10438 stop.busy.store(0, Ordering::SeqCst);
10439 assert!(
10440 !stop.finishing(),
10441 "once the run is settled the stop has landed and there is nothing to finish"
10442 );
10443 }
10444
10445 #[test]
10446 fn finishing_stays_true_until_the_last_of_several_runs_exits() {
10447 let stop = Stop::new();
10448 stop.busy.store(2, Ordering::SeqCst);
10449 stop.stop();
10450 assert!(stop.finishing(), "two runs still in flight");
10451
10452 stop.busy.store(1, Ordering::SeqCst);
10453 assert!(
10454 stop.finishing(),
10455 "one run finished, but a sibling is still working"
10456 );
10457
10458 stop.busy.store(0, Ordering::SeqCst);
10459 assert!(
10460 !stop.finishing(),
10461 "the last run out is what actually lands the stop"
10462 );
10463 }
10464
10465 #[tokio::test]
10466 async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
10467 let dir = tempfile::tempdir().unwrap();
10468 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10469 let stop = Stop::new();
10470 stop.stop();
10471
10472 let began = std::time::Instant::now();
10473 tokio::time::timeout(
10474 Duration::from_secs(2),
10475 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10476 )
10477 .await
10478 .expect("a stopped loop must return, not sit out its poll interval")
10479 .expect("the loop's own setup and teardown must not fail");
10480 assert!(
10481 began.elapsed() < opts.poll,
10482 "returned only after {:?}, which is a poll interval, not a stop",
10483 began.elapsed()
10484 );
10485 }
10486
10487 #[tokio::test]
10488 async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
10489 let dir = tempfile::tempdir().unwrap();
10490 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10491 let stop = Stop::new();
10492
10493 let asker = {
10496 let stop = stop.clone();
10497 tokio::spawn(async move {
10498 tokio::time::sleep(Duration::from_millis(20)).await;
10499 stop.stop();
10500 })
10501 };
10502
10503 let began = std::time::Instant::now();
10504 tokio::time::timeout(
10505 Duration::from_secs(2),
10506 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10507 )
10508 .await
10509 .expect("a stop asked for while idle must wake the wait")
10510 .expect("the loop's own setup and teardown must not fail");
10511 asker.await.unwrap();
10512 assert!(
10513 began.elapsed() < opts.poll,
10514 "returned only after {:?}, so the stop waited on the sleep",
10515 began.elapsed()
10516 );
10517 }
10518
10519 #[tokio::test]
10520 async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
10521 let dir = tempfile::tempdir().unwrap();
10522 let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
10523 let stop = Stop::new();
10524 stop.stop();
10525
10526 tokio::time::timeout(
10527 Duration::from_secs(2),
10528 drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
10529 )
10530 .await
10531 .expect("a stopped loop must return")
10532 .expect("the loop's own setup and teardown must not fail");
10533
10534 assert!(
10535 home.is_dir(),
10536 "the loop did publish a status file, so its removal is the teardown and not an absence"
10537 );
10538 assert!(
10539 !status_file.exists(),
10540 "a stopped loop clears its status file"
10541 );
10542 assert!(
10543 read_status(&home).is_none(),
10544 "a reader must see no daemon at all, not a heartbeat that merely stopped"
10545 );
10546 }
10547
10548 #[tokio::test]
10549 async fn housekeeping_worker_settles_questions_without_a_scheduler() {
10550 let dir = tempfile::tempdir().unwrap();
10551 let (opts, queue, _, home, worktrees) = idle_loop(dir.path());
10552
10553 let mut settled = RunState::new(
10554 dir.path().join("repo"),
10555 "main".to_owned(),
10556 "abc1234".to_owned(),
10557 "fixture".to_owned(),
10558 Config::default(),
10559 );
10560 settled.status = RunStatus::Ready;
10561 let run_dir = home.join("runs").join(&settled.id);
10562 std::fs::create_dir_all(&run_dir).unwrap();
10563 std::fs::write(
10564 run_dir.join("run.json"),
10565 serde_json::to_string_pretty(&settled).unwrap(),
10566 )
10567 .unwrap();
10568 let questions = Questions::at(home.join("questions"));
10569 let mut question = ask::Question::new(
10570 settled.id.clone(),
10571 "review".to_owned(),
10572 "reviewer-1".to_owned(),
10573 "Continue?".to_owned(),
10574 String::new(),
10575 Vec::new(),
10576 );
10577 questions.put(&mut question).unwrap();
10578
10579 run_service_worker(
10580 &home,
10581 serde_json::to_value(ServiceWorker {
10582 opts,
10583 queue: queue.root().to_path_buf(),
10584 worktrees_root: worktrees,
10585 operation: "housekeeping".to_owned(),
10586 })
10587 .unwrap(),
10588 )
10589 .await
10590 .unwrap();
10591
10592 assert_eq!(
10593 questions.get(&question.id).unwrap().status,
10594 ask::QuestionStatus::Abandoned,
10595 "the independent housekeeping worker settles ended-run questions"
10596 );
10597 }
10598
10599 #[test]
10600 fn cache_check_due_fires_immediately_then_waits_out_its_own_interval() {
10601 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10602
10603 assert!(
10604 cache_check_due(None, t0, CACHE_CHECK_INTERVAL_SECS),
10605 "never checked before: due at once"
10606 );
10607
10608 let one_sec_later = t0 + jiff::SignedDuration::from_secs(1);
10609 assert!(
10610 !cache_check_due(Some(t0), one_sec_later, CACHE_CHECK_INTERVAL_SECS),
10611 "well inside the interval: not due yet"
10612 );
10613
10614 let at_the_edge = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64);
10615 assert!(
10616 !cache_check_due(Some(t0), at_the_edge, CACHE_CHECK_INTERVAL_SECS),
10617 "exactly at the edge: not yet due, same convention as `clean::due`"
10618 );
10619
10620 let past_it = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10621 assert!(
10622 cache_check_due(Some(t0), past_it, CACHE_CHECK_INTERVAL_SECS),
10623 "past the interval: due again"
10624 );
10625 }
10626
10627 fn cache_check_opts(dir: &Path, cache_dir: &Path, limit_bytes: u64) -> Opts {
10632 let config = dir.join("magi.toml");
10633 std::fs::write(
10639 &config,
10640 format!(
10641 "[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = {limit_bytes}\n\n\
10642 [verify]\ngate = ['CARGO_TARGET_DIR={} cargo make check']\n",
10643 cache_dir.display()
10644 ),
10645 )
10646 .unwrap();
10647 Opts {
10648 config: Some(config),
10649 repo: dir.join("repo"),
10650 ..Opts::default()
10651 }
10652 }
10653
10654 #[tokio::test]
10655 async fn maybe_prune_cache_between_runs_reprunes_only_once_its_own_interval_elapses() {
10656 let dir = tempfile::tempdir().unwrap();
10657 let home = dir.path().join("home");
10658 let cache_dir = dir.path().join("cache");
10659 std::fs::create_dir_all(&cache_dir).unwrap();
10660 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10661 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10662
10663 let running = Stop::new();
10666 let mut last_checked = None;
10667 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10668 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &running, &mut last_checked, t0)
10669 .await;
10670 assert_eq!(
10671 crate::disk::dir_size(&cache_dir),
10672 0,
10673 "over the cap on the first check ever: pruned at once, no idle queue required"
10674 );
10675 assert_eq!(last_checked, Some(t0));
10676
10677 std::fs::write(cache_dir.join("b"), vec![0u8; 10]).unwrap();
10679 let too_soon = t0 + jiff::SignedDuration::from_secs(1);
10680 maybe_prune_cache_between_runs(
10681 &opts.repo,
10682 &opts,
10683 &home,
10684 &running,
10685 &mut last_checked,
10686 too_soon,
10687 )
10688 .await;
10689 assert_eq!(
10690 crate::disk::dir_size(&cache_dir),
10691 10,
10692 "too soon since the last check: left alone rather than rescanned every call"
10693 );
10694 assert_eq!(
10695 last_checked,
10696 Some(t0),
10697 "an idle check does not reset the clock"
10698 );
10699
10700 let due_again = t0 + jiff::SignedDuration::from_secs(CACHE_CHECK_INTERVAL_SECS as i64 + 1);
10702 maybe_prune_cache_between_runs(
10703 &opts.repo,
10704 &opts,
10705 &home,
10706 &running,
10707 &mut last_checked,
10708 due_again,
10709 )
10710 .await;
10711 assert_eq!(
10712 crate::disk::dir_size(&cache_dir),
10713 0,
10714 "due again: pruned back under the cap"
10715 );
10716 }
10717
10718 #[tokio::test]
10726 async fn a_stop_already_asked_for_skips_the_between_runs_cache_walk() {
10727 let dir = tempfile::tempdir().unwrap();
10728 let home = dir.path().join("home");
10729 let cache_dir = dir.path().join("cache");
10730 std::fs::create_dir_all(&cache_dir).unwrap();
10731 std::fs::write(cache_dir.join("a"), vec![0u8; 10]).unwrap();
10732 let opts = cache_check_opts(dir.path(), &cache_dir, 1);
10733
10734 let stop = Stop::new();
10735 stop.stop();
10736 assert!(
10737 !stop.finishing(),
10738 "no run is in flight at a between-runs boundary, so nothing else \
10739 would tell the operator this stop had not taken effect yet"
10740 );
10741
10742 let mut last_checked = None;
10743 let t0 = "2026-09-15T00:00:00Z".parse::<Timestamp>().unwrap();
10744 maybe_prune_cache_between_runs(&opts.repo, &opts, &home, &stop, &mut last_checked, t0)
10745 .await;
10746 assert_eq!(
10747 crate::disk::dir_size(&cache_dir),
10748 10,
10749 "over its cap, and due for the first check ever, but a stop outranks \
10750 it: the cap is a standing policy the next start measures again"
10751 );
10752 assert_eq!(
10753 last_checked, None,
10754 "a check that never happened must not claim the interval"
10755 );
10756 }
10757
10758 #[tokio::test]
10761 async fn cache_prune_reaches_a_queue_that_never_goes_idle() {
10762 let dir = tempfile::tempdir().unwrap();
10763 let cache_dir = dir.path().join("cache");
10764 std::fs::create_dir_all(&cache_dir).unwrap();
10765 std::fs::write(cache_dir.join("stale"), vec![0u8; 4096]).unwrap();
10766
10767 let mut opts = cache_check_opts(dir.path(), &cache_dir, 1);
10768 opts.poll = Duration::from_millis(20);
10769 opts.max_attempts = 1_000;
10770
10771 let queue = Queue::at(dir.path().join("queue"));
10772 for i in 0..128 {
10773 let mut task = Task::new(
10774 format!("queued work {i}"),
10775 format!("queued work {i}"),
10776 opts.repo.clone(),
10777 Source::Human,
10778 );
10779 queue.put(&mut task).unwrap();
10780 }
10781
10782 let home = dir.path().join("home");
10783 let worktrees = dir.path().join("wt");
10784 run_service_worker(
10785 &home,
10786 serde_json::to_value(ServiceWorker {
10787 opts,
10788 queue: queue.root().to_path_buf(),
10789 worktrees_root: worktrees,
10790 operation: "cache".to_owned(),
10791 })
10792 .unwrap(),
10793 )
10794 .await
10795 .unwrap();
10796 let after = queue.list();
10797 assert_eq!(after.len(), 128);
10798 assert!(
10799 after
10800 .iter()
10801 .all(|task| task.status.runnable() && task.attempts == 0)
10802 );
10803
10804 assert_eq!(
10805 crate::disk::dir_size(&cache_dir),
10806 0,
10807 "an oversized cache must not be left to grow unboundedly just because the \
10808 queue kept the loop busy the whole time"
10809 );
10810 }
10811
10812 #[test]
10813 fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
10814 let dir = tempfile::tempdir().unwrap();
10815 let queue = Queue::at(dir.path().join("queue"));
10816 let questions = Questions::at(dir.path().join("questions"));
10817 let mut task = task();
10818 queue.put(&mut task).unwrap();
10819
10820 let mut task_question = ask::Question::new(
10821 task.id.clone(),
10822 crate::conduct::NODE.to_owned(),
10823 "conduct".to_owned(),
10824 "Which backend?".to_owned(),
10825 String::new(),
10826 Vec::new(),
10827 );
10828 questions.put(&mut task_question).unwrap();
10829 task.block(vec![task_question.id.clone()], None);
10830 queue.put(&mut task).unwrap();
10831
10832 let mut run_question = ask::Question::new(
10833 "20260101-000000-run1".to_owned(),
10834 "review".to_owned(),
10835 "reviewer-1".to_owned(),
10836 "Run question".to_owned(),
10837 String::new(),
10838 Vec::new(),
10839 );
10840 questions.put(&mut run_question).unwrap();
10841
10842 let mut coincidental = ask::Question::new(
10847 task.id.clone(),
10848 "review".to_owned(),
10849 "reviewer-1".to_owned(),
10850 "Unrelated review question".to_owned(),
10851 String::new(),
10852 Vec::new(),
10853 );
10854 questions.put(&mut coincidental).unwrap();
10855
10856 reconcile_task_questions(&queue, &questions);
10857 assert!(questions.get(&task_question.id).unwrap().status.open());
10858 assert!(questions.get(&run_question.id).unwrap().status.open());
10859 assert!(questions.get(&coincidental.id).unwrap().status.open());
10860
10861 task.release();
10862 queue.put(&mut task).unwrap();
10863 reconcile_task_questions(&queue, &questions);
10864 assert_eq!(
10865 questions.get(&task_question.id).unwrap().status,
10866 ask::QuestionStatus::Abandoned
10867 );
10868 assert!(
10869 questions.get(&run_question.id).unwrap().status.open(),
10870 "run questions remain the run janitor's responsibility"
10871 );
10872 assert!(
10873 questions.get(&coincidental.id).unwrap().status.open(),
10874 "a non-conductor question must not be abandoned just because its \
10875 run id coincides with a task id"
10876 );
10877 }
10878
10879 #[test]
10880 fn a_freshly_started_running_task_is_never_stalled() {
10881 let dir = tempfile::tempdir().unwrap();
10882 let mut t = task();
10883 t.start("run-1".to_owned());
10884 assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
10887 }
10888
10889 #[test]
10890 fn a_long_running_task_with_no_live_daemon_is_stalled() {
10891 let dir = tempfile::tempdir().unwrap();
10892 let mut t = task();
10893 t.start("run-1".to_owned());
10894 t.updated_at = Timestamp::now()
10895 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10896 assert!(is_stalled(&t, dir.path(), Timestamp::now()));
10897 assert_eq!(
10898 stalled_tasks(
10899 &Queue::at(dir.path().join("q")),
10900 dir.path(),
10901 Timestamp::now()
10902 )
10903 .len(),
10904 0,
10905 "the task was never written to this queue"
10906 );
10907 }
10908
10909 #[test]
10910 fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
10911 let dir = tempfile::tempdir().unwrap();
10912 let mut t = task();
10913 t.id = "20260903-080340-0167".to_owned();
10914 t.start("20260903-080619-01c2".to_owned());
10915 t.updated_at = Timestamp::now()
10916 - jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
10917
10918 let mut status = Status::new();
10919 status.current = vec![Current {
10920 worker: None,
10921 task: t.id.clone(),
10922 run: "20260903-080619-01c2".to_owned(),
10923 }];
10924 write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
10925
10926 assert!(
10927 !is_stalled(&t, dir.path(), Timestamp::now()),
10928 "a live daemon's own heartbeat rules out stalled, however long the task has run"
10929 );
10930 }
10931
10932 fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
10936 let path = queue.path_of(id);
10937 let body = std::fs::read_to_string(&path).unwrap();
10938 let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
10939 let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
10940 v["updated_at"] = serde_json::Value::String(old.to_string());
10941 std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
10942 }
10943
10944 #[test]
10945 fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
10946 let dir = tempfile::tempdir().unwrap();
10959 let queue = Queue::at(dir.path().join("queue"));
10960 let home = dir.path().join("home");
10961
10962 let mut t = task();
10963 t.id = "20260101-000001-lock".to_owned();
10964 t.start("run-1".to_owned());
10965 queue.put(&mut t).unwrap();
10966 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
10967 std::fs::write(
10968 dir.path().join("queue").join(format!("{}.lock", t.id)),
10969 "not a pid",
10970 )
10971 .unwrap();
10972
10973 let now = Timestamp::now();
10974 assert!(
10975 reclaim_orphaned_running(&queue, 2, &Default::default()).is_empty(),
10976 "the unparseable lock is still well within STALE_CLAIM, so the claim fails \
10977 and reclaim must leave the task alone"
10978 );
10979 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
10980
10981 let stalled = stalled_tasks(&queue, &home, now);
10982 assert_eq!(
10983 stalled.len(),
10984 1,
10985 "reclaim's inability to claim it yet must not hide it from the conductor"
10986 );
10987 assert_eq!(stalled[0].id, t.id);
10988 }
10989
10990 #[test]
10991 fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
10992 let dir = tempfile::tempdir().unwrap();
10993 crate::run::set_home(dir.path().join("run-home"));
10994 let queue = Queue::at(dir.path().join("queue"));
10995 let home = dir.path().join("home");
10996 let questions = Questions::at(dir.path().join("questions"));
10997
10998 let mut t = task();
10999 t.id = "20260101-000003-dead".to_owned();
11000 t.start("missing-run".to_owned());
11001 queue.put(&mut t).unwrap();
11002 backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
11003
11004 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
11007 assert_eq!(
11008 stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
11009 [&t.id]
11010 );
11011 assert_eq!(
11012 reclaim_orphaned_running(&queue, 2, &Default::default()),
11013 [t.id.clone()]
11014 );
11015 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
11016
11017 crate::conduct::apply(
11020 &queue,
11021 &questions,
11022 &crate::conduct::Verdict {
11023 decisions: vec![crate::conduct::Decision {
11024 id: t.id.clone(),
11025 recovery: Some(crate::conduct::Recovery::Requeue),
11026 ..crate::conduct::Decision::default()
11027 }],
11028 },
11029 )
11030 .unwrap();
11031 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
11032 }
11033
11034 #[test]
11035 fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
11036 let dir = tempfile::tempdir().unwrap();
11037 let queue = Queue::at(dir.path().join("queue"));
11038 let home = dir.path().join("home");
11039
11040 let mut fresh = task();
11041 fresh.id = "20260101-000001-aaaa".to_owned();
11042 fresh.start("run-1".to_owned());
11043 queue.put(&mut fresh).unwrap();
11044
11045 let mut old = task();
11046 old.id = "20260101-000002-bbbb".to_owned();
11047 old.start("run-2".to_owned());
11048 queue.put(&mut old).unwrap();
11049 backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
11050
11051 let stalled = stalled_tasks(&queue, &home, Timestamp::now());
11052 assert_eq!(stalled.len(), 1);
11053 assert_eq!(stalled[0].id, old.id);
11054 }
11055
11056 #[test]
11057 fn queued_and_finished_task_views_partition_by_status() {
11058 let dir = tempfile::tempdir().unwrap();
11059 let queue = Queue::at(dir.path().join("queue"));
11060
11061 let mut queued = task();
11062 queued.id = "20260101-000001-aaaa".to_owned();
11063 queue.put(&mut queued).unwrap();
11064
11065 let mut failed = task();
11066 failed.id = "20260101-000002-bbbb".to_owned();
11067 failed.start("run-1".to_owned());
11068 failed.fail("gate red", 5);
11069 queue.put(&mut failed).unwrap();
11070
11071 let mut held = task();
11072 held.id = "20260101-000003-cccc".to_owned();
11073 held.hold_machine(None);
11074 queue.put(&mut held).unwrap();
11075
11076 let mut running = task();
11077 running.id = "20260101-000004-dddd".to_owned();
11078 running.start("run-2".to_owned());
11079 queue.put(&mut running).unwrap();
11080
11081 let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
11082 assert_eq!(queued_ids, [queued.id.clone()]);
11083
11084 let mut finished_ids: Vec<String> =
11085 finished_tasks(&queue).into_iter().map(|t| t.id).collect();
11086 finished_ids.sort_unstable();
11087 let mut want = vec![failed.id.clone(), held.id.clone()];
11088 want.sort_unstable();
11089 assert_eq!(finished_ids, want);
11090 }
11091
11092 #[test]
11093 fn dependency_resolution_preserves_existing_short_and_full_running_chains() {
11094 for short in [false, true] {
11095 let dir = tempfile::tempdir().unwrap();
11096 let queue = Queue::at(dir.path().join("queue"));
11097 let questions = ask::Questions::at(dir.path().join("questions"));
11098 let mut running = task();
11099 running.id = "20261013-192213-feb2".into();
11100 running.start("existing-run".into());
11101 queue.put(&mut running).unwrap();
11102 let mut middle = task();
11103 middle.id = "20261013-193611-ca5b".into();
11104 middle.block(
11105 vec![if short {
11106 "feb2".into()
11107 } else {
11108 running.id.clone()
11109 }],
11110 None,
11111 );
11112 queue.put(&mut middle).unwrap();
11113 let mut waiting = task();
11114 waiting.id = "20261013-202849-ee37".into();
11115 waiting.block(
11116 vec![if short {
11117 "ca5b".into()
11118 } else {
11119 middle.id.clone()
11120 }],
11121 Some("wait for implementation".into()),
11122 );
11123 queue.put(&mut waiting).unwrap();
11124 assert_eq!(queue.get("ca5b").unwrap().id, middle.id);
11125 let inv = crate::blockers::Inventory::load(&queue, &questions);
11126 assert_eq!(inv.waits_on(&waiting), ["ca5b (blocked → feb2 running)"]);
11127 assert!(inv.stuck().is_empty());
11128 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11129 assert!(report.quarantined.is_empty());
11130 assert!(report.asked.is_empty());
11131 resolve_blockers(&queue, &questions);
11132 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11133 assert!(questions.list().is_empty());
11134 running.succeed();
11137 queue.put(&mut running).unwrap();
11138 resolve_blockers(&queue, &questions);
11139 assert_eq!(queue.get(&middle.id).unwrap().status, TaskStatus::Queued);
11140 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11141 middle = queue.get(&middle.id).unwrap();
11142 middle.succeed();
11143 queue.put(&mut middle).unwrap();
11144 resolve_blockers(&queue, &questions);
11145 let after = queue.get(&waiting.id).unwrap();
11146 assert_eq!(after.status, TaskStatus::Queued);
11147 assert!(after.blocked_by.is_empty());
11148 assert_eq!(after.instruction, waiting.instruction);
11149 }
11150 }
11151
11152 #[test]
11153 fn dependency_resolution_unknown_records_never_hold_ask_or_release() {
11154 for future_schema in [false, true] {
11155 let name = "20261013-193611-ca5b";
11156 let mut future = task();
11157 future.id = name.into();
11158 future.schema = crate::queue::SCHEMA + 1;
11159 let contents = if future_schema {
11160 serde_json::to_string(&future).unwrap()
11161 } else {
11162 "corrupt".into()
11163 };
11164 let dir = tempfile::tempdir().unwrap();
11165 let queue = Queue::at(dir.path().join("queue"));
11166 let questions = ask::Questions::at(dir.path().join("questions"));
11167 let mut waiting = task();
11168 waiting.id = "20261013-202849-ee37".into();
11169 waiting.block(vec!["ca5b".into()], None);
11170 queue.put(&mut waiting).unwrap();
11171 std::fs::write(queue.path_of(name), contents).unwrap();
11172 assert_eq!(queue.resolve_id("ca5b").unwrap(), name);
11173 assert!(
11174 crate::queue::missing_blockers(&queue, &questions, &waiting.blocked_by).is_empty()
11175 );
11176 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11177 assert!(report.asked.is_empty());
11178 resolve_blockers(&queue, &questions);
11179 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11180 assert!(questions.list().is_empty());
11181 }
11182 let dir = tempfile::tempdir().unwrap();
11183 let queue = Queue::at(dir.path().join("queue"));
11184 let questions = ask::Questions::at(dir.path().join("questions"));
11185 let mut waiting = task();
11186 waiting.id = "20261013-202849-ee37".into();
11187 waiting.block(vec!["ca5b".into()], None);
11188 queue.put(&mut waiting).unwrap();
11189 let mut done = task();
11190 done.id = "20261013-193611-ca5b".into();
11191 done.succeed();
11192 queue.put(&mut done).unwrap();
11193 std::fs::write(queue.path_of("20261014-193611-ca5b"), "bad").unwrap();
11196 assert!(queue.get("ca5b").is_err());
11197 assert_eq!(
11198 crate::queue::dependency(&queue, &questions, "ca5b"),
11199 crate::queue::Dependency::Ambiguous
11200 );
11201 resolve_blockers(&queue, &questions);
11202 let report = crate::triage::run_once(&queue, &questions, None, Timestamp::now());
11203 assert!(report.asked.is_empty());
11204 assert_eq!(queue.get(&waiting.id).unwrap().status, TaskStatus::Blocked);
11205 assert_eq!(queue.get(&done.id).unwrap().status, TaskStatus::Done);
11206 }
11207
11208 #[test]
11209 fn dependency_resolution_preserves_independent_consent_and_holds_after_completion() {
11210 for held in [false, true] {
11211 for question in [false, true] {
11212 for reference in ["ca5b", "20261013-193611-ca5b"] {
11213 let dir = tempfile::tempdir().unwrap();
11214 let queue = Queue::at(dir.path().join("queue"));
11215 let questions = ask::Questions::at(dir.path().join("questions"));
11216 let mut prerequisite = task();
11217 prerequisite.id = "20261013-193611-ca5b".into();
11218 prerequisite.start("existing-run".into());
11219 queue.put(&mut prerequisite).unwrap();
11220 let mut waiting = task();
11221 waiting.id = "20261013-202849-ee37".into();
11222 if held {
11223 waiting.hold_machine(Some("waiting for separate consent".into()));
11224 }
11225 queue.put(&mut waiting).unwrap();
11226 let reason = "Deleting saved data requires separate owner consent";
11227 crate::conduct::apply(
11228 &queue,
11229 &questions,
11230 &crate::conduct::Verdict {
11231 decisions: vec![crate::conduct::Decision {
11232 id: waiting.id.clone(),
11233 blocked_by: vec![reference.into()],
11234 question: question.then(|| "May the saved data be deleted?".into()),
11235 recovery: (!question).then_some(crate::conduct::Recovery::Hold),
11236 reason: Some(reason.into()),
11237 ..crate::conduct::Decision::default()
11238 }],
11239 },
11240 )
11241 .unwrap();
11242 let before = queue.get(&waiting.id).unwrap();
11243 if question {
11244 assert_eq!(before.status, TaskStatus::Blocked);
11245 let q = questions.get(&before.blocked_by[0]).unwrap();
11246 assert_eq!(q.summary, "May the saved data be deleted?");
11247 assert!(q.status.open());
11248 } else {
11249 assert_eq!(before.status, TaskStatus::Held);
11250 let recorded_reason = before.hold_reason.as_deref().unwrap();
11251 assert!(recorded_reason.starts_with(reason));
11252 if held {
11253 assert!(recorded_reason.contains("waiting for separate consent"));
11254 }
11255 }
11256 prerequisite.succeed();
11257 queue.put(&mut prerequisite).unwrap();
11258 resolve_blockers(&queue, &questions);
11259 let after = queue.get(&waiting.id).unwrap();
11260 assert_eq!(after.status, before.status);
11261 assert_eq!(after.blocked_by, before.blocked_by);
11262 assert_eq!(after.hold_reason, before.hold_reason);
11263 assert_eq!(after.instruction, waiting.instruction);
11264 if question {
11265 assert!(questions.get(&after.blocked_by[0]).unwrap().status.open());
11266 }
11267 assert!(
11268 queue.next_runnable().is_none(),
11269 "dependency completion is not consent"
11270 );
11271 }
11272 }
11273 }
11274 }
11275
11276 #[test]
11277 fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
11278 let dir = tempfile::tempdir().unwrap();
11279 let queue = Queue::at(dir.path().join("queue"));
11280 let questions = ask::Questions::at(dir.path().join("questions"));
11281
11282 let mut dep = task();
11283 dep.id = "20260101-000001-dep0".to_owned();
11284 dep.succeed();
11285 queue.put(&mut dep).unwrap();
11286
11287 let mut still_going = task();
11288 still_going.id = "20260101-000002-dep1".to_owned();
11289 queue.put(&mut still_going).unwrap();
11290
11291 let mut blocked = task();
11292 blocked.id = "20260101-000003-main".to_owned();
11293 blocked.block(
11294 vec![dep.id.clone(), still_going.id.clone()],
11295 Some("waits on both".to_owned()),
11296 );
11297 queue.put(&mut blocked).unwrap();
11298
11299 resolve_blockers(&queue, &questions);
11300
11301 let after = queue.get(&blocked.id).unwrap();
11302 assert_eq!(
11303 after.status,
11304 TaskStatus::Blocked,
11305 "one dependency is still outstanding"
11306 );
11307 assert_eq!(after.blocked_by, [still_going.id.clone()]);
11308 }
11309
11310 #[test]
11311 fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
11312 let dir = tempfile::tempdir().unwrap();
11313 let queue = Queue::at(dir.path().join("queue"));
11314 let questions = ask::Questions::at(dir.path().join("questions"));
11315
11316 let mut q = crate::ask::Question::new(
11317 "20260101-000001-main".to_owned(),
11318 crate::conduct::NODE.to_owned(),
11319 "conduct".to_owned(),
11320 "Which backend?".to_owned(),
11321 String::new(),
11322 Vec::new(),
11323 );
11324 questions.put(&mut q).unwrap();
11325 q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
11326 .unwrap();
11327 questions.put(&mut q).unwrap();
11328
11329 let mut blocked = task();
11330 blocked.id = "20260101-000001-main".to_owned();
11331 blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
11332 queue.put(&mut blocked).unwrap();
11333
11334 resolve_blockers(&queue, &questions);
11335
11336 let after = queue.get(&blocked.id).unwrap();
11337 assert_eq!(
11338 after.status,
11339 TaskStatus::Queued,
11340 "the only blocker resolved"
11341 );
11342 assert_eq!(after.answers.len(), 1);
11343 assert_eq!(after.answers[0].question, "Which backend?");
11344 assert_eq!(after.answers[0].answer, "SQLite");
11345
11346 let instruction = instruction_for(&after);
11348 assert!(instruction.contains("Which backend?"));
11349 assert!(instruction.contains("SQLite"));
11350 }
11351
11352 #[test]
11353 fn resolve_blockers_holds_a_task_whose_conductor_question_was_abandoned() {
11354 let dir = tempfile::tempdir().unwrap();
11355 let queue = Queue::at(dir.path().join("queue"));
11356 let questions = ask::Questions::at(dir.path().join("questions"));
11357
11358 let mut q = crate::ask::Question::new(
11359 "20260101-000001-main".to_owned(),
11360 crate::conduct::NODE.to_owned(),
11361 "conduct".to_owned(),
11362 "Is the setup done?".to_owned(),
11363 String::new(),
11364 Vec::new(),
11365 );
11366 q.abandon("no answer within 60s of asking");
11367 questions.put(&mut q).unwrap();
11368
11369 let mut blocked = task();
11370 blocked.id = "20260101-000001-main".to_owned();
11371 blocked.block(vec![q.id.clone()], Some("setup?".to_owned()));
11372 queue.put(&mut blocked).unwrap();
11373
11374 resolve_blockers(&queue, &questions);
11375
11376 let after = queue.get(&blocked.id).unwrap();
11377 assert_eq!(
11378 after.status,
11379 TaskStatus::Held,
11380 "never left blocked on nothing"
11381 );
11382 assert!(!after.operator_held(), "a machine hold, for triage");
11383 assert!(
11384 after
11385 .hold_reason
11386 .as_deref()
11387 .unwrap_or_default()
11388 .contains("went unanswered")
11389 );
11390 }
11391
11392 #[test]
11393 fn resolve_blockers_restores_a_held_task_to_held_instead_of_queuing_it() {
11394 let dir = tempfile::tempdir().unwrap();
11400 let queue = Queue::at(dir.path().join("queue"));
11401 let questions = ask::Questions::at(dir.path().join("questions"));
11402
11403 let mut q = crate::ask::Question::new(
11404 "20260101-000001-main".to_owned(),
11405 crate::conduct::NODE.to_owned(),
11406 "conduct".to_owned(),
11407 "How should this be handled?".to_owned(),
11408 String::new(),
11409 Vec::new(),
11410 );
11411 questions.put(&mut q).unwrap();
11412 q.answer(crate::ask::Answer::Text(
11413 "leave it held, a human will look at it later".to_owned(),
11414 ))
11415 .unwrap();
11416 questions.put(&mut q).unwrap();
11417
11418 let mut held = task();
11419 held.id = "20260101-000001-main".to_owned();
11420 held.hold_machine(Some("out of attempts".to_owned()));
11421 held.block(vec![q.id.clone()], Some("what now?".to_owned()));
11422 queue.put(&mut held).unwrap();
11423
11424 resolve_blockers(&queue, &questions);
11425
11426 let after = queue.get(&held.id).unwrap();
11427 assert_eq!(after.status, TaskStatus::Held);
11428 assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
11429 assert_eq!(
11430 after.answers[0].answer,
11431 "leave it held, a human will look at it later"
11432 );
11433 }
11434
11435 #[test]
11436 fn resolve_blockers_releases_a_task_whose_dependency_was_deleted_on_purpose() {
11437 let dir = tempfile::tempdir().unwrap();
11438 let queue = Queue::at(dir.path().join("queue"));
11439 let questions = ask::Questions::at(dir.path().join("questions"));
11440 let mut dep = task();
11441 dep.id = "20260101-000001-gone".to_owned();
11442 queue.put(&mut dep).unwrap();
11443 let mut blocked = task();
11444 blocked.id = "20260101-000003-main".to_owned();
11445 blocked.block(vec![dep.id.clone()], None);
11446 queue.put(&mut blocked).unwrap();
11447
11448 let claim = queue.claim(&blocked.id).unwrap();
11450 queue.remove(&dep.id, false, &questions).unwrap();
11451 drop(claim);
11452 resolve_blockers(&queue, &questions);
11453
11454 let after = queue.get(&blocked.id).unwrap();
11455 assert_eq!(after.status, TaskStatus::Queued);
11456 assert!(after.blocked_by.is_empty());
11457 }
11458
11459 #[test]
11460 fn resolve_blockers_holds_a_task_whose_dependency_was_deleted() {
11461 let dir = tempfile::tempdir().unwrap();
11467 let queue = Queue::at(dir.path().join("queue"));
11468 let questions = ask::Questions::at(dir.path().join("questions"));
11469
11470 let mut still_going = task();
11471 still_going.id = "20260101-000002-dep1".to_owned();
11472 queue.put(&mut still_going).unwrap();
11473
11474 let mut blocked = task();
11475 blocked.id = "20260101-000003-main".to_owned();
11476 blocked.block(
11477 vec!["20260101-000001-gone".to_owned(), still_going.id.clone()],
11478 Some("waits on both".to_owned()),
11479 );
11480 queue.put(&mut blocked).unwrap();
11481
11482 resolve_blockers(&queue, &questions);
11483
11484 let after = queue.get(&blocked.id).unwrap();
11485 assert_eq!(
11486 after.status,
11487 TaskStatus::Held,
11488 "a missing dependency must not leave the task blocked forever"
11489 );
11490 assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
11491 assert!(after.blocked_by.is_empty());
11492 let reason = after.hold_reason.as_deref().unwrap_or_default();
11493 assert!(
11494 reason.contains("20260101-000001-gone"),
11495 "the missing id must be named so an operator can tell what happened: {reason}"
11496 );
11497 assert!(
11498 reason.contains(&still_going.id),
11499 "the still-valid dependency must not silently vanish from the record: {reason}"
11500 );
11501 }
11502
11503 #[test]
11504 fn instruction_for_is_unchanged_without_any_answers() {
11505 let t = task();
11506 assert_eq!(instruction_for(&t), t.instruction);
11507 }
11508
11509 #[test]
11510 fn task_attachments_are_absolute_and_a_missing_file_is_an_error() {
11511 let dir = tempfile::tempdir().unwrap();
11512 let q = Queue::at(dir.path().join("queue"));
11513 let src = dir.path().join("shot.png");
11514 std::fs::write(&src, "x").unwrap();
11515 let mut t = task();
11516 q.attach(&mut t, &[src]).unwrap();
11517 let paths = task_attachments(&q, &t).unwrap();
11518 assert_eq!(paths.len(), 1);
11519 assert!(paths[0].is_absolute() && paths[0].is_file());
11520 std::fs::remove_file(&paths[0]).unwrap();
11521 let err = task_attachments(&q, &t).unwrap_err().to_string();
11522 assert!(err.contains("shot.png"), "{err}");
11523 }
11524
11525 #[tokio::test]
11526 async fn config_failure_notices_distinguish_boundaries_and_targets_after_restart() {
11527 let dir = tempfile::tempdir().unwrap();
11528 let queue_path = dir.path().join("queue");
11529 let queue = Queue::at(queue_path.clone());
11530 let mut t = task();
11531 t.source = Source::Agent {
11532 run: "originating-chat".into(),
11533 node: crate::queue::CHAT_NODE.into(),
11534 };
11535 let fetch = anyhow::anyhow!("network unavailable").context(crate::config::ConfigFailure {
11536 stage: "config-fetch",
11537 target: "origin/main".into(),
11538 });
11539 record_config_failure(&mut t, &fetch);
11540 t.hold_machine(Some(format!("{fetch:#}")));
11541 let fetch_key = t.chat_notice_due().unwrap();
11542 t.chat_notices.push(fetch_key.clone());
11543 queue.put(&mut t).unwrap();
11544
11545 let queue = Queue::at(queue_path);
11546 let mut t = queue.get(&t.id).unwrap();
11547 let a = dir.path().join("a.toml");
11548 let b = dir.path().join("b.toml");
11549 for (path, text, new_cause) in [
11550 (&a, "[graph", true),
11551 (&a, "[verify", false),
11552 (&a, "[graph]\nimplementers = 'invalid-type'", true),
11553 (&b, "[graph]\nimplementers = 'invalid-type'", true),
11554 ] {
11555 t.release_operator("test", None);
11556 std::fs::write(path, text).unwrap();
11557 let error = Config::discover_fetched(dir.path(), Some(path))
11558 .await
11559 .unwrap_err();
11560 record_config_failure(&mut t, &error);
11561 t.hold_machine(Some(format!("{error:#}")));
11562 let due = t.chat_notice_due();
11563 assert_eq!(due.is_some(), new_cause, "{error:#}");
11564 if let Some(key) = due {
11565 assert_ne!(key, fetch_key);
11566 t.chat_notices.push(key);
11567 }
11568 }
11569 t.succeed();
11570 assert_eq!(t.chat_notice_due().as_deref(), Some("done"));
11571 }
11572
11573 #[test]
11574 fn different_missing_attachments_have_distinct_persisted_notice_causes() {
11575 let dir = tempfile::tempdir().unwrap();
11576 let queue_path = dir.path().join("queue");
11577 let queue = Queue::at(queue_path.clone());
11578 let mut t = task();
11579 t.source = crate::queue::Source::Agent {
11580 run: "originating-chat".into(),
11581 node: crate::queue::CHAT_NODE.into(),
11582 };
11583 let a = dir.path().join("a.png");
11584 let b = dir.path().join("b.png");
11585 std::fs::write(&a, "a").unwrap();
11586 std::fs::write(&b, "b").unwrap();
11587 queue.attach(&mut t, &[a, b]).unwrap();
11588 let paths = task_attachments(&queue, &t).unwrap();
11589 for path in &paths {
11590 std::fs::remove_file(path).unwrap();
11591 }
11592 let missing = task_attachments(&queue, &t).unwrap_err();
11593 t.attempts += 1;
11594 let reason = missing.to_string();
11595 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11596 let first = t.chat_notice_due().expect("first attachment notice");
11597 t.chat_notices.push(first.clone());
11598 queue.put(&mut t).unwrap();
11599
11600 let queue = Queue::at(queue_path.clone());
11601 let mut t = queue.get(&t.id).unwrap();
11602 assert!(t.chat_notice_due().is_none(), "restart does not replay");
11603 t.release_operator("test", None);
11604 t.attempts += 1;
11605 let missing = task_attachments(&queue, &t).unwrap_err();
11606 t.fail_startup_target(
11607 "startup-attachments",
11608 missing.name,
11609 "same missing attachment with regenerated wording",
11610 1,
11611 );
11612 assert!(t.chat_notice_due().is_none(), "same object stays quiet");
11613
11614 std::fs::write(&paths[0], "restored").unwrap();
11615 t.release_operator("test", None);
11616 t.attempts += 1;
11617 let missing = task_attachments(&queue, &t).unwrap_err();
11618 let reason = missing.to_string();
11619 t.fail_startup_target("startup-attachments", missing.name, reason, 1);
11620 let second = t
11621 .chat_notice_due()
11622 .expect("different attachment must notify");
11623 assert_ne!(first, second);
11624 assert!(t.hold_reason.as_ref().unwrap().contains("b.png"));
11625 }
11626
11627 #[test]
11628 fn resumed_instruction_is_unchanged_without_any_answers() {
11629 let t = task();
11630 assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
11631 }
11632
11633 #[test]
11634 fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
11635 let mut t = task();
11636 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11637 let old = t.instruction.clone();
11641
11642 let refreshed = resumed_instruction(&old, &t);
11643 assert!(refreshed.starts_with(&old), "the original text is kept");
11644 assert!(refreshed.contains("Which backend?"));
11645 assert!(refreshed.contains("SQLite"));
11646 }
11647
11648 #[test]
11649 fn resumed_instruction_keeps_an_original_answers_heading() {
11650 let mut t = task();
11651 t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
11652 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11653
11654 let refreshed = resumed_instruction(&t.instruction, &t);
11655
11656 assert!(
11657 refreshed.starts_with(&t.instruction),
11658 "an answers heading in the original instruction is not the appended block"
11659 );
11660 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
11661 assert!(refreshed.contains("Which backend?"));
11662 assert!(refreshed.contains("SQLite"));
11663
11664 let repeated = resumed_instruction(&refreshed, &t);
11665 assert_eq!(
11666 repeated, refreshed,
11667 "only the final appended block is refreshed"
11668 );
11669 }
11670
11671 #[test]
11672 fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
11673 let mut t = task();
11674 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11675
11676 let once = resumed_instruction(&t.instruction, &t);
11680 let twice = resumed_instruction(&once, &t);
11681 assert_eq!(once, twice);
11682 assert_eq!(once.matches("Which backend?").count(), 1);
11683
11684 t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
11686 let refreshed = resumed_instruction(&once, &t);
11687 assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
11688 assert!(refreshed.contains("Which backend?"));
11689 assert!(refreshed.contains("Which cache?"));
11690 }
11691
11692 #[test]
11693 fn prepare_instruction_covers_all_three_starters() {
11694 let mut t = task();
11695 t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
11696
11697 assert_eq!(
11700 prepare_instruction(&Starter::Start, None, &t),
11701 Some(instruction_for(&t))
11702 );
11703
11704 let old = t.instruction.clone();
11707 assert_eq!(
11708 prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
11709 Some(resumed_instruction(&old, &t))
11710 );
11711
11712 assert_eq!(
11716 prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
11717 None
11718 );
11719 }
11720
11721 #[test]
11722 fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
11723 assert_eq!(
11724 choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
11725 Starter::Review("magi/eba2/A".to_owned())
11726 );
11727 }
11728
11729 #[test]
11730 fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
11731 assert_eq!(
11732 choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
11733 Starter::Start,
11734 "a vanished review branch must not fall back to resuming the old run either"
11735 );
11736 }
11737
11738 #[test]
11739 fn a_refused_handover_retries_as_a_review_of_the_same_branch() {
11740 let mut t = task();
11741 t.start("old-run".to_owned());
11742 t.hold_for_handover(Some("magi/eba2/A".to_owned()), "checked out".to_owned());
11743 t.release();
11744 let branch = t.review_branch.take();
11745 assert_eq!(
11746 choose_starter(branch.as_deref(), true, Some("old-run")),
11747 Starter::Review("magi/eba2/A".to_owned()),
11748 "a review wins over resuming the old run"
11749 );
11750 }
11751
11752 #[test]
11753 fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
11754 assert_eq!(
11755 choose_starter(None, false, Some("some-run")),
11756 Starter::Resume("some-run".to_owned())
11757 );
11758 assert_eq!(choose_starter(None, false, None), Starter::Start);
11759 }
11760
11761 #[test]
11762 fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
11763 let mut released = task();
11764 released.start("stalled-run".to_owned());
11765 released.requeue();
11766 let unfinished = (!released.fresh_start)
11767 .then(|| Some("stalled-run".to_owned()))
11768 .flatten();
11769 assert_eq!(
11770 choose_starter(None, false, unfinished.as_deref()),
11771 Starter::Start,
11772 "release keeps run history but must not resume it"
11773 );
11774 assert_eq!(released.runs, ["stalled-run"]);
11775 }
11776
11777 fn stopped_public_run(status: RunStatus) -> RunState {
11778 let mut state = run_state(status);
11779 state.driver_exited = true;
11780 state.config.graph.land = true;
11781 state.config.graph.review_rounds = 1;
11782 state.reviews = vec![review_round(1)];
11783 state.merge = Some(crate::run::MergeOutcome {
11784 mode: MergeMode::Pr,
11785 ok: false,
11786 detail: "landing stopped".into(),
11787 empty: false,
11788 });
11789 state.pr = Some(crate::run::PrRecord {
11790 url: "https://example.test/pull/1".into(),
11791 number: 1,
11792 state: "open".into(),
11793 checks: "red".into(),
11794 round: 1,
11795 rounds: 1,
11796 red_at_merge: Vec::new(),
11797 });
11798 state
11799 }
11800
11801 #[test]
11802 fn stopped_public_owner_resumes_on_release_choice_and_control_after_review_exhaustion() {
11803 for status in [RunStatus::Blocked, RunStatus::Gating] {
11804 let home = tempfile::tempdir().unwrap();
11805 let mut state = stopped_public_run(status);
11806 state.save_under(home.path()).unwrap();
11807 assert!(crate::handover::has_resume_checkpoint(&state, home.path()));
11808 assert!(!exhausted_review_budget(&state));
11809 let mut task = held_task_with(&state.id);
11810 task.release();
11811 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11812 assert_eq!(
11813 choose_starter(None, false, unfinished.as_deref()),
11814 Starter::Resume(state.id.clone())
11815 );
11816 let question = action_question(&state.id, resume_action(&state.id));
11817 assert_eq!(
11818 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
11819 state.clone()
11820 )),
11821 ActionDecision::Resume(state.id.clone())
11822 );
11823 assert_eq!(
11825 crate::resume::validate(home.path(), &state.id).unwrap().id,
11826 state.id
11827 );
11828 let queue = Queue::at(home.path().join("queue"));
11829 let mut controlled = Task::new(
11830 "resume land".into(),
11831 state.instruction.clone(),
11832 state.repo.clone(),
11833 Source::Agent {
11834 run: "talk".into(),
11835 node: crate::queue::CHAT_NODE.into(),
11836 },
11837 );
11838 controlled.runs.push(state.id.clone());
11839 controlled.hold_machine(Some("land stopped".into()));
11840 queue.put(&mut controlled).unwrap();
11841 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
11842 assert_eq!(resumed.status, TaskStatus::Queued);
11843 assert_eq!(
11844 resumed.resume_override.unwrap().pinned_run,
11845 Some(state.id.clone())
11846 );
11847 assert_eq!(resumed.attempts, 0);
11848 }
11849 }
11850
11851 #[test]
11852 fn exhausted_gate_failure_with_clean_or_disabled_reviews_is_not_resumable() {
11853 for review_rounds in [0, 1] {
11854 let home = tempfile::tempdir().unwrap();
11855 let mut state = run_state(RunStatus::Blocked);
11856 state.config.graph.review_rounds = review_rounds;
11857 if review_rounds > 0 {
11858 let mut review = review_round(1);
11859 review.clean = true;
11860 state.reviews = vec![review];
11861 }
11862 state.gate_ran = true;
11863 state.gate = vec![crate::run::CommandOutcome {
11864 command: "verify".into(),
11865 code: Some(1),
11866 output_tail: "gate failed after repair budget exhausted".into(),
11867 duration_ms: 0,
11868 resource_blocked: false,
11869 }];
11870 state.save_under(home.path()).unwrap();
11871 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11872 assert!(exhausted_review_budget(&state));
11873 let mut task = held_task_with(&state.id);
11874 task.release();
11875 assert!(unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())).is_none());
11876 let question = action_question(&state.id, resume_action(&state.id));
11877 assert_ne!(
11878 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
11879 state.clone()
11880 )),
11881 ActionDecision::Resume(state.id.clone())
11882 );
11883 let error = crate::resume::validate(home.path(), &state.id)
11884 .unwrap_err()
11885 .to_string();
11886 assert!(!error.contains("review budget exhausted"), "{error}");
11887 assert!(
11888 error.contains(if review_rounds == 0 {
11889 "review is disabled"
11890 } else {
11891 "gate already ran"
11892 }),
11893 "{error}"
11894 );
11895
11896 state.gate_ran = false;
11898 state.gate.clear();
11899 assert!(!exhausted_review_budget(&state));
11900 assert_eq!(
11901 unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone())),
11902 Some(state.id)
11903 );
11904 }
11905 }
11906
11907 #[test]
11908 fn resource_blocked_verification_resumes_the_same_run() {
11909 for (kind, stage) in [
11910 ("cache", "clean"),
11911 ("coordination", "clean"),
11912 ("cache", "review-e2e"),
11913 ("coordination", "no-reviews"),
11914 ("coordination", "handoff"),
11915 ] {
11916 let home = tempfile::tempdir().unwrap();
11917 let mut state = run_state(RunStatus::Blocked);
11918 state.config.graph.review_rounds = 1;
11919 let mut review = review_round(1);
11920 review.clean = true;
11921 state.reviews = vec![review];
11922 if stage == "review-e2e" {
11923 let review = &mut state.reviews[0];
11924 review.clean = false;
11925 review.blocking = 1;
11926 review.e2e = vec![crate::run::CommandOutcome {
11927 command: "verify".into(),
11928 code: None,
11929 output_tail: "resource wait expired".into(),
11930 duration_ms: 0,
11931 resource_blocked: true,
11932 }];
11933 } else if stage == "no-reviews" {
11934 state.config.graph.review_rounds = 0;
11935 state.reviews.clear();
11936 } else if stage == "handoff" {
11937 let review = &mut state.reviews[0];
11940 review.clean = false;
11941 review.blocking = 1;
11942 review.e2e = vec![crate::run::CommandOutcome {
11943 command: "verify".into(),
11944 code: Some(0),
11945 output_tail: String::new(),
11946 duration_ms: 0,
11947 resource_blocked: false,
11948 }];
11949 }
11950 state.gate_ran = false;
11951 state.resource_block = Some(crate::run::ResourceBlock {
11952 cause: crate::run::ResourceBlockCause::Other,
11953 node: if stage == "review-e2e" {
11954 "review"
11955 } else {
11956 "gate"
11957 }
11958 .into(),
11959 kind: kind.into(),
11960 detail: "resource wait expired".into(),
11961 at: jiff::Timestamp::now(),
11962 });
11963 state.save_under(home.path()).unwrap();
11964 assert!(!crate::handover::has_resume_checkpoint(&state, home.path()));
11965 assert!(!exhausted_review_budget(&state));
11966 let mut task = held_task_with(&state.id);
11967 task.release();
11968 let unfinished = unfinished_run_with(&task.runs, task.short(), |_| Ok(state.clone()));
11969 assert_eq!(
11970 choose_starter(None, false, unfinished.as_deref()),
11971 Starter::Resume(state.id.clone())
11972 );
11973 let question = action_question(&state.id, resume_action(&state.id));
11974 assert_eq!(
11975 decide_action(home.path(), &task, &question, &PHRASES_EN, |_| Ok(
11976 state.clone()
11977 )),
11978 ActionDecision::Resume(state.id.clone())
11979 );
11980 assert_eq!(
11981 crate::resume::validate(home.path(), &state.id).unwrap().id,
11982 state.id
11983 );
11984 let queue = Queue::at(home.path().join("queue"));
11985 let mut controlled = Task::new(
11986 "resume verification".into(),
11987 state.instruction.clone(),
11988 state.repo.clone(),
11989 Source::Agent {
11990 run: "talk".into(),
11991 node: crate::queue::CHAT_NODE.into(),
11992 },
11993 );
11994 controlled.runs.push(state.id.clone());
11995 controlled.hold_machine(Some("verification resource unavailable".into()));
11996 queue.put(&mut controlled).unwrap();
11997 let resumed = crate::chat_control::resume(&queue, home.path(), &state.id).unwrap();
11998 assert_eq!(resumed.status, TaskStatus::Queued);
11999 assert_eq!(
12000 resumed.resume_override.unwrap().pinned_run,
12001 Some(state.id.clone())
12002 );
12003 assert_eq!(resumed.attempts, 0);
12004
12005 state.config.graph.review_rounds = 1;
12007 let mut unresolved = review_round(1);
12008 unresolved.blocking = 1;
12009 state.reviews = vec![unresolved];
12010 state.resource_block = None;
12011 state.save_under(home.path()).unwrap();
12012 assert!(exhausted_review_budget(&state));
12013 assert!(crate::resume::validate(home.path(), &state.id).is_err());
12014 }
12015 }
12016
12017 #[tokio::test]
12018 async fn ownership_contention_waits_and_reads_the_settled_record_before_resuming() {
12019 if !crate::standalone_test(
12020 "daemon::tests::ownership_contention_waits_and_reads_the_settled_record_before_resuming",
12021 ) {
12022 return;
12023 }
12024 let home = tempfile::tempdir().unwrap();
12025 let mut owner = stopped_public_run(RunStatus::Blocked);
12026 owner.save_under(home.path()).unwrap();
12027 let history = vec![owner.id.clone()];
12028 let repo = owner.repo.clone();
12029 let lock = crate::pr_head::owner_lock(home.path(), &owner.id).unwrap();
12030 let mut lookup = Box::pin(public_resume_owner(home.path(), &history, &repo));
12031 let initial =
12032 std::future::poll_fn(|cx| std::task::Poll::Ready(lookup.as_mut().poll(cx))).await;
12033 assert!(
12034 initial.is_pending(),
12035 "a busy record must not hold or resume the task"
12036 );
12037 owner.retired_to = Some("20260101-000001-bbbb".into());
12038 owner.save_under(home.path()).unwrap();
12039 drop(lock);
12040 let result = tokio::time::timeout(Duration::from_secs(10), lookup)
12041 .await
12042 .expect("ownership lock released")
12043 .expect("read settled owner");
12044 assert!(
12045 result.is_none(),
12046 "the retired owner must not be resumed from a stale read"
12047 );
12048 }
12049
12050 #[tokio::test]
12051 async fn retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds() {
12052 if !crate::standalone_test(
12053 "daemon::tests::retained_public_owner_wins_over_an_existing_successor_and_unknown_history_holds",
12054 ) {
12055 return;
12056 }
12057 let home = tempfile::tempdir().unwrap();
12058 let mut owner = stopped_public_run(RunStatus::Blocked);
12059 let repo = tempfile::tempdir().unwrap();
12060 crate::git::git(repo.path(), &["init"]).await.unwrap();
12061 owner.repo = crate::git::toplevel(repo.path()).await.unwrap();
12062 owner.id = "20260101-000000-aaaa".into();
12063 owner.save_under(home.path()).unwrap();
12064 let mut successor = run_state(RunStatus::Blocked);
12065 successor.id = "20260101-000001-bbbb".into();
12066 successor.save_under(home.path()).unwrap();
12067 let runs = vec![owner.id.clone(), successor.id.clone()];
12068 assert_eq!(
12069 public_resume_owner(home.path(), &runs, &owner.repo)
12070 .await
12071 .unwrap()
12072 .unwrap()
12073 .id,
12074 owner.id
12075 );
12076 let subdir = owner.repo.join("nested");
12077 std::fs::create_dir(&subdir).unwrap();
12078 assert_eq!(
12079 public_resume_owner(home.path(), &runs, &subdir)
12080 .await
12081 .unwrap()
12082 .unwrap()
12083 .id,
12084 owner.id
12085 );
12086 #[cfg(unix)]
12087 {
12088 let alias = home.path().join("repo-alias");
12089 std::os::unix::fs::symlink(&owner.repo, &alias).unwrap();
12090 assert_eq!(
12091 public_resume_owner(home.path(), &runs, &alias.join("nested"))
12092 .await
12093 .unwrap()
12094 .unwrap()
12095 .id,
12096 owner.id
12097 );
12098 }
12099 let different = tempfile::tempdir().unwrap();
12100 crate::git::git(different.path(), &["init"]).await.unwrap();
12101 assert!(
12102 public_resume_owner(home.path(), &runs, different.path())
12103 .await
12104 .is_err()
12105 );
12106 std::fs::remove_file(
12107 home.path()
12108 .join("runs")
12109 .join(&successor.id)
12110 .join("run.json"),
12111 )
12112 .unwrap();
12113 assert!(
12114 public_resume_owner(home.path(), &runs, &owner.repo)
12115 .await
12116 .is_err()
12117 );
12118 }
12119
12120 #[test]
12121 fn an_ordinary_release_keeps_a_resumable_run_available() {
12122 let mut released = task();
12123 released.start("stalled-run".to_owned());
12124 released.release();
12125 let unfinished = (!released.fresh_start)
12126 .then(|| Some("stalled-run".to_owned()))
12127 .flatten();
12128 assert_eq!(
12129 choose_starter(None, false, unfinished.as_deref()),
12130 Starter::Resume("stalled-run".to_owned()),
12131 "manual release must preserve the normal resume path"
12132 );
12133 }
12134
12135 #[test]
12136 fn resume_refusals_explain_admission_without_changing_task_or_run() {
12137 use crate::handover::{ResumeRefusal, resume_refusal_under};
12138 let home = tempfile::tempdir().unwrap();
12139 let mut exhausted = run_state(RunStatus::Blocked);
12140 exhausted.driver_exited = true;
12141 exhausted.config.graph.review_rounds = 3;
12142 exhausted.reviews = (1..=3).map(review_round).collect();
12143 for refusal in [
12144 ResumeRefusal::ReviewBudget { used: 3, limit: 3 },
12145 ResumeRefusal::Status(RunStatus::Ready),
12146 ResumeRefusal::Released("20261022-000000-aaaa".into()),
12147 ResumeRefusal::Retired("retirement-successor".into()),
12148 ResumeRefusal::OwnershipTransferred("20261022-000000-bbbb".into()),
12149 ] {
12150 let mut state = exhausted.clone();
12151 let expected = match refusal {
12152 ResumeRefusal::Status(_) => {
12153 state.status = RunStatus::Ready;
12154 "status `ready`"
12155 }
12156 ResumeRefusal::Released(ref target) => {
12157 state.released_to = Some(target.clone());
12158 "execution was released"
12159 }
12160 ResumeRefusal::Retired(ref target) => {
12161 state.retired_to = Some(target.clone());
12162 "execution was retired"
12163 }
12164 ResumeRefusal::OwnershipTransferred(ref target) => {
12165 state.pr_taken_over_by = Some(target.clone());
12166 "PR ownership was transferred"
12167 }
12168 _ => "review budget exhausted (3/3 rounds used)",
12169 };
12170 assert_eq!(
12171 resume_refusal_under(&state, Some(home.path())),
12172 Some(refusal.clone())
12173 );
12174 assert_eq!(
12175 refusal.successor(),
12176 state
12177 .released_to
12178 .as_deref()
12179 .or(state.retired_to.as_deref())
12180 .or(state.pr_taken_over_by.as_deref())
12181 );
12182 state.save_under(home.path()).unwrap();
12183 let queue = Queue::at(home.path().join("queue"));
12184 let mut task = Task::new(
12185 "retained implementation".into(),
12186 state.instruction.clone(),
12187 state.repo.clone(),
12188 Source::Agent {
12189 run: "talk".into(),
12190 node: crate::queue::CHAT_NODE.into(),
12191 },
12192 );
12193 task.runs.push(state.id.clone());
12194 task.hold_machine(Some("stopped".into()));
12195 queue.put(&mut task).unwrap();
12196 let before_task = serde_json::to_value(queue.get(&task.id).unwrap()).unwrap();
12197 let before_run = serde_json::to_value(&state).unwrap();
12198 let error = crate::chat_control::resume(&queue, home.path(), &state.id)
12199 .unwrap_err()
12200 .to_string();
12201 assert!(error.contains(expected), "{error}");
12202 assert!(!error.contains("only a stalled or blocked"), "{error}");
12203 if matches!(refusal, ResumeRefusal::Status(_)) {
12204 assert!(
12205 error.contains("is final; there is nothing left to resume"),
12206 "{error}"
12207 );
12208 }
12209 if matches!(refusal, ResumeRefusal::ReviewBudget { .. }) {
12210 assert!(error.contains("magi review <saved-branch> --task <task-id> --merge none"));
12211 assert!(error.contains("review-only successor with a new review budget"));
12212 assert!(error.contains("does not resume this run"));
12213 }
12214 for target in [
12215 &state.released_to,
12216 &state.retired_to,
12217 &state.pr_taken_over_by,
12218 ]
12219 .into_iter()
12220 .flatten()
12221 {
12222 let short = if state.retired_to.as_ref() == Some(target) {
12223 target.as_str()
12224 } else {
12225 crate::run::short_of(target)
12226 };
12227 assert!(
12228 error.contains(&format!("run {short}; inspect `magi show {short}`")),
12229 "{error}"
12230 );
12231 if short != target {
12232 assert!(!error.contains(target), "full successor id leaked: {error}");
12233 }
12234 }
12235 assert_eq!(
12236 serde_json::to_value(queue.get(&task.id).unwrap()).unwrap(),
12237 before_task
12238 );
12239 assert_eq!(
12240 serde_json::to_value(RunState::load_under(&state.id, home.path()).unwrap())
12241 .unwrap(),
12242 before_run
12243 );
12244 let question = action_question(&state.id, resume_action(&state.id));
12245 for phrases in [&PHRASES_EN, &PHRASES_JA] {
12246 let decision = decide_action(home.path(), &task, &question, phrases, |id| {
12247 RunState::load_under(id, home.path())
12248 });
12249 let ActionDecision::Refuse(detail) = decision else {
12250 panic!(
12251 "refusal expected: decision={decision:?}, refusal={refusal:?}, run={}, home={}",
12252 state.id,
12253 home.path().display()
12254 );
12255 };
12256 assert!(detail.contains(expected), "{detail}");
12257 }
12258 }
12259 exhausted.status = RunStatus::Ready;
12261 exhausted.released_to = Some("released".into());
12262 exhausted.retired_to = Some("retired".into());
12263 exhausted.pr_taken_over_by = Some("owner".into());
12264 let reason = resume_refusal_under(&exhausted, Some(home.path())).unwrap();
12265 assert_eq!(reason, ResumeRefusal::Status(RunStatus::Ready));
12266 let detail = reason.diagnostic(&exhausted);
12267 for target in ["released", "retired", "owner"] {
12268 assert!(detail.contains(&format!("magi show {target}")));
12269 }
12270 }
12271
12272 #[test]
12273 fn resume_action_ignores_question_checkpoints_in_another_home() {
12274 crate::run::pin_test_home();
12275 let home = tempfile::tempdir().unwrap();
12276 let mut state = run_state(RunStatus::Blocked);
12277 state.id = format!(
12280 "resume-context-{}",
12281 home.path().file_name().unwrap().to_str().unwrap()
12282 );
12283 state.driver_exited = true;
12284 state.config.graph.review_rounds = 1;
12285 state.reviews = vec![review_round(1)];
12286 state.save_under(home.path()).unwrap();
12287 let task = held_task_with(&state.id);
12288 let answer = action_question(&state.id, resume_action(&state.id));
12289 let mut checkpoint = approval_question(&state.id);
12290 let foreign = Questions::open();
12291 foreign.put(&mut checkpoint).unwrap();
12292 let refusal = crate::handover::resume_refusal_under(&state, Some(home.path()));
12293 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
12294 RunState::load_under(id, home.path())
12295 });
12296 let matching: Vec<_> = foreign
12297 .list()
12298 .into_iter()
12299 .filter(|q| q.run == state.id && q.status.open())
12300 .collect();
12301 std::fs::remove_file(foreign.path_of(&checkpoint.id)).unwrap();
12302 assert!(
12303 matches!(decision, ActionDecision::Refuse(_)),
12304 "decision={decision:?}, refusal={refusal:?}, run={}, home={}, foreign_home={}, matching={matching:?}",
12305 state.id,
12306 home.path().display(),
12307 crate::run::test_home().display()
12308 );
12309 assert_eq!(
12310 refusal,
12311 Some(crate::handover::ResumeRefusal::ReviewBudget { used: 1, limit: 1 })
12312 );
12313 let local = Questions::at(home.path().join("questions"));
12314 for status in [
12315 ask::QuestionStatus::Open,
12316 ask::QuestionStatus::Answered,
12317 ask::QuestionStatus::Abandoned,
12318 ] {
12319 let mut question = checkpoint.clone();
12320 match status {
12321 ask::QuestionStatus::Open => {}
12322 ask::QuestionStatus::Answered => {
12323 question
12324 .answer(ask::Answer::Choice(question.choices[0].clone()))
12325 .unwrap();
12326 }
12327 ask::QuestionStatus::Abandoned => question.abandon("fixture closed"),
12328 }
12329 local.put(&mut question).unwrap();
12330 let decision = decide_action(home.path(), &task, &answer, &PHRASES_EN, |id| {
12331 RunState::load_under(id, home.path())
12332 });
12333 if status.open() {
12334 assert_eq!(decision, ActionDecision::Resume(state.id.clone()));
12335 } else {
12336 assert!(
12337 matches!(decision, ActionDecision::Refuse(ref detail) if detail.contains("review budget exhausted")),
12338 "status={status:?}, decision={decision:?}"
12339 );
12340 }
12341 }
12342 }
12343
12344 #[test]
12345 fn resume_action_checks_the_driver_lease_in_the_supplied_home() {
12346 let home = tempfile::tempdir().unwrap();
12347 let mut state = run_state(RunStatus::Stalled);
12348 state.id = format!(
12349 "resume-driver-{}",
12350 home.path().file_name().unwrap().to_str().unwrap()
12351 );
12352 state.driver_claimed = Some(false);
12353 state.save_under(home.path()).unwrap();
12354 let task = held_task_with(&state.id);
12355 let question = action_question(&state.id, resume_action(&state.id));
12356 let decide = || {
12357 decide_action(home.path(), &task, &question, &PHRASES_EN, |id| {
12358 RunState::load_under(id, home.path())
12359 })
12360 };
12361 let lease = crate::driver::maintenance(home.path(), &state.id).unwrap();
12362 assert_eq!(state.liveness_under(home.path(), false), Liveness::Live);
12363 assert_eq!(
12364 decide(),
12365 ActionDecision::Refuse("run already has a live driver".into())
12366 );
12367 drop(lease);
12368 assert_eq!(decide(), ActionDecision::Resume(state.id.clone()));
12369 }
12370
12371 #[test]
12372 fn resume_checkpoint_admission_uses_the_supplied_question_store() {
12373 let home = tempfile::tempdir().unwrap();
12374 let mut state = run_state(RunStatus::Blocked);
12375 state.driver_exited = true;
12376 state.config.graph.review_rounds = 1;
12377 state.reviews = vec![review_round(1)];
12378 assert!(crate::handover::resume_refusal_under(&state, Some(home.path())).is_some());
12379 for checkpoint in ["publication", "approval", "armed", "landing", "question"] {
12380 let mut saved = state.clone();
12381 match checkpoint {
12382 "publication" => saved.pr_publication_pending = true,
12383 "approval" => {
12384 saved.land_approval = Some(crate::run::LandApproval {
12385 question: "approval".into(),
12386 head: "head".into(),
12387 })
12388 }
12389 "armed" => saved.land_armed_head = Some("head".into()),
12390 "landing" => saved.status = RunStatus::Landing,
12391 _ => {
12392 let mut question = ask::Question::new(
12393 saved.id.clone(),
12394 "review".into(),
12395 "review-1".into(),
12396 "continue?".into(),
12397 String::new(),
12398 vec!["continue".into()],
12399 );
12400 ask::Questions::at(home.path().join("questions"))
12401 .put(&mut question)
12402 .unwrap();
12403 }
12404 }
12405 saved.save_under(home.path()).unwrap();
12406 assert!(
12407 crate::handover::resume_refusal_under(&saved, Some(home.path())).is_none(),
12408 "{checkpoint}"
12409 );
12410 assert!(
12411 crate::resume::validate(home.path(), &saved.id).is_ok(),
12412 "{checkpoint}"
12413 );
12414 }
12415 }
12416
12417 #[test]
12418 fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
12419 let mut state = run_state(RunStatus::Blocked);
12420 state.config.graph.review_rounds = 3;
12421 state.reviews = vec![review_round(1), review_round(2), review_round(3)];
12422 assert!(exhausted_review_budget(&state));
12423
12424 state.reviews.pop();
12426 assert!(!exhausted_review_budget(&state));
12427
12428 let mut stalled = run_state(RunStatus::Stalled);
12431 stalled.config.graph.review_rounds = 1;
12432 stalled.reviews = vec![review_round(1)];
12433 assert!(!exhausted_review_budget(&stalled));
12434 }
12435
12436 fn review_round(round: usize) -> crate::run::ReviewRound {
12437 crate::run::ReviewRound {
12438 round,
12439 head: "deadbeef".to_owned(),
12440 verified_head: None,
12441 verified_at: None,
12442 reviews: Vec::new(),
12443 e2e: Vec::new(),
12444 verify_retried: false,
12445 e2e_deferred: false,
12446 e2e_defer_reason: None,
12447 fix: None,
12448 blocking: 0,
12449 answered: 1,
12450 expected: 1,
12451 clean: false,
12452 progressed: true,
12453 vote_split: false,
12454 reconsideration: Vec::new(),
12455 verdict: None,
12456 }
12457 }
12458
12459 #[test]
12460 fn awaiting_resume_is_a_failed_task_whose_last_run_parked_and_can_be_resumed() {
12461 let mut run = RunState::new(
12462 PathBuf::from("/repo"),
12463 "main".to_owned(),
12464 "abc1234def".to_owned(),
12465 "add retries".to_owned(),
12466 Config::default(),
12467 );
12468 run.status = RunStatus::Judging;
12469 run.parked = true;
12470 let mut task = Task::new(
12471 "add retries".to_owned(),
12472 "add retries".to_owned(),
12473 PathBuf::from("/repo"),
12474 crate::queue::Source::Human,
12475 );
12476 task.status = TaskStatus::Failed;
12477 task.runs = vec![run.id.clone()];
12478 let with = |t: &Task, r: &RunState| awaiting_resume_with(t, |_| Ok(r.clone()));
12479 assert!(with(&task, &run), "parked after judging is the case");
12480 let mut parked_task = task.clone();
12481 parked_task.status = TaskStatus::Parked;
12482 assert!(with(&parked_task, &run), "the Parked status is the case");
12483
12484 let mut not_parked = run.clone();
12485 not_parked.parked = false;
12486 not_parked.status = RunStatus::Stalled;
12487 assert!(!with(&task, ¬_parked), "a stall is the conductor's");
12488
12489 let mut fresh = task.clone();
12490 fresh.fresh_start = true;
12491 assert!(!with(&fresh, &run), "a requeue asked for a new competition");
12492
12493 let mut review = task.clone();
12494 review.review_branch = Some("magi/x/A".to_owned());
12495 assert!(!with(&review, &run), "review is ranked before resume");
12496
12497 let mut held = task.clone();
12498 held.status = TaskStatus::Held;
12499 assert!(!with(&held, &run), "a hold stays visible to the conductor");
12500
12501 let mut released = run.clone();
12502 released.released_to = Some("20260901-000000-new1".to_owned());
12503 assert!(!with(&task, &released), "nothing left to resume into");
12504
12505 assert!(!awaiting_resume_with(&task, |_| anyhow::bail!(
12506 "unreadable"
12507 )));
12508 }
12509
12510 #[test]
12511 fn unfinished_run_never_offers_a_run_whose_worktree_was_released() {
12512 let mut released = RunState::new(
12513 PathBuf::from("/repo"),
12514 "main".to_owned(),
12515 "abc1234def".to_owned(),
12516 "add retries".to_owned(),
12517 Config::default(),
12518 );
12519 released.status = RunStatus::Blocked;
12520 assert_eq!(
12521 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12522 Some(released.id.clone())
12523 );
12524 released.released_to = Some("20260901-000000-new1".to_owned());
12525 assert_eq!(
12526 unfinished_run_with(&[released.id.clone()], "t", |_| Ok(released.clone())),
12527 None,
12528 "there is nothing left to resume it into"
12529 );
12530 }
12531
12532 #[test]
12533 fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
12534 let mut exhausted = RunState::new(
12543 PathBuf::from("/repo"),
12544 "main".to_owned(),
12545 "abc1234def".to_owned(),
12546 "add retries".to_owned(),
12547 Config::default(),
12548 );
12549 exhausted.status = RunStatus::Blocked;
12550 exhausted.config.graph.review_rounds = 1;
12551 exhausted.reviews = vec![review_round(1)];
12552
12553 assert_eq!(
12554 unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
12555 None,
12556 "an exhausted `Blocked` run must not be offered as resumable"
12557 );
12558
12559 let mut has_budget_left = RunState::new(
12562 PathBuf::from("/repo"),
12563 "main".to_owned(),
12564 "abc1234def".to_owned(),
12565 "add retries".to_owned(),
12566 Config::default(),
12567 );
12568 has_budget_left.status = RunStatus::Blocked;
12569 has_budget_left.config.graph.review_rounds = 3;
12570 has_budget_left.reviews = vec![review_round(1)];
12571
12572 assert_eq!(
12573 unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
12574 Ok(has_budget_left.clone())
12575 }),
12576 Some(has_budget_left.id.clone())
12577 );
12578 }
12579
12580 #[test]
12581 fn unfinished_run_resumes_a_run_whose_task_head_publication_is_pending() {
12582 let mut run = RunState::new(
12583 PathBuf::from("/repo"),
12584 "main".to_owned(),
12585 "abc1234def".to_owned(),
12586 "add retries".to_owned(),
12587 Config::default(),
12588 );
12589 run.status = RunStatus::Blocked;
12590 run.config.graph.review_rounds = 1;
12591 run.reviews = vec![review_round(1)];
12592 run.pr_publication_pending = true;
12593 assert_eq!(
12594 unfinished_run_with(&[run.id.clone()], "t", |_| Ok(run.clone())),
12595 Some(run.id.clone()),
12596 "a spent review budget must not turn a failed publication into a re-competition"
12597 );
12598 }
12599
12600 #[test]
12601 fn retained_attempt_bound_survives_changed_bases_releases_and_duplicate_history() {
12602 let mut task = Task::new(
12603 "recover work".into(),
12604 "recover work".into(),
12605 PathBuf::from("/repo"),
12606 crate::queue::Source::Human,
12607 );
12608 let mut history = std::collections::BTreeMap::new();
12609 for index in 0..8 {
12610 let mut run = run_state(RunStatus::Blocked);
12611 run.id = format!("20260101-000000-{index:04}");
12612 run.base_commit = format!("upstream-{index}");
12613 if index > 0 {
12615 run.candidates.clear();
12616 }
12617 task.runs.extend([run.id.clone(), run.id.clone()]);
12618 history.insert(run.id.clone(), run);
12619 task.requeue();
12620 }
12621 let check = |task: &Task, starter: &Starter, limit| {
12622 retained_attempt_hold(task, starter, limit, |id| Ok(history[id].clone()))
12623 };
12624 assert!(
12625 check(&task, &Starter::Start, 8)
12626 .unwrap()
12627 .contains("8 unretired")
12628 );
12629 assert!(check(&task, &Starter::Start, 9).is_none());
12630 assert!(check(&task, &Starter::Review("existing".into()), 8).is_some());
12631 task.release_operator("owner", None);
12632 assert_eq!(task.attempts, 0);
12633 let restored: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
12634 assert!(check(&restored, &Starter::Start, 8).is_some());
12635 assert!(check(&restored, &Starter::Resume(task.runs[0].clone()), 8).is_none());
12636 assert!(check(&restored, &Starter::Start, 0).is_none());
12637 }
12638
12639 #[test]
12640 fn retained_attempt_bound_includes_failed_startups_without_execution_history() {
12641 let tmp = tempfile::tempdir().unwrap();
12642 let mut task = Task::new(
12643 "recover selected PR".into(),
12644 "preserve implementation".into(),
12645 tmp.path().join("repo"),
12646 crate::queue::Source::Human,
12647 );
12648 let mut failed = run_state(RunStatus::Failed);
12649 failed.id = "20260101-000000-fail".into();
12650 failed.candidates.clear();
12651 failed.prep_complete = Some(false);
12652 failed.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12653 failed.startup = Some(crate::run::Startup {
12654 kind: "pr-recovery".into(),
12655 branch: Some("selected-public-head".into()),
12656 stage: "worktree-setup".into(),
12657 complete: false,
12658 failure: Some("setup modified a tracked file and failed".into()),
12659 });
12660 let worktree = failed.worktree_root().join("under-review");
12663 std::fs::create_dir_all(&worktree).unwrap();
12664 let retained_edit = worktree.join("implementation");
12665 std::fs::write(&retained_edit, "retained setup edit").unwrap();
12666 task.record_startup_failure(crate::run::StartupFailure {
12667 run: failed.id.clone(),
12668 stage: "worktree-setup".into(),
12669 });
12670 let task: Task = serde_json::from_slice(&serde_json::to_vec(&task).unwrap()).unwrap();
12671 assert!(task.runs.is_empty() && task.passes.is_empty());
12672 assert_eq!(task.attempts, 0);
12673 let starter = Starter::Review("selected-public-head".into());
12674 assert!(
12675 retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone()))
12676 .unwrap()
12677 .contains("1 unretired")
12678 );
12679 assert!(retained_attempt_hold(&task, &starter, 2, |_| Ok(failed.clone())).is_none());
12680 assert!(
12681 retained_attempt_hold(&task, &starter, 2, |_| anyhow::bail!("unreadable"))
12682 .unwrap()
12683 .contains("Cannot verify")
12684 );
12685 let mut mixed = task.clone();
12687 mixed.runs.push(failed.id.clone());
12688 mixed
12689 .startup_failures
12690 .push(task.startup_failures[0].clone());
12691 assert!(retained_attempt_hold(&mixed, &starter, 2, |_| Ok(failed.clone())).is_none());
12692 failed.retired_to = Some("verified-recovery-owner".into());
12693 assert!(retained_attempt_hold(&task, &starter, 1, |_| Ok(failed.clone())).is_none());
12694 assert!(
12695 retained_attempt_hold(&task, &Starter::Resume(failed.id.clone()), 1, |_| {
12696 panic!("resuming the existing owner must not allocate a new attempt")
12697 })
12698 .is_none()
12699 );
12700 assert_eq!(
12701 std::fs::read_to_string(retained_edit).unwrap(),
12702 "retained setup edit"
12703 );
12704 assert!(task.runs.is_empty() && task.passes.is_empty());
12705 assert_eq!(task.attempts, 0);
12706 }
12707
12708 #[test]
12709 fn retained_attempt_bound_requires_verified_retirement_not_completion_labels() {
12710 let mut task = Task::new(
12711 "recover work".into(),
12712 "recover work".into(),
12713 PathBuf::from("/repo"),
12714 crate::queue::Source::Human,
12715 );
12716 let mut run = run_state(RunStatus::Superseded);
12717 run.candidates.clear();
12718 task.runs.push(run.id.clone());
12719 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_some());
12720 run.retired_to = Some("authority".into());
12721 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
12722 run.retired_to = None;
12723 run.released_to = Some("authority".into());
12724 assert!(retained_attempt_hold(&task, &Starter::Start, 1, |_| Ok(run.clone())).is_none());
12725 assert!(
12726 retained_attempt_hold(&task, &Starter::Start, 8, |_| anyhow::bail!("unreadable"))
12727 .unwrap()
12728 .contains("Cannot verify")
12729 );
12730 assert!(
12732 retained_attempt_hold(&task, &Starter::Resume(run.id), 1, |_| {
12733 panic!("resume must not load historical attempts")
12734 })
12735 .is_none()
12736 );
12737 }
12738
12739 #[test]
12740 fn retired_resume_requires_a_valid_same_repository_authority() {
12741 let mut source = run_state(RunStatus::Blocked);
12742 let mut empty = run_state(RunStatus::Blocked);
12743 empty.retired_to = Some(source.id.clone());
12744 let runs = vec![source.id.clone(), empty.id.clone()];
12745 let resolve = |source: &RunState, empty: &RunState| {
12746 unfinished_run_with(&runs, "fixture", |id| {
12747 if id == source.id {
12748 Ok(source.clone())
12749 } else if id == empty.id {
12750 Ok(empty.clone())
12751 } else {
12752 bail!("unreadable authority")
12753 }
12754 })
12755 };
12756 assert_eq!(resolve(&source, &empty), Some(source.id.clone()));
12757 assert_eq!(
12758 choose_starter(None, false, resolve(&source, &empty).as_deref()),
12759 Starter::Resume(source.id.clone())
12760 );
12761 source.retired_to = Some(empty.id.clone());
12762 assert!(
12763 resolve(&source, &empty).is_none(),
12764 "cycles do not revive obsolete runs"
12765 );
12766 source.retired_to = None;
12767 source.repo = PathBuf::from("/another-repository");
12768 assert!(resolve(&source, &empty).is_none());
12769 source.repo = empty.repo.clone();
12770 source.released_to = Some("later".into());
12771 assert!(resolve(&source, &empty).is_none());
12772 empty.retired_to = Some("unlisted-run".into());
12773 assert!(resolve(&source, &empty).is_none());
12774 }
12775
12776 #[test]
12777 fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
12778 let mut older_stalled = RunState::new(
12786 PathBuf::from("/repo"),
12787 "main".to_owned(),
12788 "abc1234def".to_owned(),
12789 "add retries".to_owned(),
12790 Config::default(),
12791 );
12792 older_stalled.status = RunStatus::Stalled;
12793
12794 let mut newest_exhausted = RunState::new(
12795 PathBuf::from("/repo"),
12796 "main".to_owned(),
12797 "abc1234def".to_owned(),
12798 "add retries".to_owned(),
12799 Config::default(),
12800 );
12801 newest_exhausted.status = RunStatus::Blocked;
12802 newest_exhausted.config.graph.review_rounds = 1;
12803 newest_exhausted.reviews = vec![review_round(1)];
12804
12805 assert_eq!(
12806 unfinished_run_with(
12807 &[older_stalled.id.clone(), newest_exhausted.id.clone()],
12808 "t",
12809 |_| Ok(newest_exhausted.clone())
12810 ),
12811 None,
12812 "the newest run is exhausted, so nothing here is worth resuming - \
12813 least of all the older, already-superseded run"
12814 );
12815 }
12816
12817 #[test]
12818 fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
12819 assert_eq!(
12820 unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
12821 Err(anyhow::anyhow!("fixture is absent"))
12822 }),
12823 None
12824 );
12825 }
12826
12827 fn action_question(run: &str, action: ask::ChoiceAction) -> ask::Question {
12828 let mut q = ask::Question::new(
12829 run.to_owned(),
12830 "implement".to_owned(),
12831 "impl-A".to_owned(),
12832 "continue?".to_owned(),
12833 String::new(),
12834 vec!["resume で続行する".to_owned(), "other".to_owned()],
12835 );
12836 q.actions.insert("resume で続行する".to_owned(), action);
12837 q.answer(ask::Answer::Choice("resume で続行する".to_owned()))
12838 .unwrap();
12839 q
12840 }
12841
12842 fn held_task_with(run: &str) -> Task {
12843 let mut t = task();
12844 t.runs = vec![run.to_owned()];
12845 t.hold_machine(Some("waiting for magi resume to be executed".to_owned()));
12846 t
12847 }
12848
12849 fn resume_action(run: &str) -> ask::ChoiceAction {
12850 ask::ChoiceAction::Resume { run: run.into() }
12851 }
12852
12853 #[test]
12854 fn explicitly_resumed_older_run_remains_the_release_and_question_target() {
12855 let home = crate::run::test_home();
12856 for parked in [false, true] {
12857 let mut t = task();
12858 t.start("A".into());
12859 t.start("B".into());
12860 assert_eq!(t.runs, vec!["A", "B"]);
12861 t.start("A".into());
12862 let mut resumed = run_state(RunStatus::Blocked);
12863 resumed.id = "A".into();
12864 resumed.parked = parked;
12865 resumed.driver_exited = true;
12866 t.hold_machine(Some("resumed run stopped".into()));
12867 let q = action_question("A", resume_action("A"));
12868 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
12869 assert_eq!(
12870 decide_action(&home, &t, &q, &PHRASES_EN, |id| {
12871 assert_eq!(id, "A");
12872 Ok(resumed.clone())
12873 }),
12874 ActionDecision::Resume("A".into())
12875 );
12876 assert_eq!(
12877 action_standing(&t, &action_question("B", resume_action("B"))),
12878 ActionStanding::Stale
12879 );
12880 t.release_operator("cli", None);
12881 let unfinished = unfinished_run_with(&t.runs, t.short(), |id| {
12882 assert_eq!(id, "A");
12883 Ok(resumed.clone())
12884 });
12885 assert_eq!(
12886 choose_starter(None, false, unfinished.as_deref()),
12887 Starter::Resume("A".into())
12888 );
12889 assert_eq!(t.runs, vec!["B", "A"]);
12890 }
12891 }
12892
12893 #[test]
12894 fn merge_repair_dispatch_refuses_obsolete_bindings_and_unsupported_owners() {
12895 let home = crate::run::test_home();
12896 let mut state = crate::land::tests::repair_state(true, true);
12897 state.status = RunStatus::Landing;
12898 state.parked = true;
12899 state.driver_exited = true;
12900 let mut q = ask::Question::new(
12901 state.id.clone(),
12902 land::APPROVAL_NODE.into(),
12903 "land".into(),
12904 "Merge?".into(),
12905 String::new(),
12906 vec![land::APPROVE.into(), land::HOLD.into()],
12907 );
12908 q.id = state.id.clone();
12909 state.land_approval = Some(crate::run::LandApproval {
12910 question: q.id.clone(),
12911 head: "current-head".into(),
12912 });
12913 assert!(land::upgrade_repair_choice(&mut q, &state));
12914 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12915 let mut task = held_task_with(&state.id);
12916 task.status = TaskStatus::Parked;
12917 assert_eq!(
12918 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12919 ActionDecision::Resume(state.id.clone())
12920 );
12921 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
12922 q.answered_at = Some("2026-01-01T00:00:20Z".parse().unwrap());
12923 task.operator_decided_at = Some("2026-01-01T00:00:10Z".parse().unwrap());
12924 assert_eq!(
12925 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(state.clone())),
12926 ActionDecision::Resume(state.id.clone()),
12927 "a repair answer newer than the manual hold resumes the existing owner"
12928 );
12929 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12930 for answered_at in [q.answered_at, None] {
12931 let mut older = q.clone();
12932 older.answered_at = answered_at;
12933 let refused = decide_action(&home, &task, &older, &PHRASES_EN, |_| {
12934 panic!("a later manual hold must prevent dispatch")
12935 });
12936 let ActionDecision::Refuse(reason) = refused else {
12937 panic!("expected an actionable refusal, got {refused:?}");
12938 };
12939 assert!(reason.contains("later operator decision"));
12940 assert!(reason.contains(&format!("magi task release {}", task.id)));
12941 }
12942 task.operator_decided_at = None;
12943 let mut superseded = state.clone();
12944 superseded.land_approval.as_mut().unwrap().question = "fresh-decision".into();
12945 assert_eq!(
12946 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(superseded)),
12947 ActionDecision::Stale
12948 );
12949 for case in 1..4 {
12950 let mut changed = state.clone();
12951 match case {
12952 1 => changed.status = RunStatus::Blocked,
12953 2 => changed.released_to = Some("new-owner".into()),
12954 _ => changed.land_approval.as_mut().unwrap().head.clear(),
12955 }
12956 assert!(matches!(
12957 decide_action(&home, &task, &q, &PHRASES_EN, |_| Ok(changed)),
12958 ActionDecision::Refuse(_)
12959 ));
12960 }
12961 assert!(matches!(
12962 decide_action(&home, &task, &q, &PHRASES_EN, |_| anyhow::bail!(
12963 "missing owner"
12964 )),
12965 ActionDecision::Refuse(_)
12966 ));
12967 task.runs.push("replacement".into());
12968 task.operator_decided_at = Some("2026-01-01T00:00:30Z".parse().unwrap());
12969 assert_eq!(
12970 decide_action(&home, &task, &q, &PHRASES_EN, |_| panic!(
12971 "old owner must not load"
12972 )),
12973 ActionDecision::Stale
12974 );
12975 }
12976
12977 #[test]
12978 fn merge_repair_dispatch_obeys_capacity_and_quota_while_hold_stays_decision_only() {
12979 let mut state = crate::land::tests::repair_state(true, true);
12980 state.status = RunStatus::Landing;
12981 state.parked = true;
12982 state.driver_exited = true;
12983 let mut q = ask::Question::new(
12984 state.id.clone(),
12985 land::APPROVAL_NODE.into(),
12986 "land".into(),
12987 "Merge?".into(),
12988 String::new(),
12989 vec![land::APPROVE.into(), land::HOLD.into()],
12990 );
12991 q.id = state.id.clone();
12992 state.land_approval = Some(crate::run::LandApproval {
12993 question: q.id.clone(),
12994 head: "current-head".into(),
12995 });
12996 state.save().unwrap();
12997 land::upgrade_repair_choice(&mut q, &state);
12998 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
12999 Questions::open().put(&mut q).unwrap();
13000 let tmp = tempfile::tempdir().unwrap();
13001 let queue = Queue::at(tmp.path().join("queue"));
13002 let mut task = held_task_with(&state.id);
13003 task.status = TaskStatus::Parked;
13004 queue.put(&mut task).unwrap();
13005 let empty = Arc::new(tokio::sync::Semaphore::new(0));
13006 let free = Arc::new(tokio::sync::Semaphore::new(1));
13007 assert!(claim_dispatch(&queue, &task, false, &empty, &free).is_none());
13008 assert!(claim_dispatch(&queue, &task, true, &free, &free).is_none());
13009 let claim = claim_dispatch(&queue, &task, false, &free, &free).unwrap();
13010 assert!(claim.permit.is_some());
13011 assert!(claim_dispatch(&queue, &task, false, &free, &free).is_none());
13012 drop(claim);
13013 Questions::open()
13014 .update(&q.id, |q| {
13015 q.answer = Some(ask::Answer::Choice(land::HOLD.into()));
13016 Ok(())
13017 })
13018 .unwrap();
13019 let claim = claim_dispatch(&queue, &task, true, &empty, &empty).unwrap();
13020 assert!(claim.permit.is_none());
13021 assert!(RunState::load(&state.id).unwrap().merge_repairs.is_empty());
13022 }
13023
13024 #[test]
13025 fn decide_action_resumes_only_the_latest_resumable_run() {
13026 let home = crate::run::test_home();
13027 let t = held_task_with("r1");
13028 let q = action_question("r1", resume_action("r1"));
13029 let load = |s: RunState| move |_: &str| Ok(s);
13030 assert_eq!(
13031 decide_action(
13032 &home,
13033 &t,
13034 &q,
13035 &PHRASES_EN,
13036 load(run_state(RunStatus::Blocked))
13037 ),
13038 ActionDecision::Resume("r1".into())
13039 );
13040 let q_other = action_question("r1", resume_action("r0"));
13042 assert!(matches!(
13043 decide_action(
13044 &home,
13045 &t,
13046 &q_other,
13047 &PHRASES_EN,
13048 load(run_state(RunStatus::Blocked))
13049 ),
13050 ActionDecision::Refuse(_)
13051 ));
13052 assert!(matches!(
13054 decide_action(
13055 &home,
13056 &t,
13057 &q,
13058 &PHRASES_EN,
13059 load(run_state(RunStatus::Ready))
13060 ),
13061 ActionDecision::Refuse(_)
13062 ));
13063 let mut released = run_state(RunStatus::Blocked);
13065 released.released_to = Some("elsewhere".into());
13066 assert!(matches!(
13067 decide_action(&home, &t, &q, &PHRASES_EN, load(released)),
13068 ActionDecision::Refuse(_)
13069 ));
13070 assert!(matches!(
13072 decide_action(&home, &t, &q, &PHRASES_EN, |_: &str| bail!("gone")),
13073 ActionDecision::Refuse(_)
13074 ));
13075 }
13076
13077 #[test]
13078 fn decide_action_ignores_a_question_about_an_earlier_run() {
13079 let home = crate::run::test_home();
13080 let mut t = held_task_with("r1");
13081 t.runs.push("r2".to_owned());
13082 let never = |_: &str| -> Result<RunState> { bail!("not read") };
13083 assert_eq!(
13084 decide_action(
13085 &home,
13086 &t,
13087 &action_question("r1", ask::ChoiceAction::Done),
13088 &PHRASES_EN,
13089 never
13090 ),
13091 ActionDecision::Stale
13092 );
13093 }
13094
13095 #[test]
13096 fn decide_action_maps_requeue_and_done_and_never_acts_twice() {
13097 let home = crate::run::test_home();
13098 let mut t = held_task_with("r1");
13099 let never = |_: &str| -> Result<RunState> { bail!("not read") };
13100 assert_eq!(
13101 decide_action(
13102 &home,
13103 &t,
13104 &action_question("r1", ask::ChoiceAction::Requeue),
13105 &PHRASES_EN,
13106 never
13107 ),
13108 ActionDecision::Requeue
13109 );
13110 let done_q = action_question("r1", ask::ChoiceAction::Done);
13111 assert_eq!(
13112 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
13113 ActionDecision::Done
13114 );
13115 t.mark_action_applied(&done_q.id);
13116 assert_eq!(
13117 decide_action(&home, &t, &done_q, &PHRASES_EN, never),
13118 ActionDecision::Skip
13119 );
13120
13121 let mut plain = action_question("r1", ask::ChoiceAction::Done);
13123 plain.actions.clear();
13124 assert_eq!(
13125 decide_action(&home, &held_task_with("r1"), &plain, &PHRASES_EN, never),
13126 ActionDecision::Skip
13127 );
13128 let mut running = held_task_with("r1");
13130 running.status = TaskStatus::Running;
13131 assert_eq!(
13132 decide_action(
13133 &home,
13134 &running,
13135 &action_question("r1", ask::ChoiceAction::Done),
13136 &PHRASES_EN,
13137 never
13138 ),
13139 ActionDecision::Skip
13140 );
13141 }
13142
13143 #[test]
13144 fn merge_repair_rejected_after_manual_hold_keeps_the_conductor_guard() {
13145 crate::run::pin_test_home();
13146 for legacy in [false, true] {
13147 let dir = tempfile::tempdir().unwrap();
13148 let queue = Queue::at(dir.path().join("queue"));
13149 let questions = Questions::at(dir.path().join("questions"));
13150 let run = "20260101-000000-hold";
13151 let mut task = held_task_with(run);
13152 task.status = TaskStatus::Parked;
13153 let mut q = ask::Question::new(
13154 run.into(),
13155 land::APPROVAL_NODE.into(),
13156 "land".into(),
13157 "Repair?".into(),
13158 String::new(),
13159 vec![land::REPAIR.into()],
13160 );
13161 q.actions.insert(land::REPAIR.into(), resume_action(run));
13162 q.answer(ask::Answer::Choice(land::REPAIR.into())).unwrap();
13163 q.asked_at = "2026-01-01T00:00:00Z".parse().unwrap();
13165 q.answered_at = Some("2026-01-01T00:00:01Z".parse().unwrap());
13166 questions.put(&mut q).unwrap();
13167 task.hold_manual(Some("Keep this PR paused".into()));
13168 if legacy {
13169 task.hold_source = None;
13170 }
13171 queue.put(&mut task).unwrap();
13172
13173 apply_task_choice_actions(&queue, &questions, dir.path());
13174 let after = queue.get(&task.id).unwrap();
13175 assert!(after.operator_held());
13176 assert_eq!(after.hold_source, task.hold_source);
13177 assert_eq!(after.operator_decided_at, task.operator_decided_at);
13178 assert_eq!(after.runs, task.runs);
13179 assert_eq!(after.instruction, task.instruction);
13180 assert!(after.resume_override.is_none());
13181 assert!(after.action_applied(&q.id));
13182 let reason = after.hold_reason.as_deref().unwrap();
13183 assert!(reason.starts_with("Keep this PR paused"));
13184 assert!(reason.contains("later operator decision"));
13185 assert!(reason.contains(&format!("magi task release {}", task.id)));
13186
13187 for recovery in [
13188 crate::conduct::Recovery::Review,
13189 crate::conduct::Recovery::Requeue,
13190 ] {
13191 crate::conduct::apply(
13192 &queue,
13193 &questions,
13194 &crate::conduct::Verdict {
13195 decisions: vec![crate::conduct::Decision {
13196 id: task.id.clone(),
13197 recovery: Some(recovery),
13198 ..Default::default()
13199 }],
13200 },
13201 )
13202 .unwrap();
13203 }
13204 apply_task_choice_actions(&queue, &questions, dir.path());
13207 let still_held = queue.get(&task.id).unwrap();
13208 assert!(still_held.operator_held());
13209 assert_eq!(still_held.hold_reason, after.hold_reason);
13210 assert_eq!(still_held.runs, task.runs);
13211 assert!(still_held.review_branch.is_none());
13212 assert!(!still_held.fresh_start);
13213 assert!(queue.next_runnable().is_none());
13214 }
13215 }
13216
13217 #[test]
13218 fn conductor_driver_choices_apply_from_both_queued_and_held_tasks() {
13219 for held in [false, true] {
13220 for retry in [false, true] {
13221 let dir = tempfile::tempdir().unwrap();
13222 let queue = Queue::at(dir.path().join("queue"));
13223 let questions = Questions::at(dir.path().join("questions"));
13224 let home = dir.path().join("home");
13225 let mut t = task();
13226 if held {
13227 t.hold_machine(Some("driver state unknown".into()));
13228 }
13229 queue.put(&mut t).unwrap();
13230 let verdict = serde_json::from_value(serde_json::json!({
13231 "decisions": [{
13232 "id": t.id,
13233 "question": "Driver state unknown; retry the checks or hold?",
13234 "choices": ["daemon: recheck and retry", "daemon: hold the task"],
13235 "actions": {
13236 "daemon: recheck and retry": {"do": "requeue"},
13237 "daemon: hold the task": {"do": "hold"}
13238 }
13239 }]
13240 }))
13241 .unwrap();
13242 crate::conduct::apply(&queue, &questions, &verdict).unwrap();
13243 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13244 let mut q = questions.list().pop().unwrap();
13245 let choice = if retry {
13246 "daemon: recheck and retry"
13247 } else {
13248 "daemon: hold the task"
13249 };
13250 assert_eq!(q.actions.len(), 2);
13251 q.answer(ask::Answer::Choice(choice.into())).unwrap();
13252 questions.put(&mut q).unwrap();
13253
13254 questions.beat(&q.id, ask::WaiterKind::Deputy);
13256 for _ in 0..2 {
13257 resolve_blockers(&queue, &questions);
13258 apply_task_choice_actions(&queue, &questions, &home);
13259 let waiting = queue.get(&t.id).unwrap();
13260 assert_eq!(waiting.status, TaskStatus::Blocked);
13261 assert_eq!(waiting.blocked_by, [q.id.clone()]);
13262 assert!(waiting.answers.is_empty());
13263 assert!(!waiting.action_applied(&q.id));
13264 assert!(runnable(&queue).is_empty());
13265 let free = Arc::new(tokio::sync::Semaphore::new(1));
13266 assert!(claim_dispatch(&queue, &t, false, &free, &free).is_none());
13267 }
13268 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13269
13270 resolve_blockers(&queue, &questions);
13272 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
13273 apply_task_choice_actions(&queue, &questions, &home);
13274 let after = queue.get(&t.id).unwrap();
13275 assert_eq!(
13276 after.status,
13277 if retry {
13278 TaskStatus::Queued
13279 } else {
13280 TaskStatus::Held
13281 }
13282 );
13283 assert!(after.action_applied(&q.id));
13284 assert_eq!(after.operator_held(), !retry);
13285 assert_eq!(after.answers.len(), 1);
13286
13287 let mut changed = after;
13289 if retry {
13290 changed.hold_manual(Some("later hold".into()));
13291 } else {
13292 changed.release_operator("test", None);
13293 }
13294 let expected = changed.status;
13295 queue.put(&mut changed).unwrap();
13296 apply_task_choice_actions(&queue, &questions, &home);
13297 assert_eq!(queue.get(&t.id).unwrap().status, expected);
13298 }
13299 }
13300 }
13301
13302 #[test]
13303 fn apply_choice_actions_releases_the_task_pinned_to_its_run_and_only_once() {
13304 let dir = tempfile::tempdir().unwrap();
13305 let queue = Queue::at(dir.path().join("queue"));
13306 let questions = Questions::at(dir.path().join("questions"));
13307 let home = dir.path().join("home");
13308 let mut state = run_state(RunStatus::Blocked);
13309 state.id = "20260101-000000-act1".to_owned();
13310 state.save_under(&home).unwrap();
13311
13312 let mut t = held_task_with(&state.id);
13313 queue.put(&mut t).unwrap();
13314 let mut q = action_question(&state.id, resume_action(&state.id));
13315 questions.put(&mut q).unwrap();
13316
13317 apply_task_choice_actions(&queue, &questions, &home);
13318 let after = queue.get(&t.id).unwrap();
13319 assert_eq!(after.status, TaskStatus::Queued);
13320 assert!(!after.fresh_start);
13321 assert!(after.action_applied(&q.id));
13322 let pin = after.resume_override.clone().unwrap();
13323 assert_eq!(pin.pinned_run.as_deref(), Some(state.id.as_str()));
13324 assert!(pin.forced);
13325
13326 let mut again = queue.get(&t.id).unwrap();
13328 again.hold_machine(Some("later".into()));
13329 queue.put(&mut again).unwrap();
13330 apply_task_choice_actions(&queue, &questions, &home);
13331 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13332 }
13333
13334 #[test]
13335 fn a_delivered_answer_is_still_acted_on_exactly_once() {
13336 let dir = tempfile::tempdir().unwrap();
13337 let queue = Queue::at(dir.path().join("queue"));
13338 let questions = Questions::at(dir.path().join("questions"));
13339 let home = dir.path().join("home");
13340 let mut state = run_state(RunStatus::Blocked);
13341 state.id = "20260101-000000-act2".to_owned();
13342 state.save_under(&home).unwrap();
13343
13344 let mut t = held_task_with(&state.id);
13345 queue.put(&mut t).unwrap();
13346 let mut q = action_question(&state.id, resume_action(&state.id));
13347 q.answer_delivered = true;
13348 questions.put(&mut q).unwrap();
13349
13350 apply_task_choice_actions(&queue, &questions, &home);
13353 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13354 assert!(queue.get(&t.id).unwrap().action_applied(&q.id));
13355
13356 let mut again = queue.get(&t.id).unwrap();
13358 again.hold_machine(Some("later".into()));
13359 queue.put(&mut again).unwrap();
13360 apply_task_choice_actions(&queue, &questions, &home);
13361 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13362 }
13363
13364 #[test]
13365 fn a_fresh_asker_defers_the_action_until_it_goes_quiet() {
13366 let dir = tempfile::tempdir().unwrap();
13367 let queue = Queue::at(dir.path().join("queue"));
13368 let questions = Questions::at(dir.path().join("questions"));
13369 let home = dir.path().join("home");
13370 let mut state = run_state(RunStatus::Blocked);
13371 state.id = "20260101-000000-act3".to_owned();
13372 state.save_under(&home).unwrap();
13373 let mut t = held_task_with(&state.id);
13374 queue.put(&mut t).unwrap();
13375 let mut q = action_question(&state.id, ask::ChoiceAction::Requeue);
13376 questions.put(&mut q).unwrap();
13377
13378 questions.beat(&q.id, crate::ask::WaiterKind::Asker);
13379 apply_task_choice_actions(&queue, &questions, &home);
13380 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13381
13382 std::fs::remove_file(questions.root().join(format!("{}.lease", q.id))).unwrap();
13383 apply_task_choice_actions(&queue, &questions, &home);
13384 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
13385 }
13386
13387 #[test]
13388 fn action_standing_tells_the_reasons_apart() {
13389 let mut t = held_task_with("r1");
13390 let q = action_question("r1", ask::ChoiceAction::Requeue);
13391 assert_eq!(action_standing(&t, &q), ActionStanding::Pending);
13392 let mut plain = q.clone();
13393 plain.actions.clear();
13394 assert_eq!(action_standing(&t, &plain), ActionStanding::NoAction);
13395
13396 t.mark_action_applied(&q.id);
13398 assert_eq!(action_standing(&t, &q), ActionStanding::Applied);
13399
13400 let mut t = held_task_with("r1");
13402 t.start("r2".to_owned());
13403 assert_eq!(action_standing(&t, &q), ActionStanding::Stale);
13404 let q2 = action_question("r2", ask::ChoiceAction::Requeue);
13406 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13407 t.status = TaskStatus::Blocked;
13408 assert_eq!(action_standing(&t, &q2), ActionStanding::Busy);
13409
13410 let mut c = action_question("r0", ask::ChoiceAction::Requeue);
13412 c.node = crate::conduct::NODE.to_owned();
13413 assert_eq!(action_standing(&t, &c), ActionStanding::Busy);
13414 assert!(!ActionStanding::Busy.daemon_owns());
13415 }
13416
13417 #[test]
13418 fn a_running_task_waits_and_a_dead_asker_with_a_cwd_is_still_actioned() {
13419 let dir = tempfile::tempdir().unwrap();
13420 let queue = Queue::at(dir.path().join("queue"));
13421 let questions = Questions::at(dir.path().join("questions"));
13422 let home = dir.path().join("home");
13423 let mut state = run_state(RunStatus::Blocked);
13424 state.id = "20260101-000000-act4".to_owned();
13425 state.save_under(&home).unwrap();
13426 let mut t = held_task_with(&state.id);
13427 t.status = TaskStatus::Running;
13428 queue.put(&mut t).unwrap();
13429 let mut q = action_question(&state.id, ask::ChoiceAction::Done);
13430 q.cwd = Some(dir.path().display().to_string());
13431 questions.put(&mut q).unwrap();
13432
13433 let running = queue.get(&t.id).unwrap();
13436 assert_eq!(action_standing(&running, &q), ActionStanding::Busy);
13437 assert!(matches!(
13438 crate::waiter::decide_owned(&q, None, false, 86_400, Timestamp::now(), false),
13439 crate::waiter::Action::Deliver(_)
13440 ));
13441 apply_task_choice_actions(&queue, &questions, &home);
13443 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
13444 let mut back = queue.get(&t.id).unwrap();
13445 back.hold_machine(Some("later".into()));
13446 queue.put(&mut back).unwrap();
13447 apply_task_choice_actions(&queue, &questions, &home);
13448 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Done);
13449 let mut again = queue.get(&t.id).unwrap();
13450 again.hold_machine(Some("again".into()));
13451 queue.put(&mut again).unwrap();
13452 apply_task_choice_actions(&queue, &questions, &home);
13453 assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
13454 }
13455}